소스 검색

HUE-8932 [impala] Add use_sasl configuration

Jean-Francois Desjeans Gauthier 6 년 전
부모
커밋
3cc0f4a118

+ 15 - 0
apps/beeswax/src/beeswax/conf.py

@@ -293,3 +293,18 @@ AUTH_PASSWORD_SCRIPT = Config(
   private=True,
   type=coerce_password_from_script,
   default=None)
+
+def get_use_sasl_default():
+  """Get from hive_site or backward compatibility"""
+  from hive_site import get_hiveserver2_authentication, get_use_sasl  # Cyclic dependency
+  use_sasl = get_use_sasl()
+  if use_sasl is not None:
+    return use_sasl.upper() == 'TRUE'
+  return get_hiveserver2_authentication() in ('KERBEROS', 'NONE', 'LDAP', 'PAM') # list for backward compatibility
+
+USE_SASL = Config(
+  key="use_sasl",
+  help=_t("Use SASL framework to establish connection to host"),
+  private=False,
+  type=coerce_bool,
+  dynamic_default=get_use_sasl_default)

+ 5 - 0
apps/beeswax/src/beeswax/hive_site.py

@@ -58,6 +58,8 @@ _CNF_HIVESERVER2_THRIFT_HTTP_PATH = 'hive.server2.thrift.http.path'
 
 _CNF_HIVESERVER2_THRIFT_SASL_QOP = 'hive.server2.thrift.sasl.qop'
 
+_CNF_HIVESERVER2_USE_SASL = 'hive.metastore.sasl.enabled'
+
 
 # Host is whatever up to the colon. Allow and ignore a trailing slash.
 _THRIFT_URI_RE = re.compile("^thrift://([^:]+):(\d+)[/]?$")
@@ -198,3 +200,6 @@ def get_hive_site_content():
     return ''
   else:
     return file(hive_site_path, 'r').read()
+
+def get_use_sasl():
+  return get_conf().get(_CNF_HIVESERVER2_USE_SASL)

+ 8 - 6
apps/beeswax/src/beeswax/server/dbms.py

@@ -43,13 +43,12 @@ from beeswax import hive_site
 from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
   AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
   HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
-  LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT
+  LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT, USE_SASL as HIVE_USE_SASL
 from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.models import QueryHistory, QUERY_TYPES
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -158,7 +157,8 @@ def get_query_server_config(name='beeswax', connector=None):
           'principal': kerberos_principal,
           'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
           'auth_username': AUTH_USERNAME.get(),
-          'auth_password': AUTH_PASSWORD.get()
+          'auth_password': AUTH_PASSWORD.get(),
+          'use_sasl': HIVE_USE_SASL.get()
       }
     else:
       kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
@@ -175,16 +175,18 @@ def get_query_server_config(name='beeswax', connector=None):
             },
           'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
           'auth_username': AUTH_USERNAME.get(),
-          'auth_password': AUTH_PASSWORD.get()
+          'auth_password': AUTH_PASSWORD.get(),
+          'use_sasl': HIVE_USE_SASL.get()
         }
 
     if name == 'sparksql': # Extends Hive as very similar
-      from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
+      from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT, USE_SASL as SPARK_USE_SASL
 
       query_server.update({
           'server_name': 'sparksql',
           'server_host': SPARK_SERVER_HOST.get(),
-          'server_port': SPARK_SERVER_PORT.get()
+          'server_port': SPARK_SERVER_PORT.get(),
+          'use_sasl': SPARK_USE_SASL.get()
       })
 
   debug_query_server = query_server.copy()

+ 1 - 5
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -34,7 +34,6 @@ from desktop.lib import thrift_util
 from desktop.conf import DEFAULT_USER
 from desktop.models import Document2
 from beeswax import conf
-from hadoop import cluster
 
 from TCLIService import TCLIService
 from TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
@@ -571,20 +570,17 @@ class HiveServerClient(object):
     else:
       kerberos_principal_short_name = None
 
+    use_sasl = self.query_server['use_sasl']
     if self.query_server['server_name'].startswith('impala'):
       if auth_password: # Force LDAP/PAM.. auth if auth_password is provided
-        use_sasl = True
         mechanism = HiveServerClient.HS2_MECHANISMS['NONE']
       else:
-        cluster_conf = cluster.get_cluster_conf_for_job_submission() # Note: Create YARN HttpClient while validate config, should directly read Impala property instead
-        use_sasl = cluster_conf is not None and cluster_conf.SECURITY_ENABLED.get()
         mechanism = HiveServerClient.HS2_MECHANISMS['KERBEROS']
       impersonation_enabled = self.query_server['impersonation_enabled']
     else:
       hive_mechanism = hive_site.get_hiveserver2_authentication()
       if hive_mechanism not in HiveServerClient.HS2_MECHANISMS:
         raise Exception(_('%s server authentication not supported. Valid are %s.') % (hive_mechanism, list(HiveServerClient.HS2_MECHANISMS.keys())))
-      use_sasl = hive_mechanism in ('KERBEROS', 'NONE', 'LDAP', 'PAM')
       mechanism = HiveServerClient.HS2_MECHANISMS[hive_mechanism]
       impersonation_enabled = hive_site.hiveserver2_impersonation_enabled()
 

+ 10 - 6
apps/beeswax/src/beeswax/tests.py

@@ -3010,7 +3010,7 @@ def test_hiveserver2_get_security():
     default_query_server = {'server_host': 'my_host', 'server_port': 12345}
 
     # Beeswax
-    beeswax_query_server = {'server_name': 'beeswax', 'principal': 'hive', 'auth_username': 'hue', 'auth_password': None}
+    beeswax_query_server = {'server_name': 'beeswax', 'principal': 'hive', 'auth_username': 'hue', 'auth_password': None, 'use_sasl': True}
     beeswax_query_server.update(default_query_server)
     assert_equal((True, 'PLAIN', 'hive', True, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
 
@@ -3019,22 +3019,25 @@ def test_hiveserver2_get_security():
     assert_equal((True, 'PLAIN', 'hive', True, 'hueabcd', 'abcd'), HiveServerClient(beeswax_query_server, user).get_security())
     beeswax_query_server.update({'auth_username': 'hue', 'auth_password': None})
 
-    hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = 'NOSASL'
-    hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_IMPERSONATION] = 'false'
-    assert_equal((False, 'NOSASL', 'hive', False, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
     hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = 'KERBEROS'
+    hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_IMPERSONATION] = 'false'
     assert_equal((True, 'GSSAPI', 'hive', False, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
 
+    hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = 'NOSASL'
+    beeswax_query_server.update({'use_sasl': False})
+    assert_equal((False, 'NOSASL', 'hive', False, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
+
+
     # Impala
     cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
 
     finish = cluster_conf.SECURITY_ENABLED.set_for_testing(False)
     try:
-      impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': False, 'auth_username': 'hue', 'auth_password': None}
+      impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': False, 'auth_username': 'hue', 'auth_password': None, 'use_sasl': False}
       impala_query_server.update(default_query_server)
       assert_equal((False, 'GSSAPI', 'impala', False, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
 
-      impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': True, 'auth_username': 'hue', 'auth_password': None}
+      impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': True, 'auth_username': 'hue', 'auth_password': None, 'use_sasl': False}
       impala_query_server.update(default_query_server)
       assert_equal((False, 'GSSAPI', 'impala', True, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
     finally:
@@ -3042,6 +3045,7 @@ def test_hiveserver2_get_security():
 
     finish = cluster_conf.SECURITY_ENABLED.set_for_testing(True)
     try:
+      impala_query_server.update({'use_sasl': True})
       assert_equal((True, 'GSSAPI', 'impala', True, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
     finally:
       finish()

+ 13 - 1
apps/impala/src/impala/conf.py

@@ -27,7 +27,7 @@ from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_csv, coe
 from desktop.lib.exceptions import StructuredThriftTransportException
 from desktop.lib.paths import get_desktop_root
 
-from impala.impala_flags import get_max_result_cache_size, is_impersonation_enabled
+from impala.impala_flags import get_max_result_cache_size, is_impersonation_enabled, is_kerberos_enabled
 from impala.settings import NICE_NAME
 
 
@@ -221,6 +221,18 @@ DAEMON_API_USERNAME = Config(
   dynamic_default=get_daemon_api_username
 )
 
+def get_use_sasl_default():
+  """kerberos enabled or password is specified"""
+  return is_kerberos_enabled() or AUTH_PASSWORD.get() is not None # Maps closely to legacy behavior
+
+USE_SASL = Config(
+  key="use_sasl",
+  help=_t("Use SASL framework to establish connection to host."),
+  private=False,
+  type=coerce_bool,
+  dynamic_default=get_use_sasl_default
+)
+
 
 def config_validator(user):
   # dbms is dependent on beeswax.conf (this file)

+ 2 - 1
apps/impala/src/impala/dbms.py

@@ -50,7 +50,8 @@ def get_query_server_config(connector=None):
         'QUERY_TIMEOUT_S': conf.QUERY_TIMEOUT_S.get(),
         'SESSION_TIMEOUT_S': conf.SESSION_TIMEOUT_S.get(),
         'auth_username': conf.AUTH_USERNAME.get(),
-        'auth_password': conf.AUTH_PASSWORD.get()
+        'auth_password': conf.AUTH_PASSWORD.get(),
+        'use_sasl': conf.USE_SASL.get()
     }
 
   debug_query_server = query_server.copy()

+ 4 - 0
apps/impala/src/impala/impala_flags.py

@@ -27,6 +27,7 @@ _WEBSERVER_CERTIFICATE_FILE = '-webserver_certificate_file'
 _SSL_SERVER_CERTIFICATE = '-ssl_server_certificate'
 _MAX_RESULT_CACHE_SIZE = '-max_result_cache_size'
 _AUTHORIZED_PROXY_USER_CONFIG = '-authorized_proxy_user_config'
+_PRINCIPAL = '-principal'
 
 
 def reset():
@@ -75,6 +76,9 @@ def is_impersonation_enabled():
   user_config = get_conf().get(_AUTHORIZED_PROXY_USER_CONFIG)
   return True if user_config and 'hue=' in user_config else False
 
+def is_kerberos_enabled():
+  return get_conf().get(_PRINCIPAL) is not None
+
 def _parse_impala_flags():
   from impala import conf # Cyclic dependency
   global _IMPALA_FLAGS

+ 7 - 0
apps/spark/src/spark/conf.py

@@ -83,6 +83,13 @@ SSL_CERT_CA_VERIFY = Config(
   type=coerce_bool
 )
 
+USE_SASL = Config(
+  key="use_sasl",
+  help=_t("Use SASL framework to establish connection to host."),
+  default=False,
+  type=coerce_bool
+)
+
 
 def get_livy_server_url():
   url = LIVY_SERVER_URL.get()

+ 8 - 1
desktop/conf.dist/hue.ini

@@ -1240,6 +1240,9 @@
   ## auth_username=hue
   ## auth_password=
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=false
+
   [[ssl]]
     # Path to Certificate Authority certificates.
     ## cacerts=/etc/hue/cacerts.pem
@@ -1247,7 +1250,6 @@
     # Choose whether Hue should validate certificates received from the server.
     ## validate=true
 
-
 ###########################################################################
 # Settings to configure Metastore
 ###########################################################################
@@ -1315,6 +1317,9 @@
   # Path to the impala configuration dir which has impalad_flags file
   ## impala_conf_dir=${HUE_CONF_DIR}/impala-conf
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=true
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false
@@ -1352,6 +1357,8 @@
   # Choose whether Hue should validate certificates received from the server.
   ## ssl_cert_ca_verify=true
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=false
 
 ###########################################################################
 # Settings to configure the Oozie app

+ 8 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1225,6 +1225,9 @@
   ## auth_username=hue
   ## auth_password=
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=false
+
   [[ssl]]
     # Path to Certificate Authority certificates.
     ## cacerts=/etc/hue/cacerts.pem
@@ -1300,6 +1303,9 @@
   # Path to the impala configuration dir which has impalad_flags file
   ## impala_conf_dir=${HUE_CONF_DIR}/impala-conf
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=true
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false
@@ -1337,6 +1343,8 @@
   # Choose whether Hue should validate certificates received from the server.
   ## ssl_cert_ca_verify=true
 
+  # Use SASL framework to establish connection to host.
+  ## use_sasl=false
 
 ###########################################################################
 # Settings to configure the Oozie app