Explorar o código

HUE-8758 [impala] Do not hardcode impersonation setting to true

Still leveraging a local Impala flag, but preparing for adding
option in the UI.
Romain %!s(int64=5) %!d(string=hai) anos
pai
achega
4e90cd052b

+ 36 - 15
apps/beeswax/src/beeswax/server/dbms.py

@@ -38,14 +38,16 @@ from desktop.settings import CACHES_HIVE_DISCOVERY_KEY
 from indexer.file_format import HiveFormat
 from libzookeeper import conf as libzookeeper_conf
 
-from beeswax import hive_site
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_SERVER_HOST, HIVE_HTTP_THRIFT_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
-  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
-  HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
-  LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT, USE_SASL as HIVE_USE_SASL, CLOSE_SESSIONS, has_session_pool, MAX_NUMBER_OF_SESSIONS
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_SERVER_HOST, HIVE_HTTP_THRIFT_PORT, HIVE_METASTORE_HOST, \
+    HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
+    AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
+    HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
+    LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT, USE_SASL as HIVE_USE_SASL, CLOSE_SESSIONS, has_session_pool, \
+    MAX_NUMBER_OF_SESSIONS
 from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
-from beeswax.hive_site import hiveserver2_use_ssl
+from beeswax.hive_site import hiveserver2_use_ssl, hiveserver2_impersonation_enabled, get_hiveserver2_kerberos_principal, \
+    hiveserver2_transport_mode, hiveserver2_thrift_http_path
 from beeswax.models import QueryHistory, QUERY_TYPES
 
 
@@ -126,7 +128,14 @@ def get_query_server_config(name='beeswax', connector=None):
               for server in hiveservers:
                 llap_servers= json.loads(zk.get("{0}/{1}".format(znode, server))[0])["internal"][0]
                 if llap_servers["api"] == "activeEndpoint":
-                  cache.set("llap", json.dumps({"host": llap_servers["addresses"][0]["host"], "port": llap_servers["addresses"][0]["port"]}), CACHE_TIMEOUT.get())
+                  cache.set(
+                    "llap",
+                    json.dumps({
+                        "host": llap_servers["addresses"][0]["host"],
+                        "port": llap_servers["addresses"][0]["port"]
+                      }),
+                      CACHE_TIMEOUT.get()
+                  )
             else:
               LOG.error("LLAP Endpoint not found, reverting to HiveServer2")
               cache.set("llap", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}), CACHE_TIMEOUT.get())
@@ -136,7 +145,13 @@ def get_query_server_config(name='beeswax', connector=None):
             if zk.exists(znode):
               hiveservers = zk.get_children(znode)
               for server in hiveservers:
-                cache.set("llap", json.dumps({"host": server.split(';')[0].split('=')[1].split(":")[0], "port": server.split(';')[0].split('=')[1].split(":")[1]}))
+                cache.set(
+                  "llap",
+                  json.dumps({
+                    "host": server.split(';')[0].split('=')[1].split(":")[0],
+                    "port": server.split(';')[0].split('=')[1].split(":")[1]
+                  })
+                )
           zk.stop()
         else:
           LOG.debug("Zookeeper Discovery not enabled, reverting to config values")
@@ -153,7 +168,13 @@ def get_query_server_config(name='beeswax', connector=None):
           if zk.exists(znode):
             hiveservers = zk.get_children(znode)
             server_to_use = 0 # if CONF.HIVE_SPREAD.get() randint(0, len(hiveservers)-1) else 0
-            cache.set("hiveserver2", json.dumps({"host": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0], "port": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[1]}))
+            cache.set(
+              "hiveserver2",
+              json.dumps({
+                "host": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0],
+                "port": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[1]
+              })
+            )
           else:
             cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}))
           zk.stop()
@@ -165,19 +186,19 @@ def get_query_server_config(name='beeswax', connector=None):
       from impala.dbms import get_query_server_config as impala_query_server_config
       query_server = impala_query_server_config()
     elif name == 'hms':
-      kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
+      kerberos_principal = get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
       query_server = {
           'server_name': 'hms',
           'server_host': HIVE_METASTORE_HOST.get() if not cluster_config else cluster_config.get('server_host'),
           'server_port': HIVE_METASTORE_PORT.get(),
           'principal': kerberos_principal,
-          'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
+          'transport_mode': 'http' if hiveserver2_transport_mode() == 'HTTP' else 'socket',
           'auth_username': AUTH_USERNAME.get(),
           'auth_password': AUTH_PASSWORD.get(),
           'use_sasl': HIVE_USE_SASL.get()
       }
     else:
-      kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
+      kerberos_principal = get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
       query_server = {
           'server_name': 'beeswax',
           'server_host': activeEndpoint["host"],
@@ -187,9 +208,9 @@ def get_query_server_config(name='beeswax', connector=None):
               'protocol': 'https' if hiveserver2_use_ssl() else 'http',
               'host': activeEndpoint["host"],
               'port': activeEndpoint["port"],
-              'end_point': hive_site.hiveserver2_thrift_http_path()
+              'end_point': hiveserver2_thrift_http_path()
             },
-          'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
+          'transport_mode': 'http' if hiveserver2_transport_mode() == 'HTTP' else 'socket',
           'auth_username': AUTH_USERNAME.get(),
           'auth_password': AUTH_PASSWORD.get(),
           'use_sasl': HIVE_USE_SASL.get(),
@@ -236,7 +257,7 @@ def get_query_server_config_via_connector(connector):
       'auth_username': AUTH_USERNAME.get(),
       'auth_password': AUTH_PASSWORD.get(),
 
-      'impersonation_enabled': connector['dialect'] in ('impala',),
+      'impersonation_enabled': hiveserver2_impersonation_enabled(),
       'use_sasl': connector['dialect'] in ('hive',),
       'SESSION_TIMEOUT_S': 15 * 60,
       'querycache_rows': 1000,

+ 4 - 3
apps/impala/src/impala/dbms.py

@@ -19,14 +19,14 @@ import logging
 
 from django.utils.translation import ugettext as _
 
-from desktop.conf import CLUSTER_ID
+from desktop.conf import CLUSTER_ID, has_connectors
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
-from desktop.models import Cluster, ClusterConfig
+from desktop.models import Cluster
 from beeswax.design import hql_query
 from beeswax.models import QUERY_TYPES
 from beeswax.server import dbms
-from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServerTimeoutException,\
+from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServerTimeoutException, \
   get_query_server_config as beeswax_query_server_config, get_query_server_config_via_connector
 
 from impala import conf
@@ -38,6 +38,7 @@ LOG = logging.getLogger(__name__)
 def get_query_server_config(connector=None):
   if connector and has_connectors():
     query_server = get_query_server_config_via_connector(connector)
+    query_server['impersonation_enabled'] = conf.IMPERSONATION_ENABLED.get()  # TODO Use impersonation_enabled property
   else:
     query_server = {
         'server_name': 'impala',

+ 60 - 0
apps/impala/src/impala/dbms_tests.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import sys
+
+from nose.plugins.skip import SkipTest
+from nose.tools import assert_true, assert_equal, assert_false, assert_raises
+
+from django.urls import reverse
+
+import desktop.conf as desktop_conf
+from desktop.lib.django_test_util import make_logged_in_client
+from useradmin.models import User
+
+
+from impala.dbms import get_query_server_config
+
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock
+else:
+  from mock import patch, Mock
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestDbms():
+
+  def setUp(self):
+    self.client = make_logged_in_client()
+
+
+  def test_get_connector_config(self):
+    connector = {
+      'type': 'impala-1',
+      'dialect': 'impala',
+      'options': {'server_host': 'gethue.com', 'server_port': 10000}
+    }
+
+    with patch('impala.dbms.has_connectors') as has_connectors:
+      has_connectors.return_value = True
+
+      config = get_query_server_config(connector)
+      assert_true('impersonation_enabled' in config, config)

+ 2 - 2
apps/impala/src/impala/impala_flags.py

@@ -74,10 +74,10 @@ def get_authorized_proxy_user_config():
 
 def is_impersonation_enabled():
   """
-    Returns True if user_config config contains 'hue='
+  If hue user is allowed to send the queries as itself with an additiona doas parameter containing the logged-in user username.
   """
   user_config = get_conf().get(_AUTHORIZED_PROXY_USER_CONFIG)
-  return True if user_config and 'hue=' in user_config else False
+  return user_config and 'hue=' in user_config
 
 def default_query_option(option_name):
   query_options = get_conf().get(_DEFAULT_QUERY_OPTIONS)

+ 4 - 3
apps/impala/src/impala/test_impala_flags.py

@@ -21,10 +21,11 @@ import os
 import sys
 import tempfile
 
-from nose.tools import assert_equal
+from nose.tools import assert_equal, assert_false, assert_true
 
 from impala import conf, impala_flags
 
+
 if sys.version_info[0] > 2:
   open_file = open
 else:
@@ -44,7 +45,7 @@ def test_impala_flags():
       resets.append(conf.QUERYCACHE_ROWS.set_for_testing(expected_rows))
 
     assert_equal(conf.QUERYCACHE_ROWS.get(), expected_rows)
-    assert_equal(conf.IMPERSONATION_ENABLED.get(), False)
+    assert_false(conf.IMPERSONATION_ENABLED.get())
 
     flags = """
       -webserver_certificate_file=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem
@@ -64,7 +65,7 @@ def test_impala_flags():
 
     # From Config
     assert_equal(conf.QUERYCACHE_ROWS.get(), expected_rows)
-    assert_equal(conf.IMPERSONATION_ENABLED.get(), True)
+    assert_true(conf.IMPERSONATION_ENABLED.get())
   finally:
     impala_flags.reset()
     for reset in resets:

+ 7 - 8
apps/impala/src/impala/tests.py

@@ -49,6 +49,7 @@ if sys.version_info[0] > 2:
 else:
   from mock import patch, Mock
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -123,7 +124,7 @@ class TestMockedImpala(object):
         ddms.client.query.assert_called_once_with(ddms.client.query.call_args[0][0])
         assert_true('customers' in ddms.client.query.call_args[0][0].hql_query) # diff of 1 table
 
-        get_different_tables.return_value = ['customers','','','','','','','','','','']
+        get_different_tables.return_value = ['customers', '', '', '', '', '', '', '', '', '', '']
         assert_raises(PopupException, ddms.invalidate, 'default') # diff of 11 tables. Limit is 10.
 
         ddms.invalidate('default', 'customers')
@@ -151,13 +152,11 @@ class TestImpalaIntegration(object):
     cls.db = dbms.get(cls.user, get_query_server_config(name='impala'))
     cls.DATABASE = get_db_prefix(name='impala')
 
-    queries = ["""
-      DROP TABLE IF EXISTS %(db)s.tweets;
-    """ % {'db': cls.DATABASE}, """
-      DROP DATABASE IF EXISTS %(db)s CASCADE;
-    """ % {'db': cls.DATABASE}, """
-      CREATE DATABASE %(db)s;
-    """ % {'db': cls.DATABASE}]
+    queries = [
+      'DROP TABLE IF EXISTS %(db)s.tweets;' % {'db': cls.DATABASE},
+      'DROP DATABASE IF EXISTS %(db)s CASCADE;' % {'db': cls.DATABASE},
+      'CREATE DATABASE %(db)s;' % {'db': cls.DATABASE}
+    ]
 
     for query in queries:
        resp = _make_query(cls.client, query, database='default', local=False, server_name='impala')

+ 1 - 0
desktop/core/src/desktop/lib/connectors/types.py

@@ -59,6 +59,7 @@ CONNECTOR_TYPES = [
     'settings': [
       {'name': 'server_host', 'value': ''},
       {'name': 'server_port', 'value': ''},
+      {'name': 'impersonation_enabled', 'value': False},
     ],
     'category': 'editor',
     'description': '',