浏览代码

HUE-8931 [impala] Invalidate API has a wrong parameter

Romain 6 年之前
父节点
当前提交
137958f0a2

+ 23 - 34
apps/beeswax/src/beeswax/server/dbms.py

@@ -92,27 +92,8 @@ def get(user, query_server=None, cluster=None):
 
 
 def get_query_server_config(name='beeswax', connector=None):
-  if connector and has_connectors(): # FIXME: Currently this doesn't work when has_connectors is False because options is empty
-    connector_name = full_connector_name = connector['type']
-    compute_name = None
-    if connector.get('compute'):
-      compute_name = connector['compute']['name']
-      full_connector_name = '%s-%s' % (connector_name, compute_name)
-    LOG.debug("Query cluster connector %s compute %s" % (connector_name, compute_name))
-
-    query_server = {
-        'server_name': full_connector_name,
-        'server_host': (connector['compute']['options'] if 'compute' in connector else connector['options'])['server_host'],
-        'server_port': int((connector['compute']['options'] if 'compute' in connector else connector['options'])['server_port']),
-        'principal': 'TODO',
-        'auth_username': AUTH_USERNAME.get(),
-        'auth_password': AUTH_PASSWORD.get(),
-
-        'impersonation_enabled': False, # TODO, Impala only, to add to connector class
-        'SESSION_TIMEOUT_S': 15 * 60,
-        'querycache_rows': 1000,
-        'QUERY_TIMEOUT_S': 15 * 60,
-    }
+  if connector and has_connectors(): # TODO: Give empty connector when no connector in use
+    query_server = get_query_server_config_via_connector(connector)
   else:
     LOG.debug("Query cluster %s" % name)
     if name == "llap":
@@ -213,19 +194,27 @@ def get_query_server_config(name='beeswax', connector=None):
   return query_server
 
 
-def get_cluster_config(cluster=None):
-  if cluster and cluster.get('connector'): # Connector interface
-    cluster_config = cluster
-  elif cluster and cluster.get('id') != CLUSTER_ID.get():
-    if 'altus:dataware:k8s' in cluster['id']:
-      compute_end_point = cluster['compute_end_point'][0] if type(cluster['compute_end_point']) == list else cluster['compute_end_point'] # TODO getting list from left assist
-      cluster_config = {'server_host': compute_end_point, 'name': cluster['name']} # TODO get port too
-    else:
-      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster # Deprecated
-  else:
-    cluster_config = None
-
-  return cluster_config
+def get_query_server_config_via_connector(connector):
+  connector_name = full_connector_name = connector['type']
+  compute_name = None
+  if connector.get('compute'):
+    compute_name = connector['compute']['name']
+    full_connector_name = '%s-%s' % (connector_name, compute_name)
+  LOG.debug("Query cluster connector %s compute %s" % (connector_name, compute_name))
+
+  return {
+      'server_name': full_connector_name,
+      'server_host': (connector['compute']['options'] if 'compute' in connector else connector['options'])['server_host'],
+      'server_port': int((connector['compute']['options'] if 'compute' in connector else connector['options'])['server_port']),
+      'principal': 'TODO',
+      'auth_username': AUTH_USERNAME.get(),
+      'auth_password': AUTH_PASSWORD.get(),
+
+      'impersonation_enabled': False, # TODO, Impala only, to add to connector class
+      'SESSION_TIMEOUT_S': 15 * 60,
+      'querycache_rows': 1000,
+      'QUERY_TIMEOUT_S': 15 * 60,
+  }
 
 
 class QueryServerException(Exception):

+ 3 - 4
apps/impala/src/impala/api.py

@@ -28,7 +28,6 @@ from django.views.decorators.http import require_POST
 from beeswax.api import error_handler
 from beeswax.models import Session
 from beeswax.server import dbms as beeswax_dbms
-from beeswax.server.dbms import get_cluster_config
 from beeswax.views import authorized_get_query_history
 
 from desktop.lib.django_util import JsonResponse
@@ -43,6 +42,7 @@ from libanalyze import analyze as analyzer, rules
 
 from notebook.models import make_notebook
 
+
 LOG = logging.getLogger(__name__)
 ANALYZER = rules.TopDownAnalysis() # We need to parse some files so save as global
 
@@ -55,8 +55,7 @@ def invalidate(request):
   table = request.POST.get('table', None)
   flush_all = request.POST.get('flush_all', 'false').lower() == 'true'
 
-  cluster_config = get_cluster_config(cluster)
-  query_server = dbms.get_query_server_config(cluster_config=cluster_config)
+  query_server = dbms.get_query_server_config(connector=None) # TODO: connector support
   db = beeswax_dbms.get(request.user, query_server=query_server)
 
   response = {'status': 0, 'message': ''}
@@ -192,4 +191,4 @@ def alanize_fix(request):
     response['details'] = { 'task': notebook.execute(request, batch=True) }
     response['status'] = 0
 
-  return JsonResponse(response)
+  return JsonResponse(response)

+ 58 - 0
apps/impala/src/impala/api_tests.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from builtins import object
+import json
+import logging
+
+from django.urls import reverse
+from nose.tools import assert_true, assert_equal, assert_false, assert_raises
+from mock import patch, Mock
+
+from desktop.lib.django_test_util import make_logged_in_client
+
+from impala import conf
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestImpala(object):
+
+  def setUp(self):
+    self.client = make_logged_in_client()
+
+  def test_invalidate(self):
+    with patch('impala.api.beeswax_dbms') as beeswax_dbms:
+      invalidate = Mock()
+      beeswax_dbms.get = Mock(
+        return_value=Mock(invalidate=invalidate)
+      )
+
+      response = self.client.post(reverse("impala:invalidate"), {
+          'flush_all': False,
+          'cluster': json.dumps({"credentials":{},"type":"direct","id":"default","name":"default"}),
+          'database': 'default',
+          'table': 'k8s_logs'
+        }
+      )
+
+      invalidate.assert_called()
+
+      assert_equal(response.status_code, 200)
+      content = json.loads(response.content)
+      assert_equal(content['message'], 'Successfully invalidated metadata')

+ 22 - 20
apps/impala/src/impala/dbms.py

@@ -19,37 +19,39 @@ import logging
 
 from django.utils.translation import ugettext as _
 
-from beeswax.design import hql_query
-from beeswax.models import QUERY_TYPES
-from beeswax.server import dbms
-from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServerTimeoutException,\
-  get_query_server_config as beeswax_query_server_config
-
 from desktop.conf import CLUSTER_ID
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
 from desktop.models import Cluster, ClusterConfig
+from beeswax.design import hql_query
+from beeswax.models import QUERY_TYPES
+from beeswax.server import dbms
+from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServerTimeoutException,\
+  get_query_server_config as beeswax_query_server_config, get_query_server_config_via_connector
+from notebook.conf import get_ordered_interpreters
 
 from impala import conf
-from notebook.conf import get_ordered_interpreters
 
 
 LOG = logging.getLogger(__name__)
 
 
-def get_query_server_config():
-  query_server = {
-      'server_name': 'impala',
-      'server_host': conf.SERVER_HOST.get(),
-      'server_port': conf.SERVER_PORT.get(),
-      'principal': conf.IMPALA_PRINCIPAL.get(),
-      'impersonation_enabled': conf.IMPERSONATION_ENABLED.get(),
-      'querycache_rows': conf.QUERYCACHE_ROWS.get(),
-      'QUERY_TIMEOUT_S': conf.QUERY_TIMEOUT_S.get(),
-      'SESSION_TIMEOUT_S': conf.SESSION_TIMEOUT_S.get(),
-      'auth_username': conf.AUTH_USERNAME.get(),
-      'auth_password': conf.AUTH_PASSWORD.get()
-  }
+def get_query_server_config(connector=None):
+  if connector and has_connectors():
+    query_server = get_query_server_config_via_connector(connector)
+  else:
+    query_server = {
+        'server_name': 'impala',
+        'server_host': conf.SERVER_HOST.get(),
+        'server_port': conf.SERVER_PORT.get(),
+        'principal': conf.IMPALA_PRINCIPAL.get(),
+        'impersonation_enabled': conf.IMPERSONATION_ENABLED.get(),
+        'querycache_rows': conf.QUERYCACHE_ROWS.get(),
+        'QUERY_TIMEOUT_S': conf.QUERY_TIMEOUT_S.get(),
+        'SESSION_TIMEOUT_S': conf.SESSION_TIMEOUT_S.get(),
+        'auth_username': conf.AUTH_USERNAME.get(),
+        'auth_password': conf.AUTH_PASSWORD.get()
+    }
 
   debug_query_server = query_server.copy()
   debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))

+ 1 - 0
apps/impala/src/impala/tests.py

@@ -132,6 +132,7 @@ class TestMockedImpala(object):
         assert_true(ddms.client.query.call_count == 3) # Third call
         assert_true('customers' not in ddms.client.query.call_args[0][0].hql_query) # Full invalidate
 
+
 class TestImpalaIntegration(object):
   integration = True
 

+ 1 - 0
desktop/core/src/desktop/lib/django_util.py

@@ -41,6 +41,7 @@ import desktop.lib.thrift_util
 from desktop.lib import django_mako
 from desktop.lib.json_utils import JSONEncoderForHTML
 
+
 LOG = logging.getLogger(__name__)
 
 # Values for template_lib parameter

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -327,7 +327,7 @@ def get_api(request, snippet):
     if compute == '""' or compute == 'undefined':
       compute = None
     if not compute and snippet.get('compute'): # Via notebook.ko.js
-      interpreter['compute'] =  snippet['compute']
+      interpreter['compute'] = snippet['compute']
 
   LOG.debug('Selected interpreter %s interface=%s compute=%s' % (
     interpreter['type'],

+ 0 - 41
desktop/libs/notebook/src/notebook/sql_utils_test.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from beeswax.design import hql_query
-from notebook.sql_utils import strip_trailing_semicolon, split_statements
-
-from nose.tools import assert_equal
-
-def test_split_statements():
-  assert_equal([''], hql_query(";;;").statements)
-  assert_equal(["select * where id == '10'"], hql_query("select * where id == '10'").statements)
-  assert_equal(["select * where id == '10'"], hql_query("select * where id == '10';").statements)
-  assert_equal(['select', "select * where id == '10;' limit 100"], hql_query("select; select * where id == '10;' limit 100;").statements)
-  assert_equal(['select', "select * where id == \"10;\" limit 100"], hql_query("select; select * where id == \"10;\" limit 100;").statements)
-  assert_equal(['select', "select * where id == '\"10;\"\"\"' limit 100"], hql_query("select; select * where id == '\"10;\"\"\"' limit 100;").statements)
-
-def teststrip_trailing_semicolon():
-  # Note that there are two queries (both an execute and an explain) scattered
-  # in this file that use semicolons all the way through.
-
-  # Single semicolon
-  assert_equal("foo", strip_trailing_semicolon("foo;\n"))
-  assert_equal("foo\n", strip_trailing_semicolon("foo\n;\n\n\n"))
-  # Multiple semicolons: strip only last one
-  assert_equal("fo;o;", strip_trailing_semicolon("fo;o;;     "))
-  # No semicolons
-  assert_equal("foo", strip_trailing_semicolon("foo"))