Эх сурвалжийг харах

HUE-4886 [impala] Parse unabbreviated count for Impala returned rows

Also correctly use HTTPS if SSL enabled
spaztic1215 9 жил өмнө
parent
commit
9ad58192a4

+ 0 - 3
apps/impala/src/impala/tests.py

@@ -29,10 +29,8 @@ import desktop.conf as desktop_conf
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group
 from desktop.models import Document
-from hadoop import cluster
 from hadoop.pseudo_hdfs4 import get_db_prefix, is_live_cluster
 
-from beeswax import hive_site
 from beeswax.design import hql_query
 from beeswax.models import SavedQuery, QueryHistory
 from beeswax.server import dbms
@@ -41,7 +39,6 @@ from beeswax.tests import _make_query
 
 from impala import conf
 from impala.dbms import ImpalaDbms
-from impala.server import ImpalaServerClient
 
 
 LOG = logging.getLogger(__name__)

+ 3 - 4
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -47,7 +47,6 @@ try:
   from beeswax.conf import CONFIG_WHITELIST as hive_settings, DOWNLOAD_CELL_LIMIT
   from beeswax.data_export import upload
   from beeswax.design import hql_query, strip_trailing_semicolon, split_statements
-  from beeswax.hive_site import hiveserver2_use_ssl
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
   from beeswax.server import dbms
   from beeswax.server.dbms import get_query_server_config, QueryServerException
@@ -58,7 +57,7 @@ except ImportError, e:
 
 try:
   from impala import api   # Force checking if Impala is enabled
-  from impala.conf import CONFIG_WHITELIST as impala_settings
+  from impala.conf import CONFIG_WHITELIST as impala_settings, SSL as impala_ssl_conf
   from impala.server import get_api as get_impalad_api, ImpalaDaemonApiException
 except ImportError, e:
   LOG.warn("Impala app is not enabled")
@@ -703,12 +702,12 @@ class HS2Api(Api):
 
     query_id = self._get_impala_query_id(snippet)
     session = Session.objects.get_session(self.user, application='impala')
-    protocol = 'https' if hiveserver2_use_ssl() else 'http'
+    protocol = 'https' if impala_ssl_conf.get().get('enabled') else 'http'
     server_url = '%s://%s' % (protocol, self._get_impala_server_url(session))
     if query_id:
       LOG.info("Attempting to get Impala query profile at server_url %s for query ID: %s" % (server_url, query_id))
       fragment = self._get_impala_query_profile(server_url, query_id=query_id)
-      total_records_re = "Coordinator Fragment F\d\d.+?RowsReturned: (?P<total_records>\d+).*?Averaged Fragment F\d\d"
+      total_records_re = "Coordinator Fragment F\d\d.+?RowsReturned: \d+(?:.\d+[KMB])? \((?P<total_records>\d+)\).*?Averaged Fragment F\d\d"
       total_records_match = re.search(total_records_re, fragment, re.MULTILINE | re.DOTALL)
     if total_records_match:
       total_records = int(total_records_match.group('total_records'))

+ 29 - 2
desktop/libs/notebook/src/notebook/connectors/tests/tests_hiveserver2.py

@@ -444,6 +444,7 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
     self.client.post('/beeswax/install_examples')
 
     self.user = User.objects.get(username='test')
+    add_to_group('test')
     grant_access("test", "test", "notebook")
 
     self.db = dbms.get(self.user, get_query_server_config())
@@ -733,10 +734,12 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
     statement = "SELECT app, COUNT(1) AS count FROM web_logs GROUP BY app ORDER BY count DESC;"
     doc = self.create_query_document(owner=self.user, query_type='impala', statement=statement)
     notebook = Notebook(document=doc)
-    snippet = self.execute_and_wait(doc, snippet_idx=0, timeout=60.0, wait=2.0)
 
+    snippet = self.execute_and_wait(doc, snippet_idx=0, timeout=60.0, wait=2.0)
+    self.client.post(reverse('notebook:fetch_result_data'),
+                     {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet), 'rows': 100, 'startOver': 'false'})
     response = self.client.post(reverse('notebook:fetch_result_size'),
-                              {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
+                                {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
 
     data = json.loads(response.content)
     assert_equal(0, data['status'], data)
@@ -745,3 +748,27 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
     assert_true('size' in data['result'])
     assert_equal(23, data['result']['rows'])
     assert_equal(None, data['result']['size'])
+
+
+  def test_fetch_result_abbreviated(self):
+    if not is_live_cluster():
+      raise SkipTest
+
+    # Assert that abbreviated rows returned (e.g. - 1.00K) still returns actual rows
+    statement = "SELECT * FROM web_logs;"
+    doc = self.create_query_document(owner=self.user, query_type='impala', statement=statement)
+    notebook = Notebook(document=doc)
+    snippet = self.execute_and_wait(doc, snippet_idx=0, timeout=60.0, wait=2.0)
+
+    self.client.post(reverse('notebook:fetch_result_data'),
+                     {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet), 'rows': 100,
+                      'startOver': 'false'})
+
+    response = self.client.post(reverse('notebook:fetch_result_size'),
+                              {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
+
+    data = json.loads(response.content)
+    assert_equal(0, data['status'], data)
+    assert_true('result' in data)
+    assert_true('rows' in data['result'])
+    assert_equal(1000, data['result']['rows'])