Browse Source

[core] closing older query before running new query (#2769)

* [core] closing older query before running new query

* [pylint] fixing pylint issues
Ayush Goyal 3 years ago
parent
commit
698520ad16

+ 2 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -406,7 +406,7 @@ def cancel_statement(request):
   response = {'status': -1}
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
-  snippet = json.loads(request.POST.get('snippet', '{}'))
+  snippet = {}
   operation_id = request.POST.get('operationId') or notebook['uuid']
 
   snippet = _get_snippet(request.user, notebook, snippet, operation_id)
@@ -713,7 +713,7 @@ def close_statement(request):
   response = {'status': -1}
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
-  snippet = json.loads(request.POST.get('snippet', '{}'))
+  snippet = {}
   operation_id = request.POST.get('operationId')
 
   if operation_id and not notebook.get('uuid'):

+ 119 - 48
desktop/libs/notebook/src/notebook/connectors/hiveserver2_tests.py

@@ -317,6 +317,65 @@ class TestApi():
         assert_equal(jobs[0]['url'], '')  # Is empty
 
 
+  def test_close_statement(self):
+    with patch('notebook.connectors.hiveserver2.HS2Api._get_db') as _get_db:
+      _get_db.return_value = Mock(
+        use=Mock(
+        ),
+        client=Mock(
+          query=Mock(
+            side_effect=QueryServerException(
+              Exception('Execution error!'),
+              message='Error while compiling statement: FAILED: HiveAccessControlException Permission denied'
+            )
+          ),
+        ),
+      )
+      notebook = {}
+      snippet = {
+        'id': '7ccdd296-20a3-da33-16ec-db58149aba0b', 'type': 'impala', 'status': 'running', 'statementType': 'text',
+        'statement': 'SELECT *\nFROM `default`.sample_07\nLIMIT 100\n;', 'aceCursorPosition': None, 'statementPath': '',
+        'associatedDocumentUuid': None,
+        'properties': {'settings': []},
+        'result': {
+          'id': 'd9a8dc1b-7f6d-169f-7dd7-660723cba3f4', 'type': 'table',
+          'handle': {
+            'secret': 'obUXjEDWTh+ke73YLlOlMw==', 'guid': '2iv5rEXrRE4AAAAABtXdxA==', 'operation_type': 0, 'has_result_set': True,
+            'modified_row_count': None, 'log_context': None, 'session_guid': '2440c57bc3806c6e:598514f42764cc91', 'session_id': 2094,
+            'session_type': 'impala', 'statement_id': 0, 'has_more_statements': False, 'statements_count': 1,
+            'previous_statement_hash': '39b8e5b3c37fda5ebd438da23f3e198c914750a64aa147f819a6a1e0', 'start': {'row': 0, 'column': 0},
+            'end': {'row': 0, 'column': 43}, 'statement': 'SELECT *\nFROM `default`.sample_07\nLIMIT 100'
+          }
+        }, 'database': 'default',
+        'compute': {
+          'id': 'default', 'name': 'default', 'namespace': 'default', 'interface': 'impala', 'type': 'direct', 'options': {}
+        }, 'wasBatchExecuted': False, 'dialect': 'impala'
+      }
+      api = HS2Api(self.user)
+
+      response = api.close_statement(notebook, snippet)
+      assert_equal(response['status'], 0)
+
+      snippet = {
+        'id': '7ccdd296-20a3-da33-16ec-db58149aba0b', 'type': 'impala', 'status': 'running',
+        'statementType': 'text', 'statement': 'SELECT *\nFROM `default`.sample_07\nLIMIT 100\n;',
+        'aceCursorPosition': None, 'statementPath': '', 'associatedDocumentUuid': None,
+        'properties': {'settings': []},
+        'result': {
+          'id': 'd9a8dc1b-7f6d-169f-7dd7-660723cba3f4', 'type': 'table',
+          'handle': {
+            'has_more_statements': False, 'statement_id': 0, 'statements_count': 1,
+            'previous_statement_hash': '39b8e5b3c37fda5ebd438da23f3e198c914750a64aa147f819a6a1e0'
+          }
+        }, 'database': 'default', 'compute': {'id': 'default', 'name': 'default', 'namespace': 'default',
+        'interface': 'impala', 'type': 'direct', 'options': {}}, 'wasBatchExecuted': False, 'dialect': 'impala'
+      }
+      api = HS2Api(self.user)
+      
+      response = api.close_statement(notebook, snippet)
+      assert_equal(response['status'], -1)  # snippet['result']['handel'] ['guid'] and ['secret'] are missing
+
+
   def test_get_error_message_from_query(self):
     with patch('notebook.connectors.hiveserver2.HS2Api._get_db') as _get_db:
       with patch('notebook.connectors.hiveserver2.HS2Api._get_current_statement') as _get_current_statement:
@@ -629,7 +688,8 @@ class TestHiveserver2ApiNonMock(object):
 
     assert_equal(self.api.progress({}, snippet, logs=logs), 5)
 
-    logs += """INFO  : Starting Job = job_1466104358744_0003, Tracking URL = http://jennykim-1.vpc.cloudera.com:8088/proxy/application_1466104358744_0003/
+    logs += """INFO  : Starting Job = job_1466104358744_0003, Tracking URL = """\
+      """http://jennykim-1.vpc.cloudera.com:8088/proxy/application_1466104358744_0003/
         INFO  : Kill Command = /usr/lib/hadoop/bin/hadoop job  -kill job_1466104358744_0003
         INFO  : Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 1
         INFO  : 2016-06-20 13:30:34,494 Stage-1 map = 0%,  reduce = 0%
@@ -811,7 +871,9 @@ class TestHiveserver2ApiNonMock(object):
 
 
   def test_plan_extraction_from_profile(self):
-    query_plan = self.api._get_impala_profile_plan(query_id='e147228183f1f0b3:6f086cc600000000', profile=IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE)
+    query_plan = self.api._get_impala_profile_plan(
+      query_id='e147228183f1f0b3:6f086cc600000000', profile=IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE
+    )
 
     assert_true(query_plan)
     assert_equal(IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE_PLAN, query_plan)
@@ -850,26 +912,26 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
   def create_query_document(self, owner, query_type='hive', database='default',
                             name='Test Query', description='Test Query', statement='',
                             files=None, functions=None, settings=None):
-      """
-      Creates and returns a query Document2 object
-      :param owner: owner of doc
-      :param query_type: hive, impala or spark
-      :param database: database name
-      :param name: name of document
-      :param description: description of document
-      :param statement: SQL statement (can be multi-query statement)
-      :param files: list of dicts representing files
-      :param functions: list of dicts representing functions
-      :param settings: list of dicts representing settings
-      :return: Document2 object representing query
-      """
-      if query_type not in ('hive', 'impala', 'spark'):
-          raise ValueError("Invalid query_type: %s" % query_type)
-
-      notebook = make_notebook(name=name, description=description, editor_type=query_type, statement=statement,
-                               status='ready', database=database, files=files, functions=functions, settings=settings)
-      notebook_doc, save_as = _save_notebook(notebook.get_data(), owner)
-      return notebook_doc
+    """
+    Creates and returns a query Document2 object
+    :param owner: owner of doc
+    :param query_type: hive, impala or spark
+    :param database: database name
+    :param name: name of document
+    :param description: description of document
+    :param statement: SQL statement (can be multi-query statement)
+    :param files: list of dicts representing files
+    :param functions: list of dicts representing functions
+    :param settings: list of dicts representing settings
+    :return: Document2 object representing query
+    """
+    if query_type not in ('hive', 'impala', 'spark'):
+      raise ValueError("Invalid query_type: %s" % query_type)
+
+    notebook = make_notebook(name=name, description=description, editor_type=query_type, statement=statement,
+                              status='ready', database=database, files=files, functions=functions, settings=settings)
+    notebook_doc, save_as = _save_notebook(notebook.get_data(), owner)
+    return notebook_doc
 
 
   def get_snippet(self, notebook, snippet_idx=0):
@@ -885,33 +947,33 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
 
 
   def execute_and_wait(self, query_doc, snippet_idx=0, timeout=30.0, wait=1.0):
-      notebook = Notebook(document=query_doc)
-      snippet = self.get_snippet(notebook, snippet_idx=snippet_idx)
+    notebook = Notebook(document=query_doc)
+    snippet = self.get_snippet(notebook, snippet_idx=snippet_idx)
 
-      curr = time.time()
-      end = curr + timeout
-      status = 'ready'
+    curr = time.time()
+    end = curr + timeout
+    status = 'ready'
+
+    response = self.client.post(reverse('notebook:execute'),
+                                {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
+    notebook = Notebook(document=query_doc)
+    snippet = self.get_snippet(notebook, snippet_idx=snippet_idx)
+    data = json.loads(response.content)
+    snippet['result']['handle'] = data['handle']
 
-      response = self.client.post(reverse('notebook:execute'),
+    while status != 'available' and curr <= end:
+      response = self.client.post(reverse('notebook:check_status'),
                                   {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
-      notebook = Notebook(document=query_doc)
-      snippet = self.get_snippet(notebook, snippet_idx=snippet_idx)
       data = json.loads(response.content)
-      snippet['result']['handle'] = data['handle']
-
-      while status != 'available' and curr <= end:
-        response = self.client.post(reverse('notebook:check_status'),
-                                    {'notebook': notebook.get_json(), 'snippet': json.dumps(snippet)})
-        data = json.loads(response.content)
-        status = data['query_status']['status']
-        snippet['status'] = status
-        time.sleep(wait)
-        curr = time.time()
+      status = data['query_status']['status']
+      snippet['status'] = status
+      time.sleep(wait)
+      curr = time.time()
 
-      if status != 'available':
-        raise Exception('Query failed to complete or return results.')
+    if status != 'available':
+      raise Exception('Query failed to complete or return results.')
 
-      return snippet
+    return snippet
 
 
   def test_query_with_unicode(self):
@@ -1371,7 +1433,8 @@ IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE = IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE_PLAN
  ID  Src. Node  Tgt. Node(s)  Target type  Partition filter  Pending (Expected)  First arrived  Completed   Enabled
 -------------------------------------------------------------------------------------------------------------------
 
-    Backend startup latencies: Count: 1, min / max: 1ms / 1ms, 25th %-ile: 1ms, 50th %-ile: 1ms, 75th %-ile: 1ms, 90th %-ile: 1ms, 95th %-ile: 1ms, 99.9th %-ile: 1ms
+    Backend startup latencies: Count: 1, min / max: 1ms / 1ms, 25th %-ile: 1ms, 50th %-ile: 1ms, """\
+      """75th %-ile: 1ms, 90th %-ile: 1ms, 95th %-ile: 1ms, 99.9th %-ile: 1ms
     Per Node Peak Memory Usage: self-service-analytics-2.gce.cloudera.com:22000(530.52 KB)
      - FiltersReceived: 0 (0)
      - FinalizationTimer: 0.000ns
@@ -1427,8 +1490,12 @@ IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE = IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE_PLAN
            - SendersBlockedTimer: 0.000ns
            - SendersBlockedTotalTimer(*): 0.000ns
     Coordinator Fragment F01:
-      Instance e147228183f1f0b3:6f086cc600000000 (host=self-service-analytics-2.gce.cloudera.com:22000):(Total: 76.006ms, non-child: 1.000ms, % non-child: 1.32%)
-        MemoryUsage(4s000ms): 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB
+      Instance e147228183f1f0b3:6f086cc600000000 (host=self-service-analytics-2.gce.cloudera.com:22000):"""\
+        """(Total: 76.006ms, non-child: 1.000ms, % non-child: 1.32%)
+        MemoryUsage(4s000ms): 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, """\
+          """31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, """\
+          """31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, """\
+          """31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB, 31.64 KB
          - AverageThreadTokens: 0.00
          - BloomFilterBytes: 0
          - PeakMemoryUsage: 34.12 KB (34939)
@@ -1469,7 +1536,10 @@ IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE = IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE_PLAN
            - RowsReturned: 106 (106)
            - RowsReturnedRate: 1.71 K/sec
           DataStreamReceiver:
-            BytesReceived(4s000ms): 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB
+            BytesReceived(4s000ms): 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, """\
+              """5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, """\
+              """5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, """\
+              """5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB, 5.50 KB
              - BytesReceived: 5.50 KB (5632)
              - DeserializeRowBatchTimer: 0.000ns
              - FirstBatchArrivalWaitTime: 62.005ms
@@ -1626,7 +1696,8 @@ IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE = IMPALA_CUSTOMER_QUERY_SAMPLE_PROFILE_PLAN
          - TotalRawHdfsReadTime(*): 38.003ms
          - TotalReadThroughput: 0.00 /sec
     Fragment F00:
-      Instance e147228183f1f0b3:6f086cc600000001 (host=self-service-analytics-2.gce.cloudera.com:22000):(Total: 76.006ms, non-child: 1.000ms, % non-child: 1.32%)
+      Instance e147228183f1f0b3:6f086cc600000001 (host=self-service-analytics-2.gce.cloudera.com:22000):"""\
+        """(Total: 76.006ms, non-child: 1.000ms, % non-child: 1.32%)
         Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/15.44 KB
          - AverageThreadTokens: 0.00
          - BloomFilterBytes: 0