Browse Source

HUE-8758 [impala] Add end to end unit test to execute API connector

Romain 6 years ago
parent
commit
83504e7582

+ 1 - 1
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -576,7 +576,7 @@ class HiveServerClient(object):
         use_sasl = True
         mechanism = HiveServerClient.HS2_MECHANISMS['NONE']
       else:
-        cluster_conf = cluster.get_cluster_conf_for_job_submission()
+        cluster_conf = cluster.get_cluster_conf_for_job_submission() # Note: Create YARN HttpClient while validate config, should directly read Impala property instead
         use_sasl = cluster_conf is not None and cluster_conf.SECURITY_ENABLED.get()
         mechanism = HiveServerClient.HS2_MECHANISMS['KERBEROS']
       impersonation_enabled = self.query_server['impersonation_enabled']

+ 1 - 1
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -722,4 +722,4 @@ class TopDownAnalysis:
                 "result": topContributions,
                 "template": "alan-tpl"
             }]
-        return result
+        return result

+ 1 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -148,7 +148,7 @@ def _execute_notebook(request, notebook, snippet):
         else:
           _snippet['status'] = 'failed'
 
-        if history:  # If _historify failed, history will be None
+        if history: # If _historify failed, history will be None. If we get Atomic block exception, something underneath interpreter.execute() crashed and is not handled.
           history.update_data(notebook)
           history.save()
 

+ 122 - 5
desktop/libs/notebook/src/notebook/tests.py

@@ -19,6 +19,8 @@
 import json
 
 from collections import OrderedDict
+
+from mock import patch, Mock
 from nose.plugins.attrib import attr
 from nose.tools import assert_equal, assert_true, assert_false
 
@@ -78,8 +80,9 @@ class TestNotebookApi(object):
 
     self.notebook = json.loads(self.notebook_json)
     self.doc2 = Document2.objects.create(id=50010, name=self.notebook['name'], type=self.notebook['type'], owner=self.user)
-    self.doc1 = Document.objects.link(self.doc2, owner=self.user, name=self.doc2.name,
-                                      description=self.doc2.description, extra=self.doc2.type)
+    self.doc1 = Document.objects.link(
+      self.doc2, owner=self.user, name=self.doc2.name, description=self.doc2.description, extra=self.doc2.type
+    )
 
 
   def test_save_notebook(self):
@@ -275,7 +278,7 @@ class TestNotebookApi(object):
 a.key,
 a.*
 FROM customers c, c.addresses a"""
-    response =send_exception(message)
+    response = send_exception(message)
     data = json.loads(response.content)
     assert_equal(1, data['status'])
 
@@ -283,7 +286,7 @@ FROM customers c, c.addresses a"""
 \u2002\u2002a.key,
 \u2002\u2002a.*
 FROM customers c, c.addresses a"""
-    response =send_exception(message)
+    response = send_exception(message)
     data = json.loads(response.content)
     assert_equal(1, data['status'])
 
@@ -291,11 +294,123 @@ FROM customers c, c.addresses a"""
 a.key,
 a.*
 FROM déclenché c, c.addresses a"""
-    response =send_exception(message)
+    response = send_exception(message)
     data = json.loads(response.content)
     assert_equal(1, data['status'])
 
 
+class TestConnectors():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+
+    self.user = User.objects.get(username="test")
+    grant_access("test", "default", "notebook")
+
+
+  def test_hiveserver2_impala(self):
+    notebook_json = """
+      {
+        "selectedSnippet": "impala",
+        "showHistory": false,
+        "description": "Test Impala Query",
+        "name": "Test Impala Query",
+        "sessions": [
+            {
+                "type": "impala",
+                "properties": [],
+                "id": null
+            }
+        ],
+        "type": "query-impala",
+        "id": null,
+        "snippets": [{
+          "id":"2b7d1f46-17a0-30af-efeb-33d4c29b1055","type":"impala-xx","status":"running",
+          "statement_raw":"select * from web_logs",
+          "statement":"select * from web_logs",
+          "variables":[],
+          "properties":{"settings":[],"variables":[],"files":[],"functions":[]},
+          "result":{
+              "id":"b424befa-f4f5-8799-a0b4-79753f2552b1","type":"table",
+              "handle":{"log_context":null,"statements_count":1,"end":{"column":21,"row":0},"statement_id":0,"has_more_statements":false,
+                  "start":{"column":0,"row":0},"secret":"rVRWw7YPRGqPT7LZ/TeFaA==an","has_result_set":true,
+                  "statement":"select * from web_logs","operation_type":0,"modified_row_count":null,"guid":"7xm6+epkRx6dyvYvGNYePA==an"}
+              },
+          "lastExecuted": 1462554843817,"database":"default"
+        }],
+        "uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a",
+        "isSaved":false
+    }
+    """
+
+    connector = [{
+      'name': 'Impala', 'type': 'impala-xx', 'connector_name': 'impala', 'interface': 'hiveserver2',
+      'settings': [
+          {'name': 'server_host', 'value': 'gethue.com'},
+          {'name': 'server_port', 'value': '21050'},
+        ],
+        'id': 1, 'category': 'engines', 'description': ''
+      },
+    ]
+
+    from TCLIService.ttypes import TStatusCode, TProtocolVersion, TOperationType
+    with patch('desktop.lib.connectors.api.CONNECTOR_INSTANCES', connector):
+      with patch('desktop.lib.thrift_util.get_client') as get_client:
+        tclient = Mock()
+        successfullCall = Mock(
+          return_value=Mock(
+            status=Mock(
+              statusCode=TStatusCode.SUCCESS_STATUS
+            ),
+            sessionHandle=Mock(
+              sessionId=Mock(
+                secret='\x7f\x98\x97s\xe1\xa8G\xf4\x8a\x8a\\r\x0e6\xc2\xee\xf0',
+                guid='\xfa\xb0/\x04 \xfeDX\x99\xfcq\xff2\x07\x02\xfe',
+              )
+            ),
+            configuration={},
+            serverProtocolVersion=TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V7,
+            # TFetchResultsResp
+            results=Mock(
+              startRowOffset=0,
+              rows=[],
+              columns=[]
+            ),
+            # ExecuteStatement
+            operationHandle=Mock(
+              operationId=Mock(
+                secret='\x7f\x98\x97s\xe1\xa8G\xf4\x8a\x8a\\r\x0e6\xc2\xee\xf0',
+                guid='\xfa\xb0/\x04 \xfeDX\x99\xfcq\xff2\x07\x02\xfe',
+              ),
+              hasResultSet=True,
+              operationType=TOperationType.EXECUTE_STATEMENT,
+              modifiedRowCount=0
+            ),
+          )
+        )
+
+        tclient.OpenSession = successfullCall
+        tclient.ExecuteStatement = successfullCall
+        tclient.FetchResults = successfullCall
+        tclient.GetResultSetMetadata = successfullCall
+        tclient.CloseOperation = successfullCall
+
+        get_client.return_value = tclient
+        tclient.get_coordinator_host = Mock(return_value={})
+
+
+        response = self.client.post(reverse('notebook:execute'), {
+            'notebook': notebook_json,
+            'snippet': json.dumps(json.loads(notebook_json)['snippets'][0]),
+        })
+
+      get_client.assert_called()
+
+    assert_equal(response.status_code, 200)
+    data = json.loads(response.content)
+    assert_equal(data['status'], 0)
+
+
 class MockedApi(Api):
   def execute(self, notebook, snippet):
     return {
@@ -443,6 +558,7 @@ class TestNotebookApiMocked(object):
       assert_equal(0, data['status'], data)
       assert_equal('adl:/user/hue/path.csv', data['watch_url']['destination'], data)
 
+
   def test_download_result(self):
     notebook_json = """
       {
@@ -471,6 +587,7 @@ class TestNotebookApiMocked(object):
     content = "".join(response)
     assert_true(len(content) > 0)
 
+
 def test_get_interpreters_to_show():
   default_interpreters = OrderedDict((
       ('hive', {

+ 3 - 0
desktop/libs/notebook/src/notebook/views.py

@@ -59,9 +59,11 @@ class ApiWrapper(object):
     else:
       return object.__getattribute__(self.api, name)
 
+
 def get_api(request, snippet):
   return ApiWrapper(request, snippet)
 
+
 def notebooks(request):
   editor_type = request.GET.get('type', 'notebook')
 
@@ -329,6 +331,7 @@ def copy(request):
 
   return JsonResponse(response)
 
+
 @check_document_access_permission()
 def download(request):
   if not ENABLE_DOWNLOAD.get():