Browse Source

HUE-5145 [editor] Skeleton of query risk API

Romain Rigaux 9 years ago
parent
commit
439361d

+ 1 - 0
apps/beeswax/src/beeswax/design.py

@@ -232,6 +232,7 @@ class HQLdesign(object):
     return not self.__eq__(other)
 
 
+# Note: Might be replaceable by sqlparse.split
 def split_statements(hql):
   """
   Split statements at semicolons ignoring the ones inside quotes and comments.

+ 7 - 12
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -140,26 +140,21 @@ def query_compatibility(request):
   return JsonResponse(response)
 
 
-# Mocked
 @require_POST
 @error_handler
-def query_complexity(request):
+def query_risk(request):
   response = {'status': -1}
 
-  snippet = json.loads(request.POST.get('snippet'))
+  query = json.loads(request.POST.get('query'))
 
-  if 'select * from tsqc_date t join atd_au_dtl a on (t.date = a.date)' in snippet['statement'].lower():
-    comment = 'Large join is happening'
-  elif 'large' in snippet['statement'].lower():
-    comment = 'Previously failed 5 times in a row'
-  elif 'partition' in snippet['statement'].lower():
-    comment = 'Has 50k partitions'
-  else:
-    comment = ''
+
+  api = OptimizerApi()
+
+  data = api.query_risk(query=query)
 
   response['query_complexity'] = {
     'level': random.choice(['LOW', 'MEDIUM', 'HIGH']),
-    'comment': comment
+    'comment': data
   }
   response['status'] = 0
 

+ 7 - 0
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -199,6 +199,13 @@ class OptimizerApi(object):
       raise PopupException(e, title=_('Error while accessing Optimizer'))
 
 
+  def query_risk(self, query):
+    return self._exec('get-query-risk', [
+        '--tenant', self._product_name,
+        '--query', query
+    ])
+
+
   def similar_queries(self, source_platform, query, token=None, email=None):
     if token is None:
       token = self._authenticate()

+ 12 - 0
desktop/libs/metadata/src/metadata/optimizer_client_tests.py

@@ -121,6 +121,18 @@ class TestOptimizerApi(object):
     assert_equal('success', resp['status'], resp)
 
 
+  def test_query_risk(self):
+    query = 'Select * from (Select item.id from item)'
+
+    resp = self.api.query_risk(query=query)
+
+    assert_equal('successs', resp['status'], resp)
+
+    details = json.loads(resp['details']) # Auto fix suggestion is empty in most of the cases currently
+
+    assert_equal('FAIL', details['hiveRisk']['risk'], resp)
+
+
   def test_query_compatibility(self):
     source_platform = 'MySQL'
     target_platform = 'Hive'

+ 3 - 3
desktop/libs/metadata/src/metadata/urls.py

@@ -36,14 +36,14 @@ urlpatterns = patterns('metadata.navigator_api',
 
 # Optimizer API
 urlpatterns += patterns('metadata.optimizer_api',
-  url(r'^api/optimizer_api/top_tables/?$', 'top_tables', name='top_tables'),
-  url(r'^api/optimizer_api/table_details/?$', 'table_details', name='table_details'),
+  url(r'^api/optimizer_api/top_tables/?$', 'top_tables', name='top_tables'),  
   url(r'^api/optimizer_api/query_compatibility/?$', 'query_compatibility', name='query_compatibility'),
   url(r'^api/optimizer_api/upload_history/?$', 'upload_history', name='upload_history'),
-  url(r'^api/optimizer_api/query_complexity/?$', 'query_complexity', name='query_complexity'),
   url(r'^api/optimizer_api/popular_values/?$', 'popular_values', name='popular_values'),
   url(r'^api/optimizer_api/similar_queries/?$', 'similar_queries', name='similar_queries'),
 
   #v2
   url(r'^api/optimizer_api/get_tenant/?$', 'get_tenant', name='get_tenant'),
+  url(r'^api/optimizer_api/table_details/?$', 'table_details', name='table_details'),
+  url(r'^api/optimizer_api/query_risk/?$', 'query_risk', name='query_risk'),
 )

+ 17 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -697,3 +697,20 @@ def export_result(request):
     response['status'] = 0
 
   return JsonResponse(response)
+
+
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def statement_risk(request):
+  response = {'status': -1, 'message': _('Risk analysis failed.')}
+
+  notebook = json.loads(request.POST.get('notebook', '{}'))
+  snippet = json.loads(request.POST.get('snippet', '{}'))
+
+  api = get_api(request, snippet)
+
+  response['data'] = api.query_risk(notebook, snippet)
+  response['status'] = 0
+
+  return JsonResponse(response)

+ 2 - 0
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -278,3 +278,5 @@ class Api(object):
   def export_data_as_table(self, notebook, snippet, destination, is_temporary=False, location=None): raise NotImplementedError()
 
   def export_large_data_to_hdfs(self, notebook, snippet, destination): raise NotImplementedError()
+
+  def query_risk(self, notebook, snippet): raise NotImplementedError()

+ 15 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -32,6 +32,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from desktop.lib.rest.http_client import RestException
 from desktop.models import DefaultConfiguration
+from metadata.optimizer_client import OptimizerApi
 
 from notebook.connectors.base import Api, QueryError, QueryExpired, OperationTimeout, OperationNotSupported
 
@@ -482,6 +483,18 @@ class HS2Api(Api):
     return hql, success_url
 
 
+  def query_risk(self, notebook, snippet):
+    db = self._get_db(snippet)
+
+    response = self._get_current_statement(db, snippet)
+    session = self._get_session(notebook, snippet['type'])
+    query = self._prepare_hql_query(snippet, response.pop('statement'), session)
+    
+    api = OptimizerApi()
+
+    return api.query_risk(query=query)
+
+
   def upgrade_properties(self, lang='hive', properties=None):
     upgraded_properties = copy.deepcopy(self.get_properties(lang))
 
@@ -619,7 +632,7 @@ class HS2Api(Api):
   def _get_handle(self, snippet):
     try:
       snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
-    except KeyError, ex:
+    except KeyError:
       raise Exception('Operation has no valid handle attached')
 
     for key in snippet['result']['handle'].keys():
@@ -657,7 +670,7 @@ class HS2Api(Api):
       else:
         LOG.info("No HIVE counter group found for job: %s" % job_id)
 
-       # Extract totalCounterValue from FileSystemCounter counter group
+      # Extract totalCounterValue from FileSystemCounter counter group
       fs_counters = next((group for group in counter_groups if group.get('counterGroupName') == 'org.apache.hadoop.mapreduce.FileSystemCounter'), None)
       if fs_counters:
         total_size = next((counter.get('totalCounterValue') for counter in fs_counters['counter'] if counter['name'] == 'HDFS_BYTES_WRITTEN'), None)

+ 1 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1306,7 +1306,7 @@ var EditorViewModel = (function() {
       logGA('get_complexity');
       self.complexity('');
 
-      $.post("/metadata/api/optimizer_api/query_complexity", {
+      $.post("/notebook/api/optimizer/query_risk", {
         snippet: ko.mapping.toJSON(self.getContext())
       }, function(data) {
         if (data.status == 0) {

+ 2 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -71,6 +71,8 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/notebook/close/?$', 'close_notebook', name='close_notebook'),
 
   url(r'^api/notebook/export_result/?$', 'export_result', name='export_result'),
+
+  url(r'^api/optimizer/risk/?$', 'statement_risk', name='statement_risk'),
 )
 
 # Assist API