فهرست منبع

HUE-5145 [metadata] Integrate APIs into the editor

Romain Rigaux 9 سال پیش
والد
کامیت
ef4eeed

+ 13 - 11
apps/metastore/src/metastore/static/metastore/js/metastore.ko.js

@@ -58,7 +58,7 @@ var MetastoreViewModel = (function () {
         });
       }
       return returned.sort(function (a, b) {
-    	if (options.optimizerEnabled()) {
+        if (options.optimizerEnabled()) {
           if (typeof a.optimizerStats() !== 'undefined' && a.optimizerStats() !== null) {
             if (typeof b.optimizerStats() !== 'undefined' && b.optimizerStats() !== null) {
               if (a.optimizerStats().popularity === b.optimizerStats().popularity) {
@@ -81,7 +81,7 @@ var MetastoreViewModel = (function () {
 
     self.editingTable = ko.observable(false);
     self.table = ko.observable(null);
-    
+
     self.addTags = function () {
       $.post('/metadata/api/navigator/add_tags', {
         id: ko.mapping.toJSON(self.navigatorStats().identity),
@@ -96,7 +96,7 @@ var MetastoreViewModel = (function () {
         }
       });
     };
-      
+
     self.deleteTags = function (tag) {
       $.post('/metadata/api/navigator/delete_tags', {
         id: ko.mapping.toJSON(self.navigatorStats().identity),
@@ -314,9 +314,9 @@ var MetastoreViewModel = (function () {
   };
 
   function MetastoreTableDetails(details) {
-	var self = this;
+    var self = this;
   }
-  
+
   /**
    * @param {Object} options
    * @param {MetastoreDatabase} options.database
@@ -480,24 +480,26 @@ var MetastoreViewModel = (function () {
               }, function(data) {
                 if (data && data.status == 0) {
                   self.navigatorStats(ko.mapping.fromJS(data.entity));
-                  self.getRelationships();
+                  // self.getRelationships();
                 } else {
                   $(document).trigger("info", data.message);
                 }
               }).fail(function (xhr, textStatus, errorThrown) {
                 $(document).trigger("error", xhr.responseText);
               });
-            } else if (self.optimizerEnabled) {
+            }
+            if (self.optimizerEnabled) {
               $.post('/metadata/api/optimizer_api/table_details', {
+                databaseName: self.database.name,
                 tableName: self.name
               }, function(data){
                 self.loadingQueries(false);
                 if (data && data.status == 0) {
                   self.optimizerDetails(ko.mapping.fromJS(data.details));
-                  
+
                   // Bump the most important columns first
                   var topCol = self.optimizerDetails().table_donut.topColumns().slice(0, 5);
-                  if (topCol.length >= 3 && self.favouriteColumns().length > 0) { 
+                  if (topCol.length >= 3 && self.favouriteColumns().length > 0) {
                     self.favouriteColumns($.grep(self.columns(), function(col) {
                         return topCol.indexOf(col.name()) != -1;
                       })
@@ -547,7 +549,7 @@ var MetastoreViewModel = (function () {
         }
       });
     };
-    
+
     self.deleteTags = function (tag) {console.log(tag);
       $.post('/metadata/api/navigator/delete_tags', {
         id: ko.mapping.toJSON(self.navigatorStats().identity),
@@ -560,7 +562,7 @@ var MetastoreViewModel = (function () {
         }
       });
     };
-    
+
     self.getRelationships = function () {
       $.post('/metadata/api/navigator/lineage', {
         id: self.navigatorStats().identity

+ 8 - 9
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -111,7 +111,7 @@ def table_details(request):
 
   if data['status'] == 'success':
     response['status'] = 0
-    response['details'] = data['details']
+    response['details'] = data
   else:
     response['message'] = 'Optimizer: %s' % data['details']
 
@@ -133,9 +133,9 @@ def query_compatibility(request):
 
   if data['status'] == 'success':
     response['status'] = 0
-    response['query_compatibility'] = json.loads(data['details'])
+    response['query_compatibility'] = data
   else:
-    response['message'] = 'Optimizer: %s' % data['details']
+    response['message'] = 'Optimizer: %s' % data
 
   return JsonResponse(response)
 
@@ -147,16 +147,15 @@ def query_risk(request):
 
   query = json.loads(request.POST.get('query'))
 
-
   api = OptimizerApi()
 
   data = api.query_risk(query=query)
 
-  response['query_complexity'] = {
-    'level': random.choice(['LOW', 'MEDIUM', 'HIGH']),
-    'comment': data
-  }
-  response['status'] = 0
+  if data['status'] == 'success':
+    response['status'] = 0
+    response['query_risk'] = data
+  else:
+    response['message'] = 'Optimizer: %s' % data
 
   return JsonResponse(response)
 

+ 40 - 10
desktop/libs/metadata/src/metadata/optimizer_client_tests.py

@@ -111,15 +111,38 @@ class TestOptimizerApi(object):
     database_name = 'default'
     resp = self.api.top_tables(database_name=database_name)
 
-    assert_true(isinstance(resp['results'], list), resp) # No status code currently
+    assert_true(isinstance(resp['results'], list), resp)
+
+    assert_true('eid' in resp['results'][0], resp)
+    assert_true('name' in resp['results'][0], resp)
 
 
   def test_table_details(self):  # Requires test_upload to run before
     resp = self.api.table_details(database_name='default', table_name='emps')
+
     assert_equal('success', resp['status'], resp)
+    assert_true('columnCount' in resp, resp)
+    assert_true('createCount' in resp, resp)
+    assert_true('table_ddl' in resp, resp)
+    assert_true('deleteCount' in resp, resp)
+    assert_true('iview_ddl' in resp, resp)
+    assert_true('updateCount' in resp, resp)
+    assert_true('colStats' in resp, resp)
+    assert_true('joinCount' in resp, resp)
+    assert_true('view_ddl' in resp, resp)
+    assert_true('tableStats' in resp, resp)
+    assert_true('queryCount' in resp, resp)
+    assert_true('selectCount' in resp, resp)
+    assert_true('insertCount' in resp, resp)
+    assert_true('tid' in resp, resp)
+    assert_true('type' in resp, resp)
+    assert_true('name' in resp, resp)
 
     resp = self.api.table_details(database_name='db1', table_name='Part')
+
     assert_equal('success', resp['status'], resp)
+    assert_true('tid' in resp, resp)
+    assert_true('columnCount' in resp, resp)
 
 
   def test_query_risk(self):
@@ -127,25 +150,32 @@ class TestOptimizerApi(object):
 
     resp = self.api.query_risk(query=query)
 
-    assert_equal('successs', resp['status'], resp)
+    assert_equal('success', resp['status'], resp)
 
-    details = json.loads(resp['details']) # Auto fix suggestion is empty in most of the cases currently
+    assert_true('impalaRisk' in resp, resp)
+    assert_true('riskAnalysis' in resp['impalaRisk'], resp)
+    assert_true('risk' in resp['impalaRisk'], resp)
+    assert_true('riskRecommendation' in resp['impalaRisk'], resp)
 
-    assert_equal('FAIL', details['hiveRisk']['risk'], resp)
+    assert_true('hiveRisk' in resp, resp)
+    assert_true('riskAnalysis' in resp['hiveRisk'], resp)
+    assert_true('risk' in resp['hiveRisk'], resp)
+    assert_true('riskRecommendation' in resp['hiveRisk'], resp)
 
 
   def test_query_compatibility(self):
-    source_platform = 'Hive'
-    target_platform = 'Impala'
+    source_platform = 'hive'
+    target_platform = 'impala'
     query = 'Select * from (Select item.id from item)'
 
     resp = self.api.query_compatibility(source_platform=source_platform, target_platform=target_platform, query=query)
 
-    assert_equal('successs', resp['status'], resp)
-
-    details = json.loads(resp['details']) # Auto fix suggestion is empty in most of the cases currently
+    assert_equal('success', resp['status'], resp)
 
-    assert_equal('FAIL', details['platformCompilationStatus']['Hive']['queryStatus'], resp)
+    assert_true('clauseName' in resp, resp)
+    assert_true('clauseError' in resp, resp)
+    assert_true('queryError' in resp, resp)
+    assert_true('clauseString' in resp, resp)
 
 
   def test_similar_queries(self):

+ 2 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -710,7 +710,7 @@ def statement_risk(request):
 
   api = get_api(request, snippet)
 
-  response['data'] = api.query_risk(notebook, snippet)
+  response['query_complexity'] = api.statement_risk(notebook, snippet)
   response['status'] = 0
 
   return JsonResponse(response)
@@ -729,7 +729,7 @@ def statement_compatibility(request):
 
   api = get_api(request, snippet)
 
-  response['data'] = api.statement_compatibility(notebook, snippet, source_platform=source_platform, target_platform=target_platform)
+  response['query_compatibility'] = api.statement_compatibility(notebook, snippet, source_platform=source_platform, target_platform=target_platform)
   response['status'] = 0
 
   return JsonResponse(response)

+ 10 - 5
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -487,20 +487,25 @@ class HS2Api(Api):
     db = self._get_db(snippet)
 
     response = self._get_current_statement(db, snippet)
-    session = self._get_session(notebook, snippet['type'])
-    query = self._prepare_hql_query(snippet, response.pop('statement'), session)
+    query = response['statement']
 
     api = OptimizerApi()
 
-    return api.query_risk(query=query)
+    data = api.query_risk(query=query)
+    data = data.get(snippet['type'] + 'Risk', {})
+
+    return {
+      'risk': data.get('risk'),
+      'riskAnalysis': data.get('riskAnalysis'),
+      'riskRecommendation': data.get('riskRecommendation')
+    }
 
 
   def statement_compatibility(self, notebook, snippet, source_platform, target_platform):
     db = self._get_db(snippet)
 
     response = self._get_current_statement(db, snippet)
-    session = self._get_session(notebook, snippet['type'])
-    query = self._prepare_hql_query(snippet, response.pop('statement'), session)
+    query = response['statement']
 
     api = OptimizerApi()
 

+ 10 - 8
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -613,8 +613,9 @@ var EditorViewModel = (function() {
 
     self.is_redacted = ko.observable(typeof snippet.is_redacted != "undefined" && snippet.is_redacted != null ? snippet.is_redacted : false);
 
-    self.complexity = ko.observable(typeof snippet.complexity != "undefined" && snippet.complexity != null ? snippet.complexity : '');
-    self.complexityLevel = ko.observable(typeof snippet.complexity_level != "undefined" && snippet.complexity_level != null ? snippet.complexity_level : '');
+    self.complexity = ko.observable('');
+    self.complexityLevel = ko.observable('');
+    self.complexityRecommendation = ko.observable('');
     self.hasComplexity = ko.computed(function () {
       return self.complexity().length > 0;
     });
@@ -1001,14 +1002,14 @@ var EditorViewModel = (function() {
       logGA('compatibility');
       self.suggestion(false);
 
-      $.post("/metadata/api/optimizer/compatibility", {
+      $.post("/notebook/api/optimizer/compatibility", {
         notebook: ko.mapping.toJSON(notebook.getContext()),
         snippet: ko.mapping.toJSON(self.getContext()),
         sourcePlatform: self.type(),
         targetPlatform: 'impala'
       }, function(data) {
         if (data.status == 0) {
-         self.suggestion(ko.mapping.fromJS(data.query_compatibility.platformCompilationStatus.Impala));
+         self.suggestion(ko.mapping.fromJS(data.query_compatibility.Impala));
          self.hasSuggestion(true);
         } else {
           $(document).trigger("error", data.message);
@@ -1304,16 +1305,17 @@ var EditorViewModel = (function() {
     };
 
     self.getComplexity = function () {
-      logGA('get_complexity');
+      logGA('get_query_risk');
       self.complexity('');
 
-      $.post("/notebook/api/optimizer/query_risk", {
+      $.post("/notebook/api/optimizer/statement_risk", {
         notebook: ko.mapping.toJSON(notebook.getContext()),
         snippet: ko.mapping.toJSON(self.getContext())
       }, function(data) {
         if (data.status == 0) {
-          self.complexity(data.query_complexity.comment);
-          self.complexityLevel(data.query_complexity.level);
+          self.complexityLevel(data.query_complexity.risk);
+          self.complexity(data.query_complexity.riskAnalysis);
+          self.complexityRecommendation(data.query_complexity.riskRecommendation);
         } else {
           $(document).trigger("error", data.message);
         }

+ 3 - 1
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1508,7 +1508,9 @@ ${ hueIcons.symbols() }
       </svg>
     </div>
     <div style="margin-left: 30px; line-height:20px;vertical-align: middle;">
-      <span style="margin-right:10px; font-weight: bold;" data-bind="text: complexityLevel"></span><span data-bind="text: complexity"></span>
+      <span style="margin-right:10px; font-weight: bold;" data-bind="text: complexityLevel"></span>
+      <span data-bind="text: complexity"></span>
+      </span><span data-bind="text: complexityRecommendation"></span>
     </div>
   </div>
   <div class="alert" data-bind="visible: hasSuggestion">

+ 1 - 1
desktop/libs/notebook/src/notebook/urls.py

@@ -72,7 +72,7 @@ urlpatterns += patterns('notebook.api',
 
   url(r'^api/notebook/export_result/?$', 'export_result', name='export_result'),
 
-  url(r'^api/optimizer/risk/?$', 'statement_risk', name='statement_risk'),
+  url(r'^api/optimizer/statement_risk/?$', 'statement_risk', name='statement_risk'),
   url(r'^api/optimizer/compatibility/?$', 'statement_compatibility', name='statement_compatibility'),
 )