瀏覽代碼

HUE-3228 [dashboard] Support other SQL that have a sync API

Romain Rigaux 8 年之前
父節點
當前提交
204d0c4521

+ 27 - 23
apps/impala/src/impala/dashboard_api.py

@@ -23,9 +23,9 @@ from itertools import groupby
 from django.utils.html import escape
 
 from beeswax.server import dbms
-from search.models import Collection2
 from notebook.models import make_notebook
 from notebook.connectors.base import get_api
+from search.models import Collection2
 
 
 LOG = logging.getLogger(__name__)
@@ -111,15 +111,19 @@ class SQLApi():
 
     editor = make_notebook(
         name='Execute and watch',
-        editor_type='impala',
+        editor_type=dashboard['engine'],
         statement=sql,
         database=database,
         status='ready-execute'
         # historify=False
     )
 
-    # TODO: sync
-    return editor.execute(MockRequest(self.user))
+    response = editor.execute(MockRequest(self.user))
+
+    if 'handle' in response and response['handle'].get('sync'):
+      response['result'] = self._convert_result(response['result'], dashboard, facet, query)
+
+    return response
 
 
   def fetch_result(self, dashboard, query, facet):
@@ -135,16 +139,7 @@ class SQLApi():
         start_over=start_over
     )
 
-    result['has_more'] # TODO
-
-    # TODO: add query id to allow closing
-
-    if not facet.get('type'):
-      return self._convert_impala_results(result, dashboard, query)
-    elif facet['type'] == 'function':
-      return self._convert_impala_function_facet(result, facet, query)
-    else:
-      return self._convert_impala_facet(result, facet, query) # query needed
+    return self._convert_result(result, dashboard, facet, query)
 
 
   def datasets(self, show_all=False):
@@ -154,18 +149,19 @@ class SQLApi():
 
   def fields(self, dashboard):
     database, table = self._get_database_table_names(dashboard)
+    snippet = {'type': self.engine}
+
+    table_metadata = get_api(MockRequest(self.user), snippet).autocomplete(snippet, database, table)
 
-    db = dbms.get(self.user)
-    table_metadata = db.get_table(database=database, table_name=table)
     return [{
-        'name': str(escape(col.name)),
-        'type': str(col.type),
+        'name': str(escape(col['name'])),
+        'type': str(col['type']),
         'isId': False, # TODO Kudu
         'isDynamic': False,
         'indexed': False,
         'stored': True
         # isNested
-      } for col in table_metadata.cols
+      } for col in table_metadata['extended_columns']
     ]
 
 
@@ -178,15 +174,23 @@ class SQLApi():
     return {'fields': Collection2._make_luke_from_schema_fields(fields)}
 
 
-  def close_query(self, collection, query, facet=None): pass
+  def _convert_result(self, result, dashboard, facet, query):
+    if not facet.get('type'):
+      return self._convert_notebook_results(result, dashboard, query)
+    elif facet['type'] == 'function':
+      return self._convert_notebook_function_facet(result, facet, query)
+    else:
+      return self._convert_notebook_facet(result, facet, query)
 
 
   def _get_dimension_fields(self, facet):
     return [facet] + [f for f in facet['properties']['facets'] if f['aggregate']['function'] == 'count']
 
+
   def _convert_filters_to_where(self, filters):
     return ('WHERE ' + ' AND '.join(filters)) if filters else ''
 
+
   def _get_fq(self, collection, query, facet=None):
     clauses = []
 
@@ -257,7 +261,7 @@ class SQLApi():
 
     return database, table_name
 
-  def _convert_impala_facet(self, result, facet, query):
+  def _convert_notebook_facet(self, result, facet, query):
     response = json.loads('''{
    "fieldsAttributes":[],
    "response":{
@@ -333,7 +337,7 @@ class SQLApi():
     return {'normalized_facets': [response]}
 
 
-  def _convert_impala_function_facet(self, result, facet, query):
+  def _convert_notebook_function_facet(self, result, facet, query):
     rows = list(result['data'])
 
     response = {"query": facet['id'], "counts": rows[0][0], "type": "function", "id": facet['id'], "label": facet['id']}
@@ -341,7 +345,7 @@ class SQLApi():
     return {'normalized_facets': [response]}
 
 
-  def _convert_impala_results(self, result, dashboard, query):
+  def _convert_notebook_results(self, result, dashboard, query):
     cols = [col['name'] for col in result['meta']]
 
     docs = []

+ 0 - 1
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -243,7 +243,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 </div>
 </div>
 
-  nav nav-pills hueBreadcrumbBar
 
 <script type="text/html" id="breadcrumbs">
   <h3>

+ 0 - 2
apps/search/src/search/api_engines.py

@@ -63,8 +63,6 @@ class DashboardApi(object):
 
   def terms(self, collection, field, properties): pass
 
-  def close_query(self, collection, query, facet=None): pass
-
   def fetch_result(self, collection, query, facet=None): pass
 
 

+ 60 - 43
apps/search/src/search/static/search/js/search.ko.js

@@ -1426,7 +1426,7 @@ var NewTemplate = function (vm, initial) {
 };
 
 
-var QueryResult = function (vm, initial) {
+var QueryResult = function (vm, initial) { // Similar to to Notebook Snippet
   var self = this;
 
   self.id = ko.observable(UUID());
@@ -1443,7 +1443,11 @@ var QueryResult = function (vm, initial) {
   self.result.type = ko.observable('table');
 
   self.getContext = function() {
-  return self;
+    return self;
+  }
+
+  self.asyncResult = function() {
+    return ko.mapping.toJS(self.result.result);
   }
 };
 
@@ -1634,7 +1638,10 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       notebook: ko.mapping.toJSON({type: facet.queryResult().type()}),
       snippet: ko.mapping.toJSON(facet.queryResult().getContext())
     }, function (data) {
-      if (facet.queryResult().status() == 'canceled') {
+      if (! self.collection.async()) {
+        self.fetchResult(facet);
+      }
+      else if (facet.queryResult().status() == 'canceled') {
         // Query was canceled in the meantime, do nothing
       } else {
 
@@ -1706,51 +1713,58 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       self.queryResult().status('failed');
     });
   };
-  
-  self.fetchResult = function(facet) {
-    $.post("/search/search", {
-        collection: ko.mapping.toJSON(self.collection),
-        query: ko.mapping.toJSON(self.query),
-        facet: ko.mapping.toJSON(facet),
-        fetch_result: true
-    }, function (data) {
-      if (facet.type) {
-        $.each(data.normalized_facets, function (index, facet) {
-          self._make_result_facet(facet);
-        });
-      } else {
-        self._make_grid_result(data);
-      }
 
-      self.asyncSearchesCounter.remove(facet);
+  self._loadResults = function(facet, data) {
+    if (facet.type) {
+      $.each(data.normalized_facets, function (index, facet) {
+        self._make_result_facet(facet);
+      });
+    } else {
+      self._make_grid_result(data);
+    }
+  }
 
-      //if (facet.queryResult().result['handle'].has_result_set()) {
-      //  self.fetchResultSize(facet);
-      //}
-    });
+  self.fetchResult = function(facet) {
+    if (! self.collection.async()) {
+      self._loadResults(facet, facet.queryResult().asyncResult());
+    } else {
+      $.post("/search/search", {
+          collection: ko.mapping.toJSON(self.collection),
+          query: ko.mapping.toJSON(self.query),
+          facet: ko.mapping.toJSON(facet),
+          fetch_result: true
+      }, function (data) {
+        self._loadResults(facet, data);
+        self.asyncSearchesCounter.remove(facet);
+
+        //if (facet.queryResult().result['handle'].has_result_set()) {
+        //  self.fetchResultSize(facet);
+        //}
+      });
+    }
   };
 
   self.fetchResultSize = function(facet) {
-  $.post("/notebook/api/fetch_result_size", {
-      notebook: ko.mapping.toJSON({type: facet.queryResult().type()}),
-      snippet: ko.mapping.toJSON(facet.queryResult)
-  }, function (data) {
-    if (data.status == 0) {
-      if (data.result.rows != null) {
-      facet.response().response.numFound(data.result.rows);
+    $.post("/notebook/api/fetch_result_size", {
+        notebook: ko.mapping.toJSON({type: facet.queryResult().type()}),
+        snippet: ko.mapping.toJSON(facet.queryResult)
+    }, function (data) {
+      if (data.status == 0) {
+        if (data.result.rows != null) {
+        facet.response().response.numFound(data.result.rows);
+       }
+     } else if (data.status == 5) {
+        // No supported yet for this snippet
+      } else {
+        $(document).trigger("error", data.message);
       }
-    } else if (data.status == 5) {
-      // No supported yet for this snippet
-    } else {
-      $(document).trigger("error", data.message);
-    }
-  }).fail(function (xhr, textStatus, errorThrown) {
-    $(document).trigger("error", xhr.responseText);
-  });
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
   };
 
 
-  self.search = function (callback) {   // self.searchAsync == self.collection.async()
+  self.search = function (callback) {
     $(".jHueNotify").hide();
     logGA('search');
     self.isRetrievingResults(true);
@@ -1797,7 +1811,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       });
     }
 
-    if (self.collection.async()) {
+    if (self.collection.engine() != 'solr') {
       $.each([self.collection].concat(self.collection.facets()), function(index, facet) {
         if (facet.queryResult().result.handle) {
           self.close(facet);
@@ -1820,7 +1834,10 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
           self.checkStatus(facet);
         });
       });
-      self.asyncSearchesCounter([self.collection].concat(self.collection.facets()));
+
+      if (self.collection.async()) {
+        self.asyncSearchesCounter([self.collection].concat(self.collection.facets()));
+      }
     }
 
     $.each(self.fieldAnalyses(), function (index, analyse) { // Invalidate stats analysis
@@ -1840,7 +1857,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
         }, function (data) {
           data = JSON.bigdataParse(data);
           try {
-            if (! self.collection.async()) {
+            if (self.collection.engine() == 'solr') {
               self._make_grid_result(data, callback);
             } else {
               self.collection.queryResult(new QueryResult(self, {
@@ -1861,7 +1878,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     )
     .done(function () {
       if (arguments[0] instanceof Array) {
-        if (! self.collection.async()) { // If multi queries
+        if (self.collection.engine() == 'solr') { // If multi queries
           var histograms = self.collection.getHistogramFacets();
           for (var h = 0; h < histograms.length; h++) { // Do not use $.each here
             var histoFacetId = histograms[h].id();