Browse Source

HUE-3228 [dashboard] Async submission and status fetching

Romain Rigaux 8 years ago
parent
commit
bdb29d0

+ 12 - 34
apps/impala/src/impala/dashboard_api.py

@@ -91,17 +91,6 @@ class SQLApi():
         sql += ' WHERE ' + ' AND '.join(filters)
       sql += ' LIMIT %s' % LIMIT
 
-#     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
-#     db = dbms.get(request.user)
-#     table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
-#
-#     format_ = {
-#         "sample": sample['rows'][:4],
-#         "columns": [
-#             Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
-#             for col in table_metadata.cols
-#         ]
-#     }
 
     editor = make_notebook(
         name='Execute and watch',
@@ -111,20 +100,13 @@ class SQLApi():
         status='ready-execute'
     )
     return editor.execute(MockRequest(self.user))
-  
-  def fetch_result(self, dashboard, query, facet=None):
-#     query_server = get_query_server_config(name='impala') # To move to notebook API
-#     db = dbms.get(self.user, query_server=query_server)
 
-    notebook = {}
 
-    if facet:
-      snippet = facet['snippet']
-    else:
-      snippet = dashboard['snippet']
+  def fetch_result(self, dashboard, query, facet):
+    notebook = {}
+    snippet = facet['queryResult']
 
-    start_over = True
-    
+    start_over = True # TODO
 
     result = get_api(MockRequest(self.user), snippet).fetch_result(
         notebook,
@@ -132,22 +114,18 @@ class SQLApi():
         dashboard['template']['rows'],
         start_over=start_over
     )
-    
-    result['has_more']
-
 
-#     if handle:
-#       result = db.fetch(handle, rows=dashboard['template']['rows'])
-#       db.close(handle)
+    result['has_more'] # TODO
 
     # TODO: add query id to allow closing
-    if facet:
-      if facet['type'] == 'function':
-        return self._convert_impala_function_facet(result, facet, query)
-      else:
-        return self._convert_impala_facet(result, facet, query)
-    else:
+
+    if not facet.get('type'):
       return self._convert_impala_results(result, dashboard, query)
+    elif facet['type'] == 'function':
+      return self._convert_impala_function_facet(result, facet, query)
+    else:
+      return self._convert_impala_facet(result, facet, query) # query needed
+
 
   def datasets(self, show_all=False):
 #     database, table = self._get_database_table_names(dashboard['name'])

+ 9 - 3
apps/search/src/search/api.py

@@ -45,12 +45,17 @@ def search(request):
 
   collection = json.loads(request.POST.get('collection', '{}'))
   query = json.loads(request.POST.get('query', '{}'))
-  facet = json.loads(request.POST.get('facet', '{}'))  
+  facet = json.loads(request.POST.get('facet', '{}')) 
+
   query['download'] = 'download' in request.POST
+  fetch_result = 'fetch_result' in request.POST
 
   if collection:
     try:
-      response = get_engine(request.user, collection).query(collection, query, facet)
+      if fetch_result:
+        response = get_engine(request.user, collection).fetch_result(collection, query, facet)
+      else:
+        response = get_engine(request.user, collection).query(collection, query, facet)
     except RestException, e:
       try:
         message = json.loads(e.message)
@@ -453,7 +458,8 @@ def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_t
         'isGridLayout': False,
         "hasDataForChart": True,
         "rows": 25,
-    }
+    },
+    'queryResult': {}
   }
 
 

+ 2 - 0
apps/search/src/search/api_engines.py

@@ -65,6 +65,8 @@ class DashboardApi(object):
 
   def close_query(self, collection, query, facet=None): pass
 
+  def fetch_result(self, collection, query, facet=None): pass
+
 
 class SearchApi(DashboardApi):
 

+ 129 - 35
apps/search/src/search/static/search/js/search.ko.js

@@ -677,6 +677,8 @@ var Collection = function (vm, collection) {
             (newValue == ko.HUE_CHARTS.TYPES.GRADIENTMAP ? 'gradient-map-widget' : 'bucket-widget'))
         );
       });
+
+      // TODO queryResult reload QueryResult
     }
   }
 
@@ -1415,6 +1417,17 @@ var NewTemplate = function (vm, initial) {
 };
 
 
+var QueryResult = function (vm, initial) {
+  var self = this;
+
+  self.type = ko.mapping.fromJS(initial.type);
+  self.status = ko.mapping.fromJS(initial.status || 'running');
+  self.progress = ko.mapping.fromJS(initial.progress || 0);
+
+  self.result = ko.mapping.fromJS(initial.result);
+};
+
+
 var DATE_TYPES = ['date', 'tdate'];
 var NUMBER_TYPES = ['int', 'tint', 'long', 'tlong', 'float', 'tfloat', 'double', 'tdouble', 'currency'];
 var FLOAT_TYPES = ['float', 'tfloat', 'double', 'tdouble'];
@@ -1588,6 +1601,66 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     self.search();
   };
 
+  self.checkStatus = function (facet) { // common?
+      $.post("/notebook/api/check_status", {
+        notebook: ko.mapping.toJSON({type: facet.queryResult.type()}),
+        snippet: ko.mapping.toJSON(facet.queryResult)
+      }, function (data) {
+        if (facet.queryResult.status() == 'canceled') {
+          // Query was canceled in the meantime, do nothing
+        } else {
+
+          if (data.status == 0) {
+        	  facet.queryResult.status(data.query_status.status);
+
+            if (facet.queryResult.status() == 'running' || facet.queryResult.status() == 'starting') {
+              // if (! notebook.unloaded()) { self.checkStatusTimeout = setTimeout(self.checkStatus, 1000); };
+              setTimeout(function() { self.checkStatus(facet); }, 1000);
+            }
+            else if (facet.queryResult.status() == 'available') {
+              self.fetchResult(facet);
+
+              facet.queryResult.progress(100);
+
+              if (facet.queryResult.result['handle'].has_result_set()) {
+                //self.fetchResultSize();
+            	console.log('fetch result size()');
+              }
+            }
+            else if (facet.queryResult.status() == 'success') {
+            	facet.queryResult.progress(99);
+            }
+          } else if (data.status == -3) {
+        	  facet.queryResult.status('expired');
+          } else {
+            //self._ajaxError(data); // common?
+        	$(document).trigger("error", data.message);
+          }
+        }
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText || textStatus);
+        facet.queryResult.status('failed');
+      });
+    };
+
+   self.fetchResult = function(facet) { // If coll grid or real facet
+	   $.post("/search/search", {
+           collection: ko.mapping.toJSON(self.collection),
+           query: ko.mapping.toJSON(self.query),
+           facet: ko.mapping.toJSON(facet),
+           fetch_result: true
+       }, function (data) {
+       	 if (facet.type) {
+           $.each(data.normalized_facets, function (index, facet) {
+             self._make_result_facet(facet);
+           });
+       	 } else {
+       	   facet._make_grid_result(data);
+       	 }
+       });
+   }
+  // self.searchAsync
+
   self.search = function (callback) {
     $(".jHueNotify").hide();
     logGA('search');
@@ -1640,15 +1713,20 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
         return $.post("/search/search", {
             collection: ko.mapping.toJSON(self.collection),
             query: ko.mapping.toJSON(self.query),
-            layout: ko.mapping.toJSON(self.columns),
             facet: ko.mapping.toJSON(facet),
         }, function (data) {
-            $.each(data.normalized_facets, function (index, new_facet) {
-              self._make_result_facet(new_facet);
-            });
-          return data;
+        	
+            var queryResult = new QueryResult(self, {
+                type: self.collection.engine(),
+                result: data,
+                status: 'running',
+                progress: 0,
+              });
+            facet.queryResult = queryResult;
+
+          	self.checkStatus(facet);
         });
-      });
+      }); // Join save dashboard history
     }
 
     $.each(self.fieldAnalyses(), function (index, analyse) { // Invalidate stats analysis
@@ -1666,35 +1744,21 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
           query: ko.mapping.toJSON(self.query),
           layout: ko.mapping.toJSON(self.columns)
         }, function (data) {
+          data = JSON.bigdataParse(data);
           try {
-            data = JSON.bigdataParse(data);
-
-            if (typeof callback === "function") {
-              callback(data);
-            }
-
-            $.each(data.normalized_facets, function (index, new_facet) {
-              self._make_result_facet(new_facet);
-            });
-
-            // Delete norm_facets that were deleted
-            self.response(data);
-
-            if (data.error) {
-              $(document).trigger("error", data.error);
-            }
-            else {
-              var _resultsHash = ko.mapping.toJSON(data.response.docs);
-
-              if (self.resultsHash != _resultsHash) {
-                var _docs = [];
-                var _mustacheTmpl = self.collection.template.isGridLayout() ? "" : fixTemplateDotsAndFunctionNames(self.collection.template.template());
-                $.each(data.response.docs, function (index, item) {
-                  _docs.push(self._make_result_doc(item, _mustacheTmpl, self.collection.template));
-                });
-                self.results(_docs);
-              }
-              self.resultsHash = _resultsHash;
+            if (self.collection.engine() != 'impala') {
+              self._make_grid_result(data, callback);
+            } else {
+                var query = new QueryResult(self, {
+                    type: self.collection.engine(),
+                    result: data,
+                    status: 'running',
+                    progress: 0,
+                  });
+
+                self.queryResult = query; // Todo add to model
+
+              	self.checkStatus(self);
             }
           }
           catch (e) {
@@ -1731,11 +1795,41 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     });
   };
 
+  self._make_grid_result = function(data, callback) {
+      if (typeof callback === "function") {
+        callback(data);
+      }
+
+      $.each(data.normalized_facets, function (index, new_facet) {
+        self._make_result_facet(new_facet);
+      });
+
+      // Delete norm_facets that were deleted
+      self.response(data);
+
+      if (data.error) {
+        $(document).trigger("error", data.error);
+      }
+      else {
+        var _resultsHash = ko.mapping.toJSON(data.response.docs);
+
+        if (self.resultsHash != _resultsHash) {
+          var _docs = [];
+          var _mustacheTmpl = self.collection.template.isGridLayout() ? "" : fixTemplateDotsAndFunctionNames(self.collection.template.template());
+          $.each(data.response.docs, function (index, item) {
+            _docs.push(self._make_result_doc(item, _mustacheTmpl, self.collection.template));
+          });
+          self.results(_docs);
+        }
+        self.resultsHash = _resultsHash;
+      }
+  };
+
   self._make_result_facet = function(new_facet) {
     var facet = self.getFacetFromQuery(new_facet.id);
     var _hash = ko.mapping.toJSON(new_facet);
 
-    if (!facet.has_data() || facet.resultHash() != _hash) {
+    if (! facet.has_data() || facet.resultHash() != _hash) {
       facet.counts(new_facet.counts);
 
       if (typeof new_facet.docs != 'undefined') {