浏览代码

HUE-3228 [dashboard] Migrate to Notebook API

Romain Rigaux 8 年之前
父节点
当前提交
0f0121e

+ 56 - 23
apps/impala/src/impala/dashboard_api.py

@@ -26,7 +26,8 @@ from beeswax.server.dbms import get_query_server_config
 from beeswax.design import hql_query
 from beeswax.server import dbms
 from search.models import Collection2
-from desktop.lib.i18n import force_unicode
+from notebook.models import make_notebook
+from notebook.connectors.base import get_api
 
 
 LOG = logging.getLogger(__name__)
@@ -34,6 +35,9 @@ LOG = logging.getLogger(__name__)
 
 LIMIT = 100
 
+class MockRequest():
+  def __init__(self, user):
+    self.user = user
 
 # To Split in Impala, DBMS..
 # To inherit from DashboardApi
@@ -51,11 +55,11 @@ class SQLApi():
     filters = self._get_fq(dashboard, query, facet)
 
     if facet:
-      if facet['type'] == 'field':
+      if facet['type'] == 'nested':
         fields = [facet['field']] + [f['field'] for f in facet['properties']['facets']]
         fields = ['`%s`' % f for f in fields]
 
-        hql = '''SELECT %(fields)s, COUNT(*)
+        sql = '''SELECT %(fields)s, COUNT(*)
         FROM %(database)s.%(table)s
         WHERE %(filters)s
         GROUP BY %(fields)s
@@ -68,7 +72,7 @@ class SQLApi():
             'limit': LIMIT
         }
       elif facet['type'] == 'function': # 1 dim only now
-        hql = '''SELECT %(fields)s
+        sql = '''SELECT %(fields)s
         FROM %(database)s.%(table)s
         %(filters)s''' % {
             'database': database,
@@ -78,14 +82,14 @@ class SQLApi():
         }
     else:
       fields =  '*'
-      hql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
+      sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
           'database': database,
           'table': table,
           'fields': fields
       }
       if filters:
-        hql += ' WHERE ' + ' AND '.join(filters)
-      hql += ' LIMIT %s' % LIMIT
+        sql += ' WHERE ' + ' AND '.join(filters)
+      sql += ' LIMIT %s' % LIMIT
 
 #     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
 #     db = dbms.get(request.user)
@@ -99,15 +103,42 @@ class SQLApi():
 #         ]
 #     }
 
-    query_server = get_query_server_config(name='impala') # To move to notebook API
-    db = dbms.get(self.user, query_server=query_server)
+    editor = make_notebook(
+        name='Execute and watch',
+        editor_type='impala',
+        statement=sql,
+        database=database,
+        status='ready-execute'
+    )
+    return editor.execute(MockRequest(self.user))
+  
+  def fetch_result(self, dashboard, query, facet=None):
+#     query_server = get_query_server_config(name='impala') # To move to notebook API
+#     db = dbms.get(self.user, query_server=query_server)
+
+    notebook = {}
+
+    if facet:
+      snippet = facet['snippet']
+    else:
+      snippet = dashboard['snippet']
+
+    start_over = True
+    
+
+    result = get_api(MockRequest(self.user), snippet).fetch_result(
+        notebook,
+        snippet,
+        dashboard['template']['rows'],
+        start_over=start_over
+    )
+    
+    result['has_more']
 
-    sql_query = hql_query(hql)
-    handle = db.execute_and_wait(sql_query, timeout_sec=35.0)
 
-    if handle:
-      result = db.fetch(handle, rows=dashboard['template']['rows'])
-      db.close(handle)
+#     if handle:
+#       result = db.fetch(handle, rows=dashboard['template']['rows'])
+#       db.close(handle)
 
     # TODO: add query id to allow closing
     if facet:
@@ -118,7 +149,9 @@ class SQLApi():
     else:
       return self._convert_impala_results(result, dashboard, query)
 
-  def datasets(self,  show_all=False):
+  def datasets(self, show_all=False):
+#     database, table = self._get_database_table_names(dashboard['name'])
+#     autocomplete_data = get_api(MockRequest(self.user), snippet).autocomplete(snippet, database, table, column, nested)
     return ['sample_07', 'web_logs']
 
   def fields(self, dashboard):
@@ -237,17 +270,17 @@ class SQLApi():
     response['field'] = facet['field']
     response['label'] = facet['field']
 
-    cols = list(result.cols())
-    rows = list(result.rows())
+    cols = [col['name'] for col in result['meta']]
+    rows = list(result['data']) # escape_rows
 
     response['fieldsAttributes'] = [{
          "sort":{
             "direction": None
          },
          "isDynamic": False,
-         "type": column.type,
-         "name": column.name
-      } for column in result.data_table.cols()]
+         "type": column['type'],
+         "name": column['name']
+      } for column in result['meta']]
 
     response['docs'] = [dict((header, cell) for header, cell in zip(cols, row)) for row in rows]
 
@@ -269,7 +302,7 @@ class SQLApi():
 
 
   def _convert_impala_function_facet(self, result, facet, query):
-    rows = list(result.rows())
+    rows = list(result['data'])
 
     response = {"query": facet['id'], "counts": rows[0][0], "type": "function", "id": facet['id'], "label": facet['id']}
 
@@ -277,10 +310,10 @@ class SQLApi():
 
 
   def _convert_impala_results(self, result, dashboard, query):
-    cols = list(result.cols())
+    cols = [col['name'] for col in result['meta']]
 
     docs = []
-    for row in result.rows():
+    for row in result['data']:
       docs.append(dict((header, cell) for header, cell in zip(cols, row)))
 
     response = json.loads('''{

+ 3 - 3
apps/metastore/src/metastore/templates/metastore.mako

@@ -795,9 +795,9 @@ ${ components.menubar() }
       <li><a href="#queries" data-toggle="tab" data-bind="click: function(){ $root.currentTab('table-queries'); }">${_('Queries')} (<span data-bind="text: $root.database().table().optimizerDetails().queryCount"></span>)</a></li>
       <li><a href="#joins" data-toggle="tab" data-bind="click: function(){ $root.currentTab('table-joins'); }">${_('Joins')} (<span data-bind="text: $root.database().table().optimizerDetails().joinCount"></span>)</a></li>
       <!-- /ko -->
-      <!-- ko if: $root.database().table().relationshipsDetails() -->
-      <li><a href="#relationships" data-toggle="tab" data-bind="click: function(){ $root.currentTab('table-relationships'); }">${_('Relationships')} (<span data-bind="text: $root.database().table().relationshipsDetails().inputs().length + $root.database().table().relationshipsDetails().targets().length"></span>)</a></li>
-      <!-- /ko -->
+      ##<!-- ko if: $root.database().table().relationshipsDetails() -->
+      ##<li><a href="#relationships" data-toggle="tab" data-bind="click: function(){ $root.currentTab('table-relationships'); }">${_('Relationships')} (<span data-bind="text: $root.database().table().relationshipsDetails().inputs().length + $root.database().table().relationshipsDetails().targets().length"></span>)</a></li>
+      ##<!-- /ko -->
     <!-- /ko -->
     <li><a href="#details" data-toggle="tab" data-bind="click: function(){ $root.currentTab('table-details'); }">${ _('Details') }</a></li>
   </ul>

+ 116 - 116
apps/search/src/search/static/search/js/search.ko.js

@@ -1661,74 +1661,74 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
 
 
     $.when.apply($, [
-        $.post("/search/search", {
-            collection: ko.mapping.toJSON(self.collection),
-            query: ko.mapping.toJSON(self.query),
-            layout: ko.mapping.toJSON(self.columns)
-          }, function (data) {
-            try {
-              data = JSON.bigdataParse(data);
+      $.post("/search/search", {
+          collection: ko.mapping.toJSON(self.collection),
+          query: ko.mapping.toJSON(self.query),
+          layout: ko.mapping.toJSON(self.columns)
+        }, function (data) {
+          try {
+            data = JSON.bigdataParse(data);
 
-              if (typeof callback === "function") {
-                callback(data);
-              }
+            if (typeof callback === "function") {
+              callback(data);
+            }
 
-              $.each(data.normalized_facets, function (index, new_facet) {
-                self._make_result_facet(new_facet);
-              });
+            $.each(data.normalized_facets, function (index, new_facet) {
+              self._make_result_facet(new_facet);
+            });
 
-              // Delete norm_facets that were deleted
-              self.response(data);
+            // Delete norm_facets that were deleted
+            self.response(data);
 
-              if (data.error) {
-                $(document).trigger("error", data.error);
-              }
-              else {
-                var _resultsHash = ko.mapping.toJSON(data.response.docs);
-
-                if (self.resultsHash != _resultsHash) {
-                  var _docs = [];
-                  var _mustacheTmpl = self.collection.template.isGridLayout() ? "" : fixTemplateDotsAndFunctionNames(self.collection.template.template());
-                  $.each(data.response.docs, function (index, item) {
-                    _docs.push(self._make_result_doc(item, _mustacheTmpl, self.collection.template));
-                  });
-                  self.results(_docs);
-                }
-                self.resultsHash = _resultsHash;
-              }
+            if (data.error) {
+              $(document).trigger("error", data.error);
             }
-            catch (e) {
-              console.log(e);
+            else {
+              var _resultsHash = ko.mapping.toJSON(data.response.docs);
+
+              if (self.resultsHash != _resultsHash) {
+                var _docs = [];
+                var _mustacheTmpl = self.collection.template.isGridLayout() ? "" : fixTemplateDotsAndFunctionNames(self.collection.template.template());
+                $.each(data.response.docs, function (index, item) {
+                  _docs.push(self._make_result_doc(item, _mustacheTmpl, self.collection.template));
+                });
+                self.results(_docs);
+              }
+              self.resultsHash = _resultsHash;
             }
-          },
-          "text")
+          }
+          catch (e) {
+            console.log(e);
+          }
+        },
+        "text")
       ].concat(multiQs)
     )
-      .done(function () {
-        if (arguments[0] instanceof Array) {
-          if (self.collection.engine() != 'impala') { // If multi queries
-            var histograms = self.collection.getHistogramFacets();
-            for (var h = 0; h < histograms.length; h++) { // Do not use $.each here
-              var histoFacetId = histograms[h].id();
-              var histoFacet = self.getFacetFromQuery(histoFacetId);
-              var _series = [];
-              for (var i = 1; i < arguments.length; i++) {
-                _series.push(arguments[i][0]['series']);
-              }
-              histoFacet.extraSeries(_series);
+    .done(function () {
+      if (arguments[0] instanceof Array) {
+        if (self.collection.engine() != 'impala') { // If multi queries
+          var histograms = self.collection.getHistogramFacets();
+          for (var h = 0; h < histograms.length; h++) { // Do not use $.each here
+            var histoFacetId = histograms[h].id();
+            var histoFacet = self.getFacetFromQuery(histoFacetId);
+            var _series = [];
+            for (var i = 1; i < arguments.length; i++) {
+              _series.push(arguments[i][0]['series']);
             }
+            histoFacet.extraSeries(_series);
           }
           self.response.valueHasMutated();
         }
-      })
-      .fail(function (xhr, textStatus, errorThrown) {
-        $(document).trigger("error", xhr.responseText);
-      })
-      .always(function () {
-        self.isRetrievingResults(false);
-        self.hasRetrievedResults(true);
-        $('.btn-loading').button('reset');
-      });
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    })
+    .always(function () {
+      self.isRetrievingResults(false);
+      self.hasRetrievedResults(true);
+      $('.btn-loading').button('reset');
+    });
   };
 
   self._make_result_facet = function(new_facet) {
@@ -1765,70 +1765,70 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   }
 
   self._make_result_doc = function(item, _mustacheTmpl, template) {
-      var row = [];
-      var leafletmap = {};
-      var _externalLink = item.externalLink;
-      var _details = item.details;
-      var _id = item.hueId;
-      var _childDocuments = item._childDocuments_;
-      delete item["externalLink"];
-      delete item["details"];
-      delete item["hueId"];
-      delete item["_childDocuments_"];
-      var fields = template.fieldsSelected();
-      // Display selected fields or whole json document
-      if (fields.length != 0) {
-        $.each(template.fieldsSelected(), function (index, field) {
-          row.push(item[field]);
-        });
-      } else {
-        row.push(ko.mapping.toJSON(item));
-      }
-      if (template.leafletmapOn()) {
-        leafletmap = {
-          'latitude': item[template.leafletmap.latitudeField()],
-          'longitude': item[template.leafletmap.longitudeField()],
-          'label': template.leafletmap.labelField() ? item[template.leafletmap.labelField()] : ""
-        }
+    var row = [];
+    var leafletmap = {};
+    var _externalLink = item.externalLink;
+    var _details = item.details;
+    var _id = item.hueId;
+    var _childDocuments = item._childDocuments_;
+    delete item["externalLink"];
+    delete item["details"];
+    delete item["hueId"];
+    delete item["_childDocuments_"];
+    var fields = template.fieldsSelected();
+    // Display selected fields or whole json document
+    if (fields.length != 0) {
+      $.each(template.fieldsSelected(), function (index, field) {
+        row.push(item[field]);
+      });
+    } else {
+      row.push(ko.mapping.toJSON(item));
+    }
+    if (template.leafletmapOn()) {
+      leafletmap = {
+        'latitude': item[template.leafletmap.latitudeField()],
+        'longitude': item[template.leafletmap.longitudeField()],
+        'label': template.leafletmap.labelField() ? item[template.leafletmap.labelField()] : ""
       }
-      var doc = {
-        'id': _id,
-        'row': row,
-        'item': ko.mapping.fromJS(item),
-        'showEdit': ko.observable(false),
-        'hasChanged': ko.observable(false),
-        'externalLink': ko.observable(_externalLink),
-        'details': ko.observableArray(_details),
-        'originalDetails': ko.observable(''),
-        'showDetails': ko.observable(false),
-        'leafletmap': leafletmap
-      };
-
-      if (_childDocuments) {
-        var childRecords = [];
-        $.each(_childDocuments, function (index, item) {
-          var record = self._make_result_doc(item, _mustacheTmpl, self.collection.template);
-          $.each(item, function(key, val) {
-            var _field = ko.mapping.fromJS({
-                key: key,
-                value: val,
-                hasChanged: false
-            });
-            record.details.push(_field);
+    }
+    var doc = {
+      'id': _id,
+      'row': row,
+      'item': ko.mapping.fromJS(item),
+      'showEdit': ko.observable(false),
+      'hasChanged': ko.observable(false),
+      'externalLink': ko.observable(_externalLink),
+      'details': ko.observableArray(_details),
+      'originalDetails': ko.observable(''),
+      'showDetails': ko.observable(false),
+      'leafletmap': leafletmap
+    };
+
+    if (_childDocuments) {
+      var childRecords = [];
+      $.each(_childDocuments, function (index, item) {
+        var record = self._make_result_doc(item, _mustacheTmpl, self.collection.template);
+        $.each(item, function(key, val) {
+          var _field = ko.mapping.fromJS({
+              key: key,
+              value: val,
+              hasChanged: false
           });
-          childRecords.push(record);
+          record.details.push(_field);
         });
-        doc['childDocuments'] = ko.observable(childRecords);
-      }
-      if (!template.isGridLayout()) {
-        // fix the fields that contain dots in the name
-        addTemplateFunctions(item);
-        if (self.additionalMustache != null && typeof self.additionalMustache == "function") {
-          self.additionalMustache(item);
-        }
-        doc.content = Mustache.render(_mustacheTmpl, item);
+        childRecords.push(record);
+      });
+      doc['childDocuments'] = ko.observable(childRecords);
+    }
+    if (!template.isGridLayout()) {
+      // fix the fields that contain dots in the name
+      addTemplateFunctions(item);
+      if (self.additionalMustache != null && typeof self.additionalMustache == "function") {
+        self.additionalMustache(item);
       }
-      return doc;
+      doc.content = Mustache.render(_mustacheTmpl, item);
+    }
+    return doc;
   }
 
   self.suggest = function (query, callback) {

+ 37 - 12
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1086,26 +1086,51 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_
           <div class="span12">
             <form class="form-horizontal">
               <fieldset>
-               ${ _('Active tables') }<br/> <i class="fa fa-object-group"></i>
+               ${ _('Active tables') }<br/>
                <ul>
                  <li>sample_07 <i class="fa fa-info"></i> <i class="fa fa-warning"></i> <i class="fa fa-refresh"></i></li>
                  <li>sample_08 <i class="fa fa-info"></i> </li>
                </ul>
-               Outline?<br/>
              </fieldset>
+
             <form class="form-horizontal">
               <fieldset>
-               ${ _('Suggestions') }<br/>
-
-               Risk<br/>
-               <br/>
+                ${ _('Fields') }<br/>
+                <ul>
+                  <li>'country-code' is a popular field <a href="javascript:void(0)">add</a></li>
+                  <li>'gender' would be a good dimension with low cardinality (2) <a href="javascript:void(0)">add</a></li>
+                  <li>'ts_s=17Q1' is the latest partition <a href="javascript:void(0)">add</a></li>
+                  <li>'f1'</li>
+                  <li>'f2'</li>
+                  <li>'f3'</li>
+                  <li>'f4'</li>
+                  <li>'f5'</li>
+                </ul>
+              </fieldset>
+            </form>
+        
+            <form class="form-horizontal">
+              <fieldset>
+                ${ _('Suggestions') }<br/>
+                <ul>
+                  <li>Popular fields for the tables are: [code, salary, amount]</li>
+                  <li>The query would run 2x faster by adding a WHERE date_f > '2017-01-01'</li>
+                  <li>Parameterize the query?</li>
+                  <li>Could be automated with integrated scheduler</li>
+                  <li>Data has not been refreshed since last run 3 days ago  <i class="fa fa-warning"></i> <i class="fa fa-refresh"></i></li></li>
+                  <li>A schema change happened last week, a new column 'salary_med' was added</li>
+                  <li>Data statistics are not accurate, click to refresh them</li>
+                  <li>Query ran 17 times last week</li>
+                  <li>The datasets are sometimes joined with table [Population]</li>
+                  <li>Query would be a good candidate to run interactively with Impala</li>
+                </ul>
+              </fieldset>
+            </form>
+        
+              <a href="javascript:void(0)" data-bind="click: function() { huePubSub.publish('editor.workload.upload'); }" title="${ _('Load past query history in order to improve recommendations') }">
+                <i class="fa fa-fw fa-cloud-upload"></i> ${_('Upload workload')}
+              </a>
 
-               Did you know?<br/>
-               Ran 100 times last week<br/>
-               Could be automated with integrated scheduler<br/>
-               Share results?<br/>
-               Parameterize the query?<br/>
-             </fieldset>
            </form>
          </div>
        </div>