Bläddra i källkod

HUE-3228 [search] Add SQL facet logic

Romain Rigaux 8 år sedan
förälder
incheckning
14ab43c

+ 116 - 11
apps/impala/src/impala/dashboard_api.py

@@ -36,19 +36,28 @@ class SQLApi():
   def __init__(self, user):
     self.user = user
 
-  def query(self, dashboard, query):
+  def query(self, dashboard, query, facet):
     database, table = self._get_database_table_names(dashboard['name'])
     filters = []
 
-    fields =  '*'
-    hql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
-        'database': database,
-        'table': table,
-        'fields': fields
-    }
-    if filters:
-      hql += ' WHERE ' + filters
-    hql += ' LIMIT 100'
+    if facet:
+      hql = "SELECT %(field)s, COUNT(*) AS top FROM %(database)s.%(table)s WHERE %(field)s IS NOT NULL %(filters)s GROUP BY %(field)s ORDER BY top DESC LIMIT %(limit)s" % {
+          'database': database,
+          'table': table,
+          'field': facet['field'],
+          'filters': '',
+          'limit': 100
+      }
+    else:
+      fields =  '*'
+      hql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
+          'database': database,
+          'table': table,
+          'fields': fields
+      }
+      if filters:
+        hql += ' WHERE ' + filters
+      hql += ' LIMIT 100'
 
 #     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
 #     db = dbms.get(request.user)
@@ -72,7 +81,10 @@ class SQLApi():
       result = db.fetch(handle, rows=100)
       db.close(handle)
 
-    return self._convert_impala_results(result, dashboard, query)
+    if facet:
+      return self._convert_impala_facet(result, facet)
+    else:
+      return self._convert_impala_results(result, dashboard, query)
 
   def datasets(self):
     return ['sample_07', 'web_logs']
@@ -109,6 +121,99 @@ class SQLApi():
 
     return database, table_name
 
+  def _convert_impala_facet(self, result, facet):
+    response = json.loads('''{
+   "field":"cat",
+   "fieldsAttributes":[
+      {
+         "sort":{
+            "direction":null
+         },
+         "isDynamic":false,
+         "type":"string",
+         "name":"cat"
+      },
+      {
+         "sort":{
+            "direction":null
+         },
+         "isDynamic":false,
+         "type":"aggr",
+         "name":"count(cat)"
+      }
+   ],
+   "response":{
+      "response":{
+         "start":0,
+         "numFound":16
+      }
+   },
+   "docs":[
+      {
+         "count(cat)":12,
+         "cat":"electronics"
+      },
+      {
+         "count(cat)":4,
+         "cat":"currency"
+      },
+      {
+         "count(cat)":3,
+         "cat":"memory"
+      },
+      {
+         "count(cat)":2,
+         "cat":"connector"
+      },
+      {
+         "count(cat)":2,
+         "cat":"graphics card"
+      },
+      {
+         "count(cat)":2,
+         "cat":"hard drive"
+      },
+      {
+         "count(cat)":2,
+         "cat":"search"
+      },
+      {
+         "count(cat)":2,
+         "cat":"software"
+      },
+      {
+         "count(cat)":1,
+         "cat":"camera"
+      },
+      {
+         "count(cat)":1,
+         "cat":"copier"
+      }
+   ],
+   "counts":[],
+   "dimension":1,
+   "type":"nested",
+   "id":"6ff1f8a2-3c83-637a-0c3d-7217c39dcd9e",
+   "extraSeries":[
+
+   ],
+   "label":"cat"
+}''')
+
+    response['id'] = facet['id']
+
+    docs = []
+    for row in result.rows():
+      docs.append({
+         "count": row[1],
+         "exclude": True,
+         "selected":False,
+         "value": row[0]
+      })
+    response['counts'] = docs
+
+    return {'normalized_facets': [response]}
+
   def _convert_impala_results(self, result, dashboard, query):
     cols = list(result.cols())
 

+ 4 - 3
apps/search/src/search/api.py

@@ -44,11 +44,12 @@ def search(request):
 
   collection = json.loads(request.POST.get('collection', '{}'))
   query = json.loads(request.POST.get('query', '{}'))
+  facet = json.loads(request.POST.get('facet', '{}'))  
   query['download'] = 'download' in request.POST
 
   if collection:
     try:
-      response = get_engine(request.user, collection).query(collection, query)
+      response = get_engine(request.user, collection).query(collection, query, facet)
     except RestException, e:
       try:
         message = json.loads(e.message)
@@ -353,8 +354,8 @@ def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_t
     properties['scope'] = 'world'
     properties['limit'] = 100
   else:
-    solr_api = SolrApi(SOLR_URL.get(), user)
-    range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type)
+    api = get_engine(user, collection)
+    range_properties = _new_range_facet(api, collection, facet_field, widget_type)
 
     if range_properties:
       facet_type = 'range'

+ 2 - 2
apps/search/src/search/api_engines.py

@@ -45,7 +45,7 @@ class DashboardApi(object):
 
   def datasets(self): pass
 
-  def query(self, collection, query): pass
+  def query(self, collection, query, facet): pass
 
   def suggest(self, collection, query): pass
 
@@ -70,7 +70,7 @@ class SearchApi(DashboardApi):
     DashboardApi.__init__(self, user)
     self.api = SolrApi(SOLR_URL.get(), self.user)
 
-  def query(self, collection, query):
+  def query(self, collection, query, facet=None):
     response = self.api.query(collection, query)
     return augment_solr_response(response, collection, query)
   

+ 62 - 41
apps/search/src/search/static/search/js/search.ko.js

@@ -1629,6 +1629,22 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       });
     }
 
+    if (self.collection.engine() == 'db') {
+      multiQs = $.map(self.collection.facets(), function(facet) {
+        return $.post("/search/search", {
+            collection: ko.mapping.toJSON(self.collection),
+            query: ko.mapping.toJSON(self.query),
+            layout: ko.mapping.toJSON(self.columns),
+            facet: ko.mapping.toJSON(facet),
+        }, function (data) {
+            $.each(data.normalized_facets, function (index, new_facet) {
+              self._make_result_facet(new_facet);
+            });
+        	return data;
+        });
+      });  
+    }
+    
     $.each(self.fieldAnalyses(), function (index, analyse) { // Invalidate stats analysis
       analyse.stats.data.removeAll();
     });
@@ -1652,36 +1668,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
               }
 
               $.each(data.normalized_facets, function (index, new_facet) {
-                var facet = self.getFacetFromQuery(new_facet.id);
-                var _hash = ko.mapping.toJSON(new_facet);
-
-                if (!facet.has_data() || facet.resultHash() != _hash) {
-                  facet.counts(new_facet.counts);
-
-                  if (typeof new_facet.docs != 'undefined') {
-                    var _docs = [];
-
-                    // Update template
-                    var _facet_model = self.collection.getFacetById(new_facet.id);
-                    var _fields = []
-                    $.each(new_facet.fieldsAttributes, function(index, item) {
-                      _fields.push(ko.mapping.fromJS(item));
-                    });
-                    _facet_model.template.fieldsAttributes(_fields);
-
-                    $.each(new_facet.docs, function (index, item) {
-                      _docs.push(self._make_result_doc(item, "", _facet_model.template));
-                    });
-                    facet.results(_docs);
-                    facet.response(new_facet.response);
-                  }
-                  facet.label(new_facet.label);
-                  facet.field(new_facet.field);
-                  facet.dimension(new_facet.dimension);
-                  facet.extraSeries(typeof new_facet.extraSeries != 'undefined' ? new_facet.extraSeries : []);
-                  facet.resultHash(_hash);
-                  facet.has_data(true);
-                }
+                self._make_result_facet(new_facet);
               });
 
               // Delete norm_facets that were deleted
@@ -1712,18 +1699,19 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       ].concat(multiQs)
     )
       .done(function () {
-        if (arguments[0] instanceof Array) { // If multi queries
-          var histograms = self.collection.getHistogramFacets();
-          for (var h = 0; h < histograms.length; h++) { // Do not use $.each here
-            var histoFacetId = histograms[h].id();
-            var histoFacet = self.getFacetFromQuery(histoFacetId);
-            var _series = [];
-            for (var i = 1; i < arguments.length; i++) {
-              _series.push(arguments[i][0]['series']);
-            }
-            histoFacet.extraSeries(_series);
+        if (arguments[0] instanceof Array) {
+          if (self.collection.engine() != 'db') { // If multi queries
+	          var histograms = self.collection.getHistogramFacets();
+	          for (var h = 0; h < histograms.length; h++) { // Do not use $.each here
+	            var histoFacetId = histograms[h].id();
+	            var histoFacet = self.getFacetFromQuery(histoFacetId);
+	            var _series = [];
+	            for (var i = 1; i < arguments.length; i++) {
+	              _series.push(arguments[i][0]['series']);
+	            }
+	            histoFacet.extraSeries(_series);
+	          }
           }
-          ;
           self.response.valueHasMutated();
         }
       })
@@ -1737,6 +1725,39 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       });
   };
 
+  self._make_result_facet = function(new_facet) {
+      var facet = self.getFacetFromQuery(new_facet.id);
+      var _hash = ko.mapping.toJSON(new_facet);
+
+      if (!facet.has_data() || facet.resultHash() != _hash) {
+        facet.counts(new_facet.counts);
+
+        if (typeof new_facet.docs != 'undefined') {
+          var _docs = [];
+
+          // Update template
+          var _facet_model = self.collection.getFacetById(new_facet.id);
+          var _fields = []
+          $.each(new_facet.fieldsAttributes, function(index, item) {
+            _fields.push(ko.mapping.fromJS(item));
+          });
+          _facet_model.template.fieldsAttributes(_fields);
+
+          $.each(new_facet.docs, function (index, item) {
+            _docs.push(self._make_result_doc(item, "", _facet_model.template));
+          });
+          facet.results(_docs);
+          facet.response(new_facet.response);
+        }
+        facet.label(new_facet.label);
+        facet.field(new_facet.field);
+        facet.dimension(new_facet.dimension);
+        facet.extraSeries(typeof new_facet.extraSeries != 'undefined' ? new_facet.extraSeries : []);
+        facet.resultHash(_hash);
+        facet.has_data(true);
+      }
+  }
+  
   self._make_result_doc = function(item, _mustacheTmpl, template) {
       var row = [];
       var leafletmap = {};