浏览代码

HUE-5116 [search] Support N dimension with aggregates

Romain Rigaux 9 年之前
父节点
当前提交
da5ba80

+ 41 - 17
apps/search/src/search/models.py

@@ -738,6 +738,8 @@ def augment_solr_response(response, collection, query):
 
 
   selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
   selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
 
 
+  print response.get('responseHeader').get('params').get('json.facet')
+
   if response and response.get('facet_counts'):
   if response and response.get('facet_counts'):
     for facet in collection['facets']:
     for facet in collection['facets']:
       category = facet['type']
       category = facet['type']
@@ -823,6 +825,8 @@ def augment_solr_response(response, collection, query):
         extraSeries = []
         extraSeries = []
         counts = response['facets'][name]['buckets']
         counts = response['facets'][name]['buckets']
 
 
+        # For dim in dimensions
+
         # Number or Date range
         # Number or Date range
         if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
         if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
           dimension = 3
           dimension = 3
@@ -846,15 +850,29 @@ def augment_solr_response(response, collection, query):
           count = response['facets'][name]
           count = response['facets'][name]
           _convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
           _convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
           dimension = len(facet_fields)
           dimension = len(facet_fields)
-        elif not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] not in ('count', 'unique'):
-          # Single dimension or dimension 2 with analytics
+        elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
+          # Dimension 1 with 1 count or agg
           dimension = 1
           dimension = 1
-          counts = [_v for _f in counts for _v in (_f['val'], _f['d2'] if 'd2' in _f else _f['count'])]
-          counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), counts)
+
+          column = 'count'
+          if len(collection_facet['properties']['facets']) == 1:
+            agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
+            legend = agg_keys[0].split(':', 2)[1]
+            column = agg_keys[0]
+          else:
+            legend = facet['field'] # 'count(%s)' % legend
+
+          counts = [_v for _f in counts for _v in (_f['val'], _f[column])] # TODO: Create additional ordered dict for table view + download
+          counts = pairwise2(legend, selected_values.get(facet['id'], []), counts) # TODO use 'cat' for legend in graph
         else:
         else:
-          # Dimension 1 with counts and 2 with analytics
+          # Dimension 2 with analytics or 1 with N aggregates
           dimension = 2
           dimension = 2
-          counts = _augment_stats_2d(name, facet, counts, selected_values)
+          agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('d_')] # TODO sort
+
+          counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys)
+
+          actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
+          counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
 
 
         facet = {
         facet = {
           'id': collection_facet['id'],
           'id': collection_facet['id'],
@@ -959,17 +977,18 @@ def _augment_pivot_2d(name, facet_id, counts, selected_values):
   return augmented
   return augmented
 
 
 
 
-def _augment_stats_2d(name, facet, counts, selected_values):
+def _augment_stats_2d(name, facet, counts, selected_values, agg_keys):
   fq_fields = []
   fq_fields = []
   fq_values = []
   fq_values = []
   fq_filter = []
   fq_filter = []
   _selected_values = [f['value'] for f in selected_values.get(facet['id'], [])]
   _selected_values = [f['value'] for f in selected_values.get(facet['id'], [])]
   _fields = [facet['field']] + [facet['field'] for facet in facet['properties']['facets']]
   _fields = [facet['field']] + [facet['field'] for facet in facet['properties']['facets']]
 
 
-  return __augment_stats_2d(counts, facet['field'], fq_fields, fq_values, fq_filter, _selected_values, _fields)
+  return __augment_stats_2d(counts, facet['field'], fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys)
 
 
 
 
-def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected_values, _fields):
+# Clear one dimension
+def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys):
   augmented = []
   augmented = []
 
 
   for bucket in counts:
   for bucket in counts:
@@ -979,15 +998,20 @@ def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected
     _fq_fields = fq_fields + _fields[0:1]
     _fq_fields = fq_fields + _fields[0:1]
     _fq_values = fq_values + [val]
     _fq_values = fq_values + [val]
 
 
-    if 'd3' in bucket:
-      augmented.append(_get_augmented(bucket['d3'], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
-    elif 'd2' in bucket:
-      if type(bucket['d2']) == dict:
-        augmented += __augment_stats_2d(bucket['d2']['buckets'], val, _fq_fields, _fq_values, fq_filter, _selected_values, _fields[1:])
+    for agg_key in agg_keys:
+      if agg_key == 'count':
+        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
+      elif agg_key.startswith('agg_'):
+        label = fq_values[0] if len(_fq_fields) >= 2 else agg_key.split(':', 2)[1]
+        augmented.append(_get_augmented(bucket[agg_key], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
       else:
       else:
-        augmented.append(_get_augmented(bucket['d2'], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
-    else:
-      augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
+        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
+
+        # go rec
+        _agg_keys = [key for key, value in bucket[agg_key]['buckets'][0].items() if key.lower().startswith('agg_') or key.lower().startswith('d_')] # TODO sort
+        if not _agg_keys:
+          _agg_keys.append('count')
+        augmented += __augment_stats_2d(bucket[agg_key]['buckets'], val, _fq_fields, _fq_values, fq_filter, _selected_values, _fields[1:], _agg_keys)
 
 
   return augmented
   return augmented
 
 

+ 14 - 13
apps/search/src/search/templates/common_search.mako

@@ -144,6 +144,7 @@ from django.utils.translation import ugettext as _
                        <i class="fa fa-table"></i>
                        <i class="fa fa-table"></i>
          </a>
          </a>
     </div>
     </div>
+
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableResultset() },
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableResultset() },
                     draggable: {data: draggableHtmlResultset(),
                     draggable: {data: draggableHtmlResultset(),
                     isEnabled: availableDraggableResultset,
                     isEnabled: availableDraggableResultset,
@@ -160,6 +161,7 @@ from django.utils.translation import ugettext as _
                        <i class="fa fa-code"></i>
                        <i class="fa fa-code"></i>
          </a>
          </a>
     </div>
     </div>
+
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableFilter() },
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableFilter() },
                     draggable: {data: draggableFilter(), isEnabled: availableDraggableFilter,
                     draggable: {data: draggableFilter(), isEnabled: availableDraggableFilter,
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
@@ -170,17 +172,6 @@ from django.utils.translation import ugettext as _
          </a>
          </a>
     </div>
     </div>
 
 
-    <div data-bind="visible: $root.isLatest,
-                    css: { 'draggable-widget': true, 'disabled': !availableDraggableNumbers() },
-                    draggable: {data: draggableCounter(), isEnabled: availableDraggableNumbers,
-                    options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
-                              'stop': function(event, ui){$('.card-body').slideDown('fast', function(){$(window).scrollTop(lastWindowScrollPosition)});}}}"
-         title="${_('Counter')}" rel="tooltip" data-placement="top">
-         <a data-bind="style: { cursor: $root.availableDraggableNumbers() ? 'move' : 'default' }">
-                       <i class="fa fa-superscript" style="font-size: 110%"></i>
-         </a>
-    </div>
-
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableLeaflet()},
     <div data-bind="css: { 'draggable-widget': true, 'disabled': !availableDraggableLeaflet()},
                     draggable: {data: draggableLeafletMap(), isEnabled: availableDraggableLeaflet,
                     draggable: {data: draggableLeafletMap(), isEnabled: availableDraggableLeaflet,
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
@@ -191,8 +182,18 @@ from django.utils.translation import ugettext as _
          </a>
          </a>
     </div>
     </div>
 
 
-
+    <div data-bind="visible: $root.isLatest,
+                    css: { 'draggable-widget': true, 'disabled': !availableDraggableNumbers() },
+                    draggable: {data: draggableCounter(), isEnabled: availableDraggableNumbers,
+                    options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
+                              'stop': function(event, ui){$('.card-body').slideDown('fast', function(){$(window).scrollTop(lastWindowScrollPosition)});}}}"
+         title="${_('Counter')}" rel="tooltip" data-placement="top">
+         <a data-bind="style: { cursor: $root.availableDraggableNumbers() ? 'move' : 'default' }">
+                       <i class="fa fa-superscript" style="font-size: 110%"></i>
+         </a>
+    </div>
       </%def>
       </%def>
+
       <%def name="widgets()">
       <%def name="widgets()">
     <div data-bind="visible: ! $root.isLatest(), css: { 'draggable-widget': true, 'disabled': !availableDraggableChart() },
     <div data-bind="visible: ! $root.isLatest(), css: { 'draggable-widget': true, 'disabled': !availableDraggableChart() },
                     draggable: {data: draggableFacet(), isEnabled: availableDraggableChart,
                     draggable: {data: draggableFacet(), isEnabled: availableDraggableChart,
@@ -1028,7 +1029,7 @@ ${ dashboard.layout_skeleton() }
           <div style="margin-bottom: 3px; white-space: nowrap; position:relative">
           <div style="margin-bottom: 3px; white-space: nowrap; position:relative">
             <input type="checkbox" data-bind="checkedValue: name, checked: $root.collection.template.fieldsSelected" style="margin: 0" />
             <input type="checkbox" data-bind="checkedValue: name, checked: $root.collection.template.fieldsSelected" style="margin: 0" />
             <div data-bind="text: name, css:{'field-selector': true, 'hoverable': $root.collection.template.fieldsSelected.indexOf(name()) > -1}, click: highlightColumn" style="margin-right:10px"></div>
             <div data-bind="text: name, css:{'field-selector': true, 'hoverable': $root.collection.template.fieldsSelected.indexOf(name()) > -1}, click: highlightColumn" style="margin-right:10px"></div>
-            <i class="fa fa-question-circle muted pointer analysis" data-bind="click: function(data, e) { $root.fieldAnalysesName(name()); $root.showFieldAnalysis(data, e); }, attr: {'title': '${ _ko('Click to analyze field') } ' + name() + ' (' + type() + ')'}" style="position:absolute; left: 168px; background-color: #FFF"></i>
+            <i class="fa fa-question-circle muted pointer analysis" data-bind="click: function(data, e) { $root.fieldAnalysesName(name()); $root.showFieldAnalysis(data, e); }, attr: {'title': '${ _ko('Analyze ') } ' + name() + ' (' + type() + ')'}" style="position:absolute; left: 168px; background-color: #FFF"></i>
           </div>
           </div>
         </div>
         </div>
         <div data-bind="visible: $root.collection.template.filteredAttributeFields().length == 0" style="padding-left:4px; padding-top:5px; font-size:40px; color:#CCC">
         <div data-bind="visible: $root.collection.template.filteredAttributeFields().length == 0" style="padding-left:4px; padding-top:5px; font-size:40px; color:#CCC">

+ 32 - 14
desktop/libs/libsolr/src/libsolr/api.py

@@ -172,7 +172,7 @@ class SolrApi(object):
               del _f['mincount'] # Numeric fields do not support
               del _f['mincount'] # Numeric fields do not support
 
 
           if facet['properties']['facets']:
           if facet['properties']['facets']:
-            self._n_facet_dimension(facet, _f, facet['properties']['facets'], 2)
+            self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1)
             if facet['widgetType'] == 'text-facet-widget':
             if facet['widgetType'] == 'text-facet-widget':
               _f['sort'] = {'d2': facet['properties']['sort']}
               _f['sort'] = {'d2': facet['properties']['sort']}
               # domain = '-d2:NaN' # Solr 6.4
               # domain = '-d2:NaN' # Solr 6.4
@@ -253,25 +253,40 @@ class SolrApi(object):
 
 
   def _n_facet_dimension(self, widget, _f, facets, dim):
   def _n_facet_dimension(self, widget, _f, facets, dim):
     facet = facets[0]
     facet = facets[0]
-    f_name = 'd%s' % dim
+    f_name = 'd_%02d:%s' % (dim, facet['field'])
 
 
     if facet['aggregate']['function'] == 'count':
     if facet['aggregate']['function'] == 'count':
-      _f['facet'] = {
-          f_name: {
-              'type': 'terms',
-              'field': '%(field)s' % facet,
-              'limit': int(facet.get('limit', 10)),
-              'mincount': int(facet['mincount'])
-          }
+      if dim == 1:
+        _f['facet'] = {f_name: {}}
+        _f = _f['facet']
+
+      _f[f_name] = {
+          'type': 'terms',
+          'field': '%(field)s' % facet,
+          'limit': int(facet.get('limit', 10)),
+          'mincount': int(facet['mincount'])
       }
       }
       if widget['widgetType'] == 'tree2-widget' and facets[-1]['aggregate']['function'] != 'count':
       if widget['widgetType'] == 'tree2-widget' and facets[-1]['aggregate']['function'] != 'count':
         _f['subcount'] = self._get_aggregate_function(facets[-1])
         _f['subcount'] = self._get_aggregate_function(facets[-1])
+
       if len(facets) > 1: # Get n+1 dimension
       if len(facets) > 1: # Get n+1 dimension
-        self._n_facet_dimension(widget, _f['facet'][f_name], facets[1:], dim+1)
+        if facets[1]['aggregate']['function'] == 'count':
+          self._n_facet_dimension(widget, _f['facet'][f_name], facets[1:], dim + 1)
+        else:
+          self._n_facet_dimension(widget, _f[f_name], facets[1:], dim)
     else:
     else:
+      agg_function = self._get_aggregate_function(facet)
       _f['facet'] = {
       _f['facet'] = {
-          f_name: self._get_aggregate_function(facet)
+          'agg_00:%s' % agg_function: agg_function
       }
       }
+      for i, _f_agg in enumerate(facets[1:], 1): # Get n+1 dimension
+        if _f_agg['aggregate']['function'] != 'count':
+          agg_function = self._get_aggregate_function(_f_agg)
+          _f['facet']['agg_%02d:%s' % (i, agg_function)] = agg_function
+        else:
+          print 'aaa'
+          self._n_facet_dimension(widget, _f['facet'], [_f_agg], dim + 1)
+        # else go into _get_aggregate_function()
 
 
 
 
   def suggest(self, collection, query):
   def suggest(self, collection, query):
@@ -616,7 +631,7 @@ class SolrApi(object):
 
 
     if not f['ops']:
     if not f['ops']:
       f['ops'] = [{'function': 'field', 'value': facet['field'], 'ops': []}]
       f['ops'] = [{'function': 'field', 'value': facet['field'], 'ops': []}]
-    
+
     return self.__get_aggregate_function(f)
     return self.__get_aggregate_function(f)
 
 
   def __get_aggregate_function(self, f):
   def __get_aggregate_function(self, f):
@@ -626,10 +641,13 @@ class SolrApi(object):
       fields = []
       fields = []
       for _f in f['ops']:
       for _f in f['ops']:
         fields.append(self.__get_aggregate_function(_f))
         fields.append(self.__get_aggregate_function(_f))
-      if f['function'] == 'percentile':
+      if f['function'] == 'median':
+        f['function'] = 'percentile'
+        fields.append('50')
+      elif f['function'] == 'percentiles':
         fields.extend(map(lambda a: str(a), [_p['value'] for _p in f['percentiles']]))
         fields.extend(map(lambda a: str(a), [_p['value'] for _p in f['percentiles']]))
       return '%s(%s)' % (f['function'], ','.join(fields))
       return '%s(%s)' % (f['function'], ','.join(fields))
-    
+
   def _get_range_borders(self, collection, query):
   def _get_range_borders(self, collection, query):
     props = {}
     props = {}
     GAPS = {
     GAPS = {