Browse Source

[search] Support multi analytics series

Support limit in sub nested facets
Romain Rigaux 10 years ago
parent
commit
e920d27df1

+ 26 - 13
apps/search/src/search/models.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
+import collections
 import itertools
 import itertools
 import json
 import json
 import logging
 import logging
@@ -603,7 +604,10 @@ def pairwise2(field, fq_filter, iterable):
   a, b = itertools.tee(iterable)
   a, b = itertools.tee(iterable)
   for element in a:
   for element in a:
     pairs.append({
     pairs.append({
-        'cat': field, 'value': element, 'count': next(a), 'selected': element in selected_values,
+        'cat': field,
+        'value': element,
+        'count': next(a),
+        'selected': element in selected_values,
         'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element])
         'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element])
     })
     })
   return pairs
   return pairs
@@ -743,27 +747,36 @@ def augment_solr_response(response, collection, query):
       elif category == 'nested' and name in response['facets']:
       elif category == 'nested' and name in response['facets']:
         value = response['facets'][name]
         value = response['facets'][name]
         collection_facet = get_facet_field(category, name, collection['facets'])
         collection_facet = get_facet_field(category, name, collection['facets'])
-        print collection_facet
+        extraSeries = []
         counts = response['facets'][name]['buckets']
         counts = response['facets'][name]['buckets']
-        print counts
 
 
+        # Date range
         if collection_facet['properties']['isDate']:
         if collection_facet['properties']['isDate']:
           dimension = 3
           dimension = 3
-          end = 1
-          counts = [_v for _f in counts for _v in (_f['val'], _f['d2'] if 'd2' in _f else _f['count'])]
-          counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
+          # Single dimension or dimension 2 with analytics
+          if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate'] not in ('count', 'unique'):
+            counts = [_v for _f in counts for _v in (_f['val'], _f['d2'] if 'd2' in _f else _f['count'])]
+            counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
+          else:
+            # Dimension 1 with counts and 2 with analytics
+            _series = collections.defaultdict(list)
+            for f in counts:
+              for bucket in (f['d2']['buckets'] if 'd2' in f else []):
+                _series[bucket['val']].append(f['val'])
+                _series[bucket['val']].append(bucket['d2'] if 'd2' in bucket else bucket['count'])
+            for name, val in _series.iteritems():
+              _c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet)
+              extraSeries.append({'counts': _c, 'label': name})
+            counts = []
         elif not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate'] not in ('count', 'unique'):
         elif not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate'] not in ('count', 'unique'):
+          # Single dimension or dimension 2 with analytics
           dimension = 1
           dimension = 1
-          # counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000}
-          # [{u'count': 5, u'val': u'CT'}, {u'count': 5, u'val': u'NJ'}, {u'count': 5, u'val': u'NY'}]
           counts = [_v for _f in counts for _v in (_f['val'], _f['d2'] if 'd2' in _f else _f['count'])]
           counts = [_v for _f in counts for _v in (_f['val'], _f['d2'] if 'd2' in _f else _f['count'])]
           counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), counts)
           counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), counts)
         else:
         else:
+          # Dimension 1 with counts and 2 with analytics
           dimension = 2
           dimension = 2
-          counts = _augment_stats_2d(name, facet, response['facets'][name]['buckets'], selected_values)
-          print counts
-
-        print dimension
+          counts = _augment_stats_2d(name, facet, counts, selected_values)
 
 
         if collection_facet['properties']['sort'] == 'asc':
         if collection_facet['properties']['sort'] == 'asc':
           counts.reverse()
           counts.reverse()
@@ -774,7 +787,7 @@ def augment_solr_response(response, collection, query):
           'type': category,
           'type': category,
           'label': collection_facet['label'],
           'label': collection_facet['label'],
           'counts': counts,
           'counts': counts,
-          'extraSeries': [], # unused?
+          'extraSeries': extraSeries,
           'dimension': dimension
           'dimension': dimension
         }
         }
 
 

+ 1 - 1
apps/search/src/search/static/search/js/search.ko.js

@@ -1067,7 +1067,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   self.draggableTree = ko.observable(bareWidgetBuilder("Tree", "tree-widget"));
   self.draggableTree = ko.observable(bareWidgetBuilder("Tree", "tree-widget"));
   self.draggableHeatmap = ko.observable(bareWidgetBuilder("Heatmap", "heatmap-widget"));
   self.draggableHeatmap = ko.observable(bareWidgetBuilder("Heatmap", "heatmap-widget"));
   self.draggableCounter = ko.observable(bareWidgetBuilder("Counter", "hit-widget"));
   self.draggableCounter = ko.observable(bareWidgetBuilder("Counter", "hit-widget"));
-  self.draggableBucket = ko.observable(bareWidgetBuilder("Histogram", "bucket-widget"));
+  self.draggableBucket = ko.observable(bareWidgetBuilder("Chart", "bucket-widget"));
 
 
   self.availableDateFields = ko.computed(function() {
   self.availableDateFields = ko.computed(function() {
     return $.grep(self.collection.availableFacetFields(), function(field) { return DATE_TYPES.indexOf(field.type()) != -1; });
     return $.grep(self.collection.availableFacetFields(), function(field) { return DATE_TYPES.indexOf(field.type()) != -1; });

+ 1 - 1
apps/search/src/search/templates/search.mako

@@ -257,7 +257,7 @@ ${ commonheader(_('Search'), "search", user, "80px") | n,unicode }
                     draggable: {data: draggableBucket(), isEnabled: availableDraggableChart,
                     draggable: {data: draggableBucket(), isEnabled: availableDraggableChart,
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
                     options: {'start': function(event, ui){lastWindowScrollPosition = $(window).scrollTop();$('.card-body').slideUp('fast');},
                               'stop': function(event, ui){$('.card-body').slideDown('fast', function(){$(window).scrollTop(lastWindowScrollPosition)});}}}"
                               'stop': function(event, ui){$('.card-body').slideDown('fast', function(){$(window).scrollTop(lastWindowScrollPosition)});}}}"
-         title="${_('Histogram Chart')}" rel="tooltip" data-placement="top">
+         title="${_('Chart')}" rel="tooltip" data-placement="top">
          <a data-bind="style: { cursor: $root.availableDraggableChart() ? 'move' : 'default' }">
          <a data-bind="style: { cursor: $root.availableDraggableChart() ? 'move' : 'default' }">
                        <i class="hcha hcha-bar-chart"></i>
                        <i class="hcha hcha-bar-chart"></i>
          </a>
          </a>

+ 26 - 9
desktop/libs/libsolr/src/libsolr/api.py

@@ -54,13 +54,11 @@ class SolrApi(object):
       self._client.set_kerberos_auth()
       self._client.set_kerberos_auth()
     self._root = resource.Resource(self._client)
     self._root = resource.Resource(self._client)
 
 
-
   def _get_params(self):
   def _get_params(self):
     if self.security_enabled:
     if self.security_enabled:
       return (('doAs', self._user ),)
       return (('doAs', self._user ),)
     return (('user.name', DEFAULT_USER), ('doAs', self._user),)
     return (('user.name', DEFAULT_USER), ('doAs', self._user),)
 
 
-
   def _get_q(self, query):
   def _get_q(self, query):
     q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
     q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
     return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
     return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
@@ -80,50 +78,62 @@ class SolrApi(object):
     GAPS = {
     GAPS = {
         '5MINUTES': {
         '5MINUTES': {
             'histogram-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
             'histogram-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
+            'bucket-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
             'facet-widget': {'coeff': '+1', 'unit': 'MINUTES'}, # ~10 slots
             'facet-widget': {'coeff': '+1', 'unit': 'MINUTES'}, # ~10 slots
         },
         },
         '30MINUTES': {
         '30MINUTES': {
             'histogram-widget': {'coeff': '+20', 'unit': 'SECONDS'},
             'histogram-widget': {'coeff': '+20', 'unit': 'SECONDS'},
+            'bucket-widget': {'coeff': '+20', 'unit': 'SECONDS'},
             'facet-widget': {'coeff': '+5', 'unit': 'MINUTES'},
             'facet-widget': {'coeff': '+5', 'unit': 'MINUTES'},
         },
         },
         '1HOURS': {
         '1HOURS': {
             'histogram-widget': {'coeff': '+30', 'unit': 'SECONDS'},
             'histogram-widget': {'coeff': '+30', 'unit': 'SECONDS'},
+            'bucket-widget': {'coeff': '+30', 'unit': 'SECONDS'},
             'facet-widget': {'coeff': '+10', 'unit': 'MINUTES'},
             'facet-widget': {'coeff': '+10', 'unit': 'MINUTES'},
         },
         },
         '12HOURS': {
         '12HOURS': {
             'histogram-widget': {'coeff': '+7', 'unit': 'MINUTES'},
             'histogram-widget': {'coeff': '+7', 'unit': 'MINUTES'},
+            'bucket-widget': {'coeff': '+7', 'unit': 'MINUTES'},
             'facet-widget': {'coeff': '+1', 'unit': 'HOURS'},
             'facet-widget': {'coeff': '+1', 'unit': 'HOURS'},
         },
         },
         '1DAYS': {
         '1DAYS': {
             'histogram-widget': {'coeff': '+15', 'unit': 'MINUTES'},
             'histogram-widget': {'coeff': '+15', 'unit': 'MINUTES'},
+            'bucket-widget': {'coeff': '+15', 'unit': 'MINUTES'},
             'facet-widget': {'coeff': '+3', 'unit': 'HOURS'},
             'facet-widget': {'coeff': '+3', 'unit': 'HOURS'},
         },
         },
         '2DAYS': {
         '2DAYS': {
             'histogram-widget': {'coeff': '+30', 'unit': 'MINUTES'},
             'histogram-widget': {'coeff': '+30', 'unit': 'MINUTES'},
+            'bucket-widget': {'coeff': '+30', 'unit': 'MINUTES'},
             'facet-widget': {'coeff': '+6', 'unit': 'HOURS'},
             'facet-widget': {'coeff': '+6', 'unit': 'HOURS'},
         },
         },
         '7DAYS': {
         '7DAYS': {
             'histogram-widget': {'coeff': '+3', 'unit': 'HOURS'},
             'histogram-widget': {'coeff': '+3', 'unit': 'HOURS'},
+            'bucket-widget': {'coeff': '+3', 'unit': 'HOURS'},
             'facet-widget': {'coeff': '+1', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+1', 'unit': 'DAYS'},
         },
         },
         '1MONTHS': {
         '1MONTHS': {
             'histogram-widget': {'coeff': '+12', 'unit': 'HOURS'},
             'histogram-widget': {'coeff': '+12', 'unit': 'HOURS'},
+            'bucket-widget': {'coeff': '+12', 'unit': 'HOURS'},
             'facet-widget': {'coeff': '+5', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+5', 'unit': 'DAYS'},
         },
         },
         '3MONTHS': {
         '3MONTHS': {
             'histogram-widget': {'coeff': '+1', 'unit': 'DAYS'},
             'histogram-widget': {'coeff': '+1', 'unit': 'DAYS'},
+            'bucket-widget': {'coeff': '+1', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+30', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+30', 'unit': 'DAYS'},
         },
         },
         '1YEARS': {
         '1YEARS': {
             'histogram-widget': {'coeff': '+3', 'unit': 'DAYS'},
             'histogram-widget': {'coeff': '+3', 'unit': 'DAYS'},
+            'bucket-widget': {'coeff': '+3', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+12', 'unit': 'MONTHS'},
             'facet-widget': {'coeff': '+12', 'unit': 'MONTHS'},
         },
         },
         '2YEARS': {
         '2YEARS': {
             'histogram-widget': {'coeff': '+7', 'unit': 'DAYS'},
             'histogram-widget': {'coeff': '+7', 'unit': 'DAYS'},
+            'bucket-widget': {'coeff': '+7', 'unit': 'DAYS'},
             'facet-widget': {'coeff': '+3', 'unit': 'MONTHS'},
             'facet-widget': {'coeff': '+3', 'unit': 'MONTHS'},
         },
         },
         '10YEARS': {
         '10YEARS': {
             'histogram-widget': {'coeff': '+1', 'unit': 'MONTHS'},
             'histogram-widget': {'coeff': '+1', 'unit': 'MONTHS'},
+            'bucket-widget': {'coeff': '+1', 'unit': 'MONTHS'},
             'facet-widget': {'coeff': '+1', 'unit': 'YEARS'},
             'facet-widget': {'coeff': '+1', 'unit': 'YEARS'},
         }
         }
     }
     }
@@ -148,6 +158,14 @@ class SolrApi(object):
 
 
     return props
     return props
 
 
+  def _get_time_filter_query(self, timeFilter, facet):
+      gap = timeFilter['gap'][facet['widgetType']]
+      return {
+        'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': gap['unit']},
+        'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': gap['unit']},
+        'gap': '%(coeff)s%(unit)s/%(unit)s' % gap, # add a 'auto'
+      }    
+
   def _get_fq(self, collection, query):
   def _get_fq(self, collection, query):
     params = ()
     params = ()
 
 
@@ -237,12 +255,7 @@ class SolrApi(object):
           }
           }
 
 
           if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'):
           if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'):
-            gap = timeFilter['gap'][facet['widgetType']]
-            keys.update({
-              'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': gap['unit']},
-              'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': gap['unit']},
-              'gap': '%(coeff)s%(unit)s/%(unit)s' % gap, # add a 'auto'
-            })
+            keys.update(self._get_time_filter_query(timeFilter, facet))
 
 
           params += (
           params += (
              ('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
              ('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
@@ -272,6 +285,8 @@ class SolrApi(object):
                 'end': facet['properties']['end'],
                 'end': facet['properties']['end'],
                 'gap': facet['properties']['gap'],
                 'gap': facet['properties']['gap'],
             })
             })
+            if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'bucket-widget'):
+              _f.update(self._get_time_filter_query(timeFilter, facet))
           else:
           else:
             _f.update({
             _f.update({
                 'type': 'terms',
                 'type': 'terms',
@@ -284,7 +299,9 @@ class SolrApi(object):
               _f['facet'] = {
               _f['facet'] = {
                   'd2': {
                   'd2': {
                       'type': 'terms',
                       'type': 'terms',
-                      'field': '%(field)s' % facet['properties']['facets'][0]
+                      'field': '%(field)s' % facet['properties']['facets'][0],
+                      'limit': int(facet['properties']['facets'][0].get('limit', 10)),
+                      'mincount': int(facet['properties']['facets'][0]['mincount'])
                   }
                   }
               }
               }
               if len(facet['properties']['facets']) > 1: # Get 3rd dimension calculation
               if len(facet['properties']['facets']) > 1: # Get 3rd dimension calculation