|
@@ -68,206 +68,6 @@ class SolrApi(object):
|
|
|
if self.security_enabled:
|
|
if self.security_enabled:
|
|
|
self._root.invoke('HEAD', '/')
|
|
self._root.invoke('HEAD', '/')
|
|
|
|
|
|
|
|
- def _get_params(self):
|
|
|
|
|
- if self.security_enabled:
|
|
|
|
|
- return (('doAs', self._user ),)
|
|
|
|
|
- return (('user.name', SERVER_USER.get()), ('doAs', self._user),)
|
|
|
|
|
-
|
|
|
|
|
- def _get_q(self, query):
|
|
|
|
|
- q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
|
|
|
|
|
- return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
|
|
|
|
|
-
|
|
|
|
|
- def _get_aggregate_function(self, facet):
|
|
|
|
|
- props = {
|
|
|
|
|
- 'field': facet['field'],
|
|
|
|
|
- 'aggregate': facet['properties']['aggregate'] if 'properties' in facet else facet['aggregate']
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- if props['aggregate'] == 'median':
|
|
|
|
|
- return 'percentile(%(field)s,50)' % props
|
|
|
|
|
- else:
|
|
|
|
|
- return '%(aggregate)s(%(field)s)' % props
|
|
|
|
|
-
|
|
|
|
|
- def _get_range_borders(self, collection, query):
|
|
|
|
|
- props = {}
|
|
|
|
|
- GAPS = {
|
|
|
|
|
- '5MINUTES': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
- 'timeline-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
- 'bucket-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
- 'bar-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
- 'facet-widget': {'coeff': '+1', 'unit': 'MINUTES'}, # ~10 slots
|
|
|
|
|
- },
|
|
|
|
|
- '30MINUTES': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+5', 'unit': 'MINUTES'},
|
|
|
|
|
- },
|
|
|
|
|
- '1HOURS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+10', 'unit': 'MINUTES'},
|
|
|
|
|
- },
|
|
|
|
|
- '12HOURS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+1', 'unit': 'HOURS'},
|
|
|
|
|
- },
|
|
|
|
|
- '1DAYS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
- },
|
|
|
|
|
- '2DAYS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+6', 'unit': 'HOURS'},
|
|
|
|
|
- },
|
|
|
|
|
- '7DAYS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
- },
|
|
|
|
|
- '1MONTHS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+5', 'unit': 'DAYS'},
|
|
|
|
|
- },
|
|
|
|
|
- '3MONTHS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+30', 'unit': 'DAYS'},
|
|
|
|
|
- },
|
|
|
|
|
- '1YEARS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+12', 'unit': 'MONTHS'},
|
|
|
|
|
- },
|
|
|
|
|
- '2YEARS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+3', 'unit': 'MONTHS'},
|
|
|
|
|
- },
|
|
|
|
|
- '10YEARS': {
|
|
|
|
|
- 'histogram-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
- 'timeline-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
- 'bucket-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
- 'bar-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
- 'facet-widget': {'coeff': '+1', 'unit': 'YEARS'},
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- time_field = collection['timeFilter'].get('field')
|
|
|
|
|
-
|
|
|
|
|
- if time_field and (collection['timeFilter']['value'] != 'all' or collection['timeFilter']['type'] == 'fixed'):
|
|
|
|
|
- # fqs overrides main time filter
|
|
|
|
|
- fq_time_ids = [fq['id'] for fq in query['fqs'] if fq['field'] == time_field]
|
|
|
|
|
- props['time_filter_overrides'] = fq_time_ids
|
|
|
|
|
- props['time_field'] = time_field
|
|
|
|
|
-
|
|
|
|
|
- if collection['timeFilter']['type'] == 'rolling':
|
|
|
|
|
- props['field'] = collection['timeFilter']['field']
|
|
|
|
|
- props['from'] = 'NOW-%s' % collection['timeFilter']['value']
|
|
|
|
|
- props['to'] = 'NOW'
|
|
|
|
|
- props['gap'] = GAPS.get(collection['timeFilter']['value'])
|
|
|
|
|
- elif collection['timeFilter']['type'] == 'fixed':
|
|
|
|
|
- props['field'] = collection['timeFilter']['field']
|
|
|
|
|
- props['from'] = collection['timeFilter']['from']
|
|
|
|
|
- props['to'] = collection['timeFilter']['to']
|
|
|
|
|
- props['fixed'] = True
|
|
|
|
|
-
|
|
|
|
|
- return props
|
|
|
|
|
-
|
|
|
|
|
- def _get_time_filter_query(self, timeFilter, facet):
|
|
|
|
|
- if 'fixed' in timeFilter:
|
|
|
|
|
- props = {}
|
|
|
|
|
- stat_facet = {'min': timeFilter['from'], 'max': timeFilter['to']}
|
|
|
|
|
- _compute_range_facet(facet['widgetType'], stat_facet, props, stat_facet['min'], stat_facet['max'])
|
|
|
|
|
- gap = props['gap']
|
|
|
|
|
- unit = re.split('\d+', gap)[1]
|
|
|
|
|
- return {
|
|
|
|
|
- 'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': unit},
|
|
|
|
|
- 'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': unit},
|
|
|
|
|
- 'gap': '%(gap)s' % props, # add a 'auto'
|
|
|
|
|
- }
|
|
|
|
|
- else:
|
|
|
|
|
- gap = timeFilter['gap'][facet['widgetType']]
|
|
|
|
|
- return {
|
|
|
|
|
- 'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': gap['unit']},
|
|
|
|
|
- 'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': gap['unit']},
|
|
|
|
|
- 'gap': '%(coeff)s%(unit)s/%(unit)s' % gap, # add a 'auto'
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- def _get_fq(self, collection, query):
|
|
|
|
|
- params = ()
|
|
|
|
|
- timeFilter = {}
|
|
|
|
|
-
|
|
|
|
|
- if collection:
|
|
|
|
|
- timeFilter = self._get_range_borders(collection, query)
|
|
|
|
|
- if timeFilter and not timeFilter.get('time_filter_overrides'):
|
|
|
|
|
- params += (('fq', urllib.unquote(utf_quoter('%(field)s:[%(from)s TO %(to)s]' % timeFilter))),)
|
|
|
|
|
-
|
|
|
|
|
- # Merge facets queries on same fields
|
|
|
|
|
- grouped_fqs = groupby(query['fqs'], lambda x: (x['type'], x['field']))
|
|
|
|
|
- merged_fqs = []
|
|
|
|
|
- for key, group in grouped_fqs:
|
|
|
|
|
- field_fq = next(group)
|
|
|
|
|
- for fq in group:
|
|
|
|
|
- for f in fq['filter']:
|
|
|
|
|
- field_fq['filter'].append(f)
|
|
|
|
|
- merged_fqs.append(field_fq)
|
|
|
|
|
-
|
|
|
|
|
- for fq in merged_fqs:
|
|
|
|
|
- if fq['type'] == 'field':
|
|
|
|
|
- fields = fq['field'] if type(fq['field']) == list else [fq['field']] # 2D facets support
|
|
|
|
|
- for field in fields:
|
|
|
|
|
- f = []
|
|
|
|
|
- for _filter in fq['filter']:
|
|
|
|
|
- values = _filter['value'] if type(_filter['value']) == list else [_filter['value']] # 2D facets support
|
|
|
|
|
- if fields.index(field) < len(values): # Lowest common field denominator
|
|
|
|
|
- value = values[fields.index(field)]
|
|
|
|
|
- exclude = '-' if _filter['exclude'] else ''
|
|
|
|
|
- if value is not None and ' ' in force_unicode(value):
|
|
|
|
|
- value = force_unicode(value).replace('"', '\\"')
|
|
|
|
|
- f.append('%s%s:"%s"' % (exclude, field, value))
|
|
|
|
|
- else:
|
|
|
|
|
- f.append('%s{!field f=%s}%s' % (exclude, field, value))
|
|
|
|
|
- _params ='{!tag=%(id)s}' % fq + ' '.join(f)
|
|
|
|
|
- params += (('fq', urllib.unquote(utf_quoter(_params))),)
|
|
|
|
|
- elif fq['type'] == 'range':
|
|
|
|
|
- params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
|
|
|
|
|
- utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'], f['to']))) for field, f in zip(fq['filter'], fq['properties'])])),)
|
|
|
|
|
- elif fq['type'] == 'range-up':
|
|
|
|
|
- params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
|
|
|
|
|
- utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'] if fq['is_up'] else '*', '*' if fq['is_up'] else f['from'])))
|
|
|
|
|
- for field, f in zip(fq['filter'], fq['properties'])])),)
|
|
|
|
|
- elif fq['type'] == 'map':
|
|
|
|
|
- _keys = fq.copy()
|
|
|
|
|
- _keys.update(fq['properties'])
|
|
|
|
|
- params += (('fq', '{!tag=%(id)s}' % fq + urllib.unquote(
|
|
|
|
|
- utf_quoter('%(lat)s:[%(lat_sw)s TO %(lat_ne)s} AND %(lon)s:[%(lon_sw)s TO %(lon_ne)s}' % _keys))),)
|
|
|
|
|
-
|
|
|
|
|
- return params
|
|
|
|
|
|
|
|
|
|
def query(self, collection, query):
|
|
def query(self, collection, query):
|
|
|
solr_query = {}
|
|
solr_query = {}
|
|
@@ -337,7 +137,7 @@ class SolrApi(object):
|
|
|
'field': facet['field'],
|
|
'field': facet['field'],
|
|
|
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'text-facet-widget' else 0),
|
|
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'text-facet-widget' else 0),
|
|
|
'mincount': int(facet['properties']['mincount']),
|
|
'mincount': int(facet['properties']['mincount']),
|
|
|
- 'sort': {'count': facet['properties']['sort']}
|
|
|
|
|
|
|
+ 'sort': {'count': facet['properties']['sort']},
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if 'start' in facet['properties'] and not facet['properties'].get('type') == 'field':
|
|
if 'start' in facet['properties'] and not facet['properties'].get('type') == 'field':
|
|
@@ -420,9 +220,15 @@ class SolrApi(object):
|
|
|
fields.add(collection['template']['leafletmap']['longitudeField'])
|
|
fields.add(collection['template']['leafletmap']['longitudeField'])
|
|
|
if collection['template']['leafletmap'].get('labelField'):
|
|
if collection['template']['leafletmap'].get('labelField'):
|
|
|
fields.add(collection['template']['leafletmap']['labelField'])
|
|
fields.add(collection['template']['leafletmap']['labelField'])
|
|
|
- params += (('fl', urllib.unquote(utf_quoter(','.join(list(fields))))),)
|
|
|
|
|
|
|
+ fl = urllib.unquote(utf_quoter(','.join(list(fields))))
|
|
|
else:
|
|
else:
|
|
|
- params += (('fl', '*'),)
|
|
|
|
|
|
|
+ fl = '*'
|
|
|
|
|
+
|
|
|
|
|
+ nested_fields = self._get_nested_fields(collection)
|
|
|
|
|
+ if nested_fields:
|
|
|
|
|
+ fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields)))
|
|
|
|
|
+
|
|
|
|
|
+ params += (('fl', fl),)
|
|
|
|
|
|
|
|
params += (
|
|
params += (
|
|
|
('hl', 'true'),
|
|
('hl', 'true'),
|
|
@@ -772,6 +578,225 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
|
|
|
|
+
|
|
|
|
|
+ def _get_params(self):
|
|
|
|
|
+ if self.security_enabled:
|
|
|
|
|
+ return (('doAs', self._user ),)
|
|
|
|
|
+ return (('user.name', SERVER_USER.get()), ('doAs', self._user),)
|
|
|
|
|
+
|
|
|
|
|
+ def _get_q(self, query):
|
|
|
|
|
+ q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
|
|
|
|
|
+ return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
|
|
|
|
|
+
|
|
|
|
|
+ def _get_aggregate_function(self, facet):
|
|
|
|
|
+ props = {
|
|
|
|
|
+ 'field': facet['field'],
|
|
|
|
|
+ 'aggregate': facet['properties']['aggregate'] if 'properties' in facet else facet['aggregate']
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if props['aggregate'] == 'median':
|
|
|
|
|
+ return 'percentile(%(field)s,50)' % props
|
|
|
|
|
+ else:
|
|
|
|
|
+ return '%(aggregate)s(%(field)s)' % props
|
|
|
|
|
+
|
|
|
|
|
+ def _get_range_borders(self, collection, query):
|
|
|
|
|
+ props = {}
|
|
|
|
|
+ GAPS = {
|
|
|
|
|
+ '5MINUTES': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
+ 'bar-widget': {'coeff': '+3', 'unit': 'SECONDS'}, # ~100 slots
|
|
|
|
|
+ 'facet-widget': {'coeff': '+1', 'unit': 'MINUTES'}, # ~10 slots
|
|
|
|
|
+ },
|
|
|
|
|
+ '30MINUTES': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+20', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+5', 'unit': 'MINUTES'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '1HOURS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+30', 'unit': 'SECONDS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+10', 'unit': 'MINUTES'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '12HOURS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+7', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+1', 'unit': 'HOURS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '1DAYS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+15', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '2DAYS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+30', 'unit': 'MINUTES'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+6', 'unit': 'HOURS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '7DAYS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+3', 'unit': 'HOURS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '1MONTHS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+12', 'unit': 'HOURS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+5', 'unit': 'DAYS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '3MONTHS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+1', 'unit': 'DAYS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+30', 'unit': 'DAYS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '1YEARS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+3', 'unit': 'DAYS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+12', 'unit': 'MONTHS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '2YEARS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+7', 'unit': 'DAYS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+3', 'unit': 'MONTHS'},
|
|
|
|
|
+ },
|
|
|
|
|
+ '10YEARS': {
|
|
|
|
|
+ 'histogram-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
+ 'timeline-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
+ 'bucket-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
+ 'bar-widget': {'coeff': '+1', 'unit': 'MONTHS'},
|
|
|
|
|
+ 'facet-widget': {'coeff': '+1', 'unit': 'YEARS'},
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time_field = collection['timeFilter'].get('field')
|
|
|
|
|
+
|
|
|
|
|
+ if time_field and (collection['timeFilter']['value'] != 'all' or collection['timeFilter']['type'] == 'fixed'):
|
|
|
|
|
+ # fqs overrides main time filter
|
|
|
|
|
+ fq_time_ids = [fq['id'] for fq in query['fqs'] if fq['field'] == time_field]
|
|
|
|
|
+ props['time_filter_overrides'] = fq_time_ids
|
|
|
|
|
+ props['time_field'] = time_field
|
|
|
|
|
+
|
|
|
|
|
+ if collection['timeFilter']['type'] == 'rolling':
|
|
|
|
|
+ props['field'] = collection['timeFilter']['field']
|
|
|
|
|
+ props['from'] = 'NOW-%s' % collection['timeFilter']['value']
|
|
|
|
|
+ props['to'] = 'NOW'
|
|
|
|
|
+ props['gap'] = GAPS.get(collection['timeFilter']['value'])
|
|
|
|
|
+ elif collection['timeFilter']['type'] == 'fixed':
|
|
|
|
|
+ props['field'] = collection['timeFilter']['field']
|
|
|
|
|
+ props['from'] = collection['timeFilter']['from']
|
|
|
|
|
+ props['to'] = collection['timeFilter']['to']
|
|
|
|
|
+ props['fixed'] = True
|
|
|
|
|
+
|
|
|
|
|
+ return props
|
|
|
|
|
+
|
|
|
|
|
+ def _get_time_filter_query(self, timeFilter, facet):
|
|
|
|
|
+ if 'fixed' in timeFilter:
|
|
|
|
|
+ props = {}
|
|
|
|
|
+ stat_facet = {'min': timeFilter['from'], 'max': timeFilter['to']}
|
|
|
|
|
+ _compute_range_facet(facet['widgetType'], stat_facet, props, stat_facet['min'], stat_facet['max'])
|
|
|
|
|
+ gap = props['gap']
|
|
|
|
|
+ unit = re.split('\d+', gap)[1]
|
|
|
|
|
+ return {
|
|
|
|
|
+ 'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': unit},
|
|
|
|
|
+ 'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': unit},
|
|
|
|
|
+ 'gap': '%(gap)s' % props, # add a 'auto'
|
|
|
|
|
+ }
|
|
|
|
|
+ else:
|
|
|
|
|
+ gap = timeFilter['gap'][facet['widgetType']]
|
|
|
|
|
+ return {
|
|
|
|
|
+ 'start': '%(from)s/%(unit)s' % {'from': timeFilter['from'], 'unit': gap['unit']},
|
|
|
|
|
+ 'end': '%(to)s/%(unit)s' % {'to': timeFilter['to'], 'unit': gap['unit']},
|
|
|
|
|
+ 'gap': '%(coeff)s%(unit)s/%(unit)s' % gap, # add a 'auto'
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ def _get_fq(self, collection, query):
|
|
|
|
|
+ params = ()
|
|
|
|
|
+ timeFilter = {}
|
|
|
|
|
+
|
|
|
|
|
+ if collection:
|
|
|
|
|
+ timeFilter = self._get_range_borders(collection, query)
|
|
|
|
|
+ if timeFilter and not timeFilter.get('time_filter_overrides'):
|
|
|
|
|
+ params += (('fq', urllib.unquote(utf_quoter('%(field)s:[%(from)s TO %(to)s]' % timeFilter))),)
|
|
|
|
|
+
|
|
|
|
|
+ # Merge facets queries on same fields
|
|
|
|
|
+ grouped_fqs = groupby(query['fqs'], lambda x: (x['type'], x['field']))
|
|
|
|
|
+ merged_fqs = []
|
|
|
|
|
+ for key, group in grouped_fqs:
|
|
|
|
|
+ field_fq = next(group)
|
|
|
|
|
+ for fq in group:
|
|
|
|
|
+ for f in fq['filter']:
|
|
|
|
|
+ field_fq['filter'].append(f)
|
|
|
|
|
+ merged_fqs.append(field_fq)
|
|
|
|
|
+
|
|
|
|
|
+ for fq in merged_fqs:
|
|
|
|
|
+ if fq['type'] == 'field':
|
|
|
|
|
+ fields = fq['field'] if type(fq['field']) == list else [fq['field']] # 2D facets support
|
|
|
|
|
+ for field in fields:
|
|
|
|
|
+ f = []
|
|
|
|
|
+ for _filter in fq['filter']:
|
|
|
|
|
+ values = _filter['value'] if type(_filter['value']) == list else [_filter['value']] # 2D facets support
|
|
|
|
|
+ if fields.index(field) < len(values): # Lowest common field denominator
|
|
|
|
|
+ value = values[fields.index(field)]
|
|
|
|
|
+ exclude = '-' if _filter['exclude'] else ''
|
|
|
|
|
+ if value is not None and ' ' in force_unicode(value):
|
|
|
|
|
+ value = force_unicode(value).replace('"', '\\"')
|
|
|
|
|
+ f.append('%s%s:"%s"' % (exclude, field, value))
|
|
|
|
|
+ else:
|
|
|
|
|
+ f.append('%s{!field f=%s}%s' % (exclude, field, value))
|
|
|
|
|
+ _params ='{!tag=%(id)s}' % fq + ' '.join(f)
|
|
|
|
|
+ params += (('fq', urllib.unquote(utf_quoter(_params))),)
|
|
|
|
|
+ elif fq['type'] == 'range':
|
|
|
|
|
+ params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
|
|
|
|
|
+ utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'], f['to']))) for field, f in zip(fq['filter'], fq['properties'])])),)
|
|
|
|
|
+ elif fq['type'] == 'range-up':
|
|
|
|
|
+ params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
|
|
|
|
|
+ utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'] if fq['is_up'] else '*', '*' if fq['is_up'] else f['from'])))
|
|
|
|
|
+ for field, f in zip(fq['filter'], fq['properties'])])),)
|
|
|
|
|
+ elif fq['type'] == 'map':
|
|
|
|
|
+ _keys = fq.copy()
|
|
|
|
|
+ _keys.update(fq['properties'])
|
|
|
|
|
+ params += (('fq', '{!tag=%(id)s}' % fq + urllib.unquote(
|
|
|
|
|
+ utf_quoter('%(lat)s:[%(lat_sw)s TO %(lat_ne)s} AND %(lon)s:[%(lon_sw)s TO %(lon_ne)s}' % _keys))),)
|
|
|
|
|
+
|
|
|
|
|
+ nested_fields = self._get_nested_fields(collection)
|
|
|
|
|
+ if nested_fields:
|
|
|
|
|
+ params += (('fq', urllib.unquote(utf_quoter(' OR '.join(nested_fields)))),)
|
|
|
|
|
+
|
|
|
|
|
+ return params
|
|
|
|
|
+
|
|
|
|
|
+ def _get_nested_fields(self, collection):
|
|
|
|
|
+ return [field['filter'] for field in self._flatten_schema(collection['nested']['schema']) if field['selected']] if collection['nested']['enabled'] else []
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def _flatten_schema(self, level):
|
|
|
|
|
+ fields = []
|
|
|
|
|
+ for field in level:
|
|
|
|
|
+ fields.append(field)
|
|
|
|
|
+ if field['values']:
|
|
|
|
|
+ fields.extend(self._flatten_schema(field['values']))
|
|
|
|
|
+ return fields
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@classmethod
|
|
@classmethod
|
|
|
def _get_json(cls, response):
|
|
def _get_json(cls, response):
|
|
|
if type(response) != dict:
|
|
if type(response) != dict:
|