models.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import collections
  18. import itertools
  19. import json
  20. import logging
  21. import numbers
  22. import re
  23. from django.core.urlresolvers import reverse
  24. from django.utils.html import escape
  25. from django.utils.translation import ugettext as _
  26. from desktop.lib.i18n import smart_unicode, smart_str
  27. from desktop.models import get_data_link
  28. from dashboard.dashboard_api import get_engine
  29. LOG = logging.getLogger(__name__)
  30. NESTED_FACET_FORM = {'field': '', 'mincount': 1, 'limit': 5, 'sort': 'desc', 'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentiles': [{'value': 50}]}}
  31. class Collection2(object):
  32. def __init__(self, user, name='Default', data=None, document=None, engine='solr'):
  33. self.document = document
  34. if document is not None:
  35. self.data = json.loads(document.data)
  36. elif data is not None:
  37. self.data = json.loads(data)
  38. else:
  39. self.data = {
  40. 'collection': self.get_default(user, name, engine),
  41. 'layout': []
  42. }
  43. def get_json(self, user):
  44. return json.dumps(self.get_props(user))
  45. def get_props(self, user):
  46. props = self.data
  47. if self.document is not None:
  48. props['collection']['id'] = self.document.id
  49. props['collection']['label'] = self.document.name
  50. props['collection']['description'] = self.document.description
  51. # For backward compatibility
  52. if 'rows' not in props['collection']['template']:
  53. props['collection']['template']['rows'] = 25
  54. if 'showGrid' not in props['collection']['template']:
  55. props['collection']['template']['showGrid'] = True
  56. if 'showChart' not in props['collection']['template']:
  57. props['collection']['template']['showChart'] = False
  58. if 'chartSettings' not in props['collection']['template']:
  59. props['collection']['template']['chartSettings'] = {
  60. 'chartType': 'bars',
  61. 'chartSorting': 'none',
  62. 'chartScatterGroup': None,
  63. 'chartScatterSize': None,
  64. 'chartScope': 'world',
  65. 'chartX': None,
  66. 'chartYSingle': None,
  67. 'chartYMulti': [],
  68. 'chartData': [],
  69. 'chartMapLabel': None,
  70. }
  71. if 'enabled' not in props['collection']:
  72. props['collection']['enabled'] = True
  73. if 'engine' not in props['collection']:
  74. props['collection']['engine'] = 'solr'
  75. if 'leafletmap' not in props['collection']['template']:
  76. props['collection']['template']['leafletmap'] = {'latitudeField': None, 'longitudeField': None, 'labelField': None}
  77. if 'timeFilter' not in props['collection']:
  78. props['collection']['timeFilter'] = {
  79. 'field': '',
  80. 'type': 'rolling',
  81. 'value': 'all',
  82. 'from': '',
  83. 'to': '',
  84. 'truncate': True
  85. }
  86. if 'suggest' not in props['collection']:
  87. props['collection']['suggest'] = {'enabled': False, 'dictionary': ''}
  88. for field in props['collection']['template']['fieldsAttributes']:
  89. if 'type' not in field:
  90. field['type'] = 'string'
  91. if 'nested' not in props['collection']:
  92. props['collection']['nested'] = {
  93. 'enabled': False,
  94. 'schema': []
  95. }
  96. for facet in props['collection']['facets']:
  97. properties = facet['properties']
  98. if 'gap' in properties and not 'initial_gap' in properties:
  99. properties['initial_gap'] = properties['gap']
  100. if 'start' in properties and not 'initial_start' in properties:
  101. properties['initial_start'] = properties['start']
  102. if 'end' in properties and not 'initial_end' in properties:
  103. properties['initial_end'] = properties['end']
  104. if 'domain' not in properties:
  105. properties['domain'] = {'blockParent': [], 'blockChildren': []}
  106. if facet['widgetType'] == 'histogram-widget':
  107. if 'timelineChartType' not in properties:
  108. properties['timelineChartType'] = 'bar'
  109. if 'enableSelection' not in properties:
  110. properties['enableSelection'] = True
  111. if 'extraSeries' not in properties:
  112. properties['extraSeries'] = []
  113. if facet['widgetType'] == 'map-widget' and facet['type'] == 'field':
  114. facet['type'] = 'pivot'
  115. properties['facets'] = []
  116. properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
  117. if 'qdefinitions' not in props['collection']:
  118. props['collection']['qdefinitions'] = []
  119. return props
  120. def get_default(self, user, name, engine='solr'):
  121. fields = self.fields_data(user, name, engine)
  122. id_field = [field['name'] for field in fields if field.get('isId')]
  123. if id_field:
  124. id_field = id_field[0]
  125. else:
  126. id_field = '' # Schemaless might not have an id
  127. TEMPLATE = {
  128. "extracode": escape("<style type=\"text/css\">\nem {\n font-weight: bold;\n background-color: yellow;\n}</style>\n\n<script>\n</script>"),
  129. "highlighting": [""],
  130. "properties": {"highlighting_enabled": True},
  131. "template": """
  132. <div class="row-fluid">
  133. <div class="row-fluid">
  134. <div class="span12">%s</div>
  135. </div>
  136. <br/>
  137. </div>""" % ' '.join(['{{%s}}' % field['name'] for field in fields]),
  138. "isGridLayout": True,
  139. "showFieldList": True,
  140. "showGrid": True,
  141. "showChart": False,
  142. "chartSettings" : {
  143. 'chartType': 'bars',
  144. 'chartSorting': 'none',
  145. 'chartScatterGroup': None,
  146. 'chartScatterSize': None,
  147. 'chartScope': 'world',
  148. 'chartX': None,
  149. 'chartYSingle': None,
  150. 'chartYMulti': [],
  151. 'chartData': [],
  152. 'chartMapLabel': None,
  153. },
  154. "fieldsAttributes": [self._make_gridlayout_header_field(field) for field in fields],
  155. "fieldsSelected": [],
  156. "leafletmap": {'latitudeField': None, 'longitudeField': None, 'labelField': None},
  157. "rows": 25,
  158. }
  159. FACETS = []
  160. return {
  161. 'id': None,
  162. 'name': name,
  163. 'engine': engine,
  164. 'label': name,
  165. 'enabled': False,
  166. 'template': TEMPLATE,
  167. 'facets': FACETS,
  168. 'fields': fields,
  169. 'idField': id_field,
  170. }
  171. @classmethod
  172. def _make_field(cls, field, attributes):
  173. return {
  174. 'name': str(escape(field)),
  175. 'type': str(attributes.get('type', '')),
  176. 'isId': attributes.get('required') and attributes.get('uniqueKey'),
  177. 'isDynamic': 'dynamicBase' in attributes
  178. }
  179. @classmethod
  180. def _make_gridlayout_header_field(cls, field, isDynamic=False):
  181. return {'name': field['name'], 'type': field['type'], 'sort': {'direction': None}, 'isDynamic': isDynamic}
  182. @classmethod
  183. def _make_luke_from_schema_fields(cls, schema_fields):
  184. return dict([
  185. (f['name'], {
  186. 'copySources': [],
  187. 'type': f['type'],
  188. 'required': True,
  189. 'uniqueKey': f.get('uniqueKey'),
  190. 'flags': u'%s-%s-----OF-----l' % ('I' if f['indexed'] else '-', 'S' if f['stored'] else '-'), u'copyDests': []
  191. })
  192. for f in schema_fields['fields']
  193. ])
  194. def get_absolute_url(self):
  195. return reverse('search:index') + '?collection=%s' % self.id
  196. def fields(self, user):
  197. return sorted([str(field.get('name', '')) for field in self.fields_data(user)])
  198. def fields_data(self, user, name, engine='solr'):
  199. api = get_engine(user, engine)
  200. try:
  201. schema_fields = api.fields(name)
  202. schema_fields = schema_fields['schema']['fields']
  203. except Exception, e:
  204. LOG.warn('/luke call did not succeed: %s' % e)
  205. try:
  206. fields = api.schema_fields(name)
  207. schema_fields = Collection2._make_luke_from_schema_fields(fields)
  208. except Exception, e:
  209. LOG.error('Could not access collection: %s' % e)
  210. return []
  211. return sorted([self._make_field(field, attributes) for field, attributes in schema_fields.iteritems()])
  212. def update_data(self, post_data):
  213. data_dict = self.data
  214. data_dict.update(post_data)
  215. self.data = data_dict
  216. @property
  217. def autocomplete(self):
  218. return self.data['autocomplete']
  219. @autocomplete.setter
  220. def autocomplete(self, autocomplete):
  221. properties_ = self.data
  222. properties_['autocomplete'] = autocomplete
  223. self.data = json.dumps(properties_)
  224. @classmethod
  225. def get_field_list(cls, collection):
  226. if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
  227. fields = set(collection['template']['fieldsSelected'] + ([collection['idField']] if collection['idField'] else []))
  228. # Add field if needed
  229. if collection['template']['leafletmap'].get('latitudeField'):
  230. fields.add(collection['template']['leafletmap']['latitudeField'])
  231. if collection['template']['leafletmap'].get('longitudeField'):
  232. fields.add(collection['template']['leafletmap']['longitudeField'])
  233. if collection['template']['leafletmap'].get('labelField'):
  234. fields.add(collection['template']['leafletmap']['labelField'])
  235. return list(fields)
  236. else:
  237. return ['*']
  238. def get_facet_field(category, field, facets):
  239. if category in ('nested', 'function'):
  240. id_pattern = '%(id)s'
  241. else:
  242. id_pattern = '%(field)s-%(id)s'
  243. facets = filter(lambda facet: facet['type'] == category and id_pattern % facet == field, facets)
  244. if facets:
  245. return facets[0]
  246. else:
  247. return None
  248. def pairwise2(field, fq_filter, iterable):
  249. pairs = []
  250. selected_values = [f['value'] for f in fq_filter]
  251. a, b = itertools.tee(iterable)
  252. for element in a:
  253. pairs.append({
  254. 'cat': field,
  255. 'value': element,
  256. 'count': next(a),
  257. 'selected': element in selected_values,
  258. 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element])
  259. })
  260. return pairs
  261. def range_pair(field, cat, fq_filter, iterable, end, collection_facet):
  262. # e.g. counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000}
  263. pairs = []
  264. selected_values = [f['value'] for f in fq_filter]
  265. is_single_unit_gap = re.match('^[\+\-]?1[A-Za-z]*$', str(collection_facet['properties']['gap'])) is not None
  266. is_up = collection_facet['properties']['sort'] == 'asc'
  267. if collection_facet['properties']['sort'] == 'asc' and (collection_facet['type'] == 'range-up' or collection_facet['properties'].get('type') == 'range-up'):
  268. prev = None
  269. n = []
  270. for e in iterable:
  271. if prev is not None:
  272. n.append(e)
  273. n.append(prev)
  274. prev = None
  275. else:
  276. prev = e
  277. iterable = n
  278. iterable.reverse()
  279. a, to = itertools.tee(iterable)
  280. next(to, None)
  281. counts = iterable[1::2]
  282. total_counts = counts.pop(0) if collection_facet['properties']['sort'] == 'asc' else 0
  283. for element in a:
  284. next(to, None)
  285. to_value = next(to, end)
  286. count = next(a)
  287. pairs.append({
  288. 'field': field, 'from': element, 'value': count, 'to': to_value, 'selected': element in selected_values,
  289. 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element]),
  290. 'is_single_unit_gap': is_single_unit_gap,
  291. 'total_counts': total_counts,
  292. 'is_up': is_up
  293. })
  294. total_counts += counts.pop(0) if counts else 0
  295. if collection_facet['properties']['sort'] == 'asc' and collection_facet['type'] != 'range-up' and collection_facet['properties'].get('type') != 'range-up':
  296. pairs.reverse()
  297. return pairs
  298. def augment_solr_response(response, collection, query):
  299. augmented = response
  300. augmented['normalized_facets'] = []
  301. NAME = '%(field)s-%(id)s'
  302. normalized_facets = []
  303. selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
  304. if response and response.get('facet_counts'):
  305. for facet in collection['facets']:
  306. category = facet['type']
  307. if category == 'field' and response['facet_counts']['facet_fields']:
  308. name = NAME % facet
  309. collection_facet = get_facet_field(category, name, collection['facets'])
  310. counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name])
  311. if collection_facet['properties']['sort'] == 'asc':
  312. counts.reverse()
  313. facet = {
  314. 'id': collection_facet['id'],
  315. 'field': facet['field'],
  316. 'type': category,
  317. 'label': collection_facet['label'],
  318. 'counts': counts,
  319. }
  320. normalized_facets.append(facet)
  321. elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']:
  322. name = NAME % facet
  323. collection_facet = get_facet_field(category, name, collection['facets'])
  324. counts = response['facet_counts']['facet_ranges'][name]['counts']
  325. end = response['facet_counts']['facet_ranges'][name]['end']
  326. counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
  327. facet = {
  328. 'id': collection_facet['id'],
  329. 'field': facet['field'],
  330. 'type': category,
  331. 'label': collection_facet['label'],
  332. 'counts': counts,
  333. 'extraSeries': []
  334. }
  335. normalized_facets.append(facet)
  336. elif category == 'query' and response['facet_counts']['facet_queries']:
  337. for name, value in response['facet_counts']['facet_queries'].iteritems():
  338. collection_facet = get_facet_field(category, name, collection['facets'])
  339. facet = {
  340. 'id': collection_facet['id'],
  341. 'query': name,
  342. 'type': category,
  343. 'label': name,
  344. 'counts': value,
  345. }
  346. normalized_facets.append(facet)
  347. elif category == 'pivot':
  348. name = NAME % facet
  349. if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']:
  350. if facet['properties']['scope'] == 'stack':
  351. count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values)
  352. else:
  353. count = response['facet_counts']['facet_pivot'][name]
  354. _augment_pivot_nd(facet['id'], count, selected_values)
  355. else:
  356. count = []
  357. facet = {
  358. 'id': facet['id'],
  359. 'field': name,
  360. 'type': category,
  361. 'label': name,
  362. 'counts': count,
  363. }
  364. normalized_facets.append(facet)
  365. if response and response.get('facets'):
  366. for facet in collection['facets']:
  367. category = facet['type']
  368. name = facet['id'] # Nested facets can only have one name
  369. if category == 'function' and name in response['facets']:
  370. value = response['facets'][name]
  371. collection_facet = get_facet_field(category, name, collection['facets'])
  372. facet = {
  373. 'id': collection_facet['id'],
  374. 'query': name,
  375. 'type': category,
  376. 'label': name,
  377. 'counts': value,
  378. }
  379. normalized_facets.append(facet)
  380. elif category == 'nested' and name in response['facets']:
  381. value = response['facets'][name]
  382. collection_facet = get_facet_field(category, name, collection['facets'])
  383. extraSeries = []
  384. counts = response['facets'][name]['buckets']
  385. cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
  386. last_x_col = 0
  387. last_xx_col = 0
  388. for i, f in enumerate(facet['properties']['facets']):
  389. if f['aggregate']['function'] == 'count':
  390. cols.append(f['field'])
  391. last_xx_col = last_x_col
  392. last_x_col = i + 2
  393. from libsolr.api import SolrApi
  394. cols.append(SolrApi._get_aggregate_function(f))
  395. rows = []
  396. # For dim in dimensions
  397. # Number or Date range
  398. if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
  399. dimension = 3 if collection_facet['properties']['isDate'] else 1
  400. # Single dimension or dimension 2 with analytics
  401. if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1:
  402. column = 'count'
  403. if len(collection_facet['properties']['facets']) == 1:
  404. agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
  405. legend = agg_keys[0].split(':', 2)[1]
  406. column = agg_keys[0]
  407. else:
  408. legend = facet['field'] # 'count(%s)' % legend
  409. agg_keys = [column]
  410. _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
  411. counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
  412. counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
  413. else:
  414. # Dimension 1 with counts and 2 with analytics
  415. agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
  416. agg_keys.sort(key=lambda a: a[4:])
  417. if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
  418. agg_keys.insert(0, 'count')
  419. counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
  420. _series = collections.defaultdict(list)
  421. for row in rows:
  422. for i, cell in enumerate(row):
  423. if i > last_x_col:
  424. legend = cols[i]
  425. if last_xx_col != last_x_col:
  426. legend = '%s %s' % (cols[i], row[last_x_col])
  427. _series[legend].append(row[last_xx_col])
  428. _series[legend].append(cell)
  429. for _name, val in _series.iteritems():
  430. _c = range_pair(facet['field'], _name, selected_values.get(facet['id'], []), val, 1, collection_facet)
  431. extraSeries.append({'counts': _c, 'label': _name})
  432. counts = []
  433. elif collection_facet['properties'].get('isOldPivot'):
  434. facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count']
  435. column = 'count'
  436. agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
  437. agg_keys.sort(key=lambda a: a[4:])
  438. if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
  439. agg_keys.insert(0, 'count')
  440. counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
  441. #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
  442. dimension = len(facet_fields)
  443. elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
  444. # Dimension 1 with 1 count or agg
  445. dimension = 1
  446. column = 'count'
  447. agg_keys = counts and [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
  448. if len(collection_facet['properties']['facets']) == 1 and agg_keys:
  449. column = agg_keys[0]
  450. else:
  451. agg_keys = [column]
  452. legend = facet['field']
  453. _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
  454. counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
  455. counts = pairwise2(legend, selected_values.get(facet['id'], []), counts)
  456. else:
  457. # Dimension 2 with analytics or 1 with N aggregates
  458. dimension = 2
  459. agg_keys = counts and [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
  460. agg_keys.sort(key=lambda a: a[4:])
  461. if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
  462. agg_keys.insert(0, 'count')
  463. counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
  464. actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
  465. counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
  466. num_bucket = response['facets'][name]['numBuckets'] if 'numBuckets' in response['facets'][name] else len(response['facets'][name])
  467. facet = {
  468. 'id': collection_facet['id'],
  469. 'field': facet['field'],
  470. 'type': category,
  471. 'label': collection_facet['label'],
  472. 'counts': counts,
  473. 'extraSeries': extraSeries,
  474. 'dimension': dimension,
  475. 'response': {'response': {'start': 0, 'numFound': num_bucket}}, # Todo * nested buckets + offsets
  476. 'docs': [dict(zip(cols, row)) for row in rows],
  477. 'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols]
  478. }
  479. normalized_facets.append(facet)
  480. # Remove unnecessary facet data
  481. if response:
  482. response.pop('facet_counts')
  483. response.pop('facets')
  484. augment_response(collection, query, response)
  485. if normalized_facets:
  486. augmented['normalized_facets'].extend(normalized_facets)
  487. return augmented
  488. def augment_response(collection, query, response):
  489. # HTML escaping
  490. if not query.get('download'):
  491. id_field = collection.get('idField', '')
  492. for doc in response['response']['docs']:
  493. link = None
  494. if 'link-meta' in doc:
  495. meta = json.loads(doc['link-meta'])
  496. link = get_data_link(meta)
  497. elif 'link' in doc:
  498. meta = {'type': 'link', 'link': doc['link']}
  499. link = get_data_link(meta)
  500. for field, value in doc.iteritems():
  501. if isinstance(value, numbers.Number):
  502. escaped_value = value
  503. elif field == '_childDocuments_': # Nested documents
  504. escaped_value = value
  505. elif isinstance(value, list): # Multivalue field
  506. escaped_value = [smart_unicode(escape(val), errors='replace') for val in value]
  507. else:
  508. value = smart_unicode(value, errors='replace')
  509. escaped_value = escape(value)
  510. doc[field] = escaped_value
  511. doc['externalLink'] = link
  512. doc['details'] = []
  513. doc['hueId'] = smart_unicode(doc.get(id_field, ''))
  514. highlighted_fields = response.get('highlighting', {}).keys()
  515. if highlighted_fields and not query.get('download'):
  516. id_field = collection.get('idField')
  517. if id_field:
  518. for doc in response['response']['docs']:
  519. if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields:
  520. highlighting = response['highlighting'][smart_unicode(doc[id_field])]
  521. if highlighting:
  522. escaped_highlighting = {}
  523. for field, hls in highlighting.iteritems():
  524. _hls = [escape(smart_unicode(hl, errors='replace')).replace('&lt;em&gt;', '<em>').replace('&lt;/em&gt;', '</em>') for hl in hls]
  525. escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls
  526. doc.update(escaped_highlighting)
  527. else:
  528. response['warning'] = _("The Solr schema requires an id field for performing the result highlighting")
  529. def _augment_pivot_2d(name, facet_id, counts, selected_values):
  530. values = set()
  531. for dimension in counts:
  532. for pivot in dimension['pivot']:
  533. values.add(pivot['value'])
  534. values = sorted(list(values))
  535. augmented = []
  536. for dimension in counts:
  537. count = {}
  538. pivot_field = ''
  539. for pivot in dimension['pivot']:
  540. count[pivot['value']] = pivot['count']
  541. pivot_field = pivot['field']
  542. for val in values:
  543. fq_values = [dimension['value'], val]
  544. fq_fields = [dimension['field'], pivot_field]
  545. fq_filter = selected_values.get(facet_id, [])
  546. _selected_values = [f['value'] for f in fq_filter]
  547. augmented.append({
  548. "count": count.get(val, 0),
  549. "value": val,
  550. "cat": dimension['value'],
  551. 'selected': fq_values in _selected_values,
  552. 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
  553. 'fq_fields': fq_fields,
  554. 'fq_values': fq_values,
  555. })
  556. return augmented
  557. def _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows):
  558. fq_fields = []
  559. fq_values = []
  560. fq_filter = []
  561. _selected_values = [f['value'] for f in selected_values.get(facet['id'], [])]
  562. _fields = [facet['field']] + [facet['field'] for facet in facet['properties']['facets']]
  563. return __augment_stats_2d(counts, facet['field'], fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows)
  564. # Clear one dimension
  565. def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows):
  566. augmented = []
  567. for bucket in counts: # For each dimension, go through each bucket and pick up the counts or aggregates, then go recursively in the next dimension
  568. val = bucket['val']
  569. count = bucket['count']
  570. dim_row = [val]
  571. _fq_fields = fq_fields + _fields[0:1]
  572. _fq_values = fq_values + [val]
  573. for agg_key in agg_keys:
  574. if agg_key == 'count':
  575. dim_row.append(count)
  576. augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
  577. elif agg_key.startswith('agg_'):
  578. label = fq_values[0] if len(_fq_fields) >= 2 else agg_key.split(':', 2)[1]
  579. if agg_keys.index(agg_key) == 0: # One count by dimension
  580. dim_row.append(count)
  581. if not agg_key in bucket: # No key if value is 0
  582. bucket[agg_key] = 0
  583. dim_row.append(bucket[agg_key])
  584. augmented.append(_get_augmented(bucket[agg_key], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
  585. else:
  586. augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values)) # Needed?
  587. # List nested fields
  588. _agg_keys = []
  589. if agg_key in bucket and bucket[agg_key]['buckets']: # Protect against empty buckets
  590. for key, value in bucket[agg_key]['buckets'][0].items():
  591. if key.lower().startswith('agg_') or key.lower().startswith('dim_'):
  592. _agg_keys.append(key)
  593. _agg_keys.sort(key=lambda a: a[4:])
  594. # Go rec
  595. if not _agg_keys or len(_agg_keys) == 1 and _agg_keys[0].lower().startswith('dim_'):
  596. _agg_keys.insert(0, 'count')
  597. next_dim = []
  598. new_rows = []
  599. if agg_key in bucket:
  600. augmented += __augment_stats_2d(bucket[agg_key]['buckets'], val, _fq_fields, _fq_values, fq_filter, _selected_values, _fields[1:], _agg_keys, next_dim)
  601. for row in next_dim:
  602. new_rows.append(dim_row + row)
  603. dim_row = new_rows
  604. if dim_row and type(dim_row[0]) == list:
  605. rows.extend(dim_row)
  606. else:
  607. rows.append(dim_row)
  608. return augmented
  609. def _get_augmented(count, val, label, fq_values, fq_fields, fq_filter, _selected_values):
  610. return {
  611. "count": count,
  612. "value": val,
  613. "cat": label,
  614. 'selected': fq_values in _selected_values,
  615. 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
  616. 'fq_fields': fq_fields,
  617. 'fq_values': fq_values
  618. }
  619. def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''):
  620. for c in counts:
  621. fq_fields = (fields if fields else []) + [c['field']]
  622. fq_values = (values if values else []) + [smart_str(c['value'])]
  623. if 'pivot' in c:
  624. _augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values)
  625. fq_filter = selected_values.get(facet_id, [])
  626. _selected_values = [f['value'] for f in fq_filter]
  627. c['selected'] = fq_values in _selected_values
  628. c['exclude'] = False
  629. c['fq_fields'] = fq_fields
  630. c['fq_values'] = fq_values
  631. def _convert_nested_to_augmented_pivot_nd(facet_fields, facet_id, counts, selected_values, fields='', values='', dimension=2):
  632. for c in counts['buckets']:
  633. c['field'] = facet_fields[0]
  634. fq_fields = (fields if fields else []) + [c['field']]
  635. fq_values = (values if values else []) + [smart_str(c['val'])]
  636. c['value'] = c.pop('val')
  637. bucket = 'd%s' % dimension
  638. if bucket in c:
  639. next_dimension = facet_fields[1:]
  640. if next_dimension:
  641. _convert_nested_to_augmented_pivot_nd(next_dimension, facet_id, c[bucket], selected_values, fq_fields, fq_values, dimension=dimension+1)
  642. c['pivot'] = c.pop(bucket)['buckets']
  643. else:
  644. c['count'] = c.pop(bucket)
  645. fq_filter = selected_values.get(facet_id, [])
  646. _selected_values = [f['value'] for f in fq_filter]
  647. c['selected'] = fq_values in _selected_values
  648. c['exclude'] = False
  649. c['fq_fields'] = fq_fields
  650. c['fq_values'] = fq_values
  651. def augment_solr_exception(response, collection):
  652. response.update(
  653. {
  654. "facet_counts": {
  655. },
  656. "highlighting": {
  657. },
  658. "normalized_facets": [
  659. {
  660. "field": facet['field'],
  661. "counts": [],
  662. "type": facet['type'],
  663. "label": facet['label']
  664. }
  665. for facet in collection['facets']
  666. ],
  667. "responseHeader": {
  668. "status": -1,
  669. "QTime": 0,
  670. "params": {
  671. }
  672. },
  673. "response": {
  674. "start": 0,
  675. "numFound": 0,
  676. "docs": [
  677. ]
  678. }
  679. })