api.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import json
  18. import logging
  19. from django.utils.encoding import force_unicode
  20. from django.utils.translation import ugettext as _
  21. from desktop.lib.django_util import JsonResponse
  22. from desktop.lib.exceptions_renderable import PopupException
  23. from desktop.lib.rest.http_client import RestException
  24. from libsolr.api import SolrApi
  25. from search.conf import SOLR_URL
  26. from dashboard.conf import get_engines, USE_GRIDSTER
  27. from dashboard.controller import can_edit_index
  28. from dashboard.dashboard_api import get_engine
  29. from dashboard.data_export import download as export_download
  30. from dashboard.decorators import allow_viewer_only
  31. from dashboard.facet_builder import _guess_gap, _zoom_range_facet, _new_range_facet
  32. from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception,\
  33. NESTED_FACET_FORM, COMPARE_FACET, QUERY_FACET, extract_solr_exception_message
  34. LOG = logging.getLogger(__name__)
  35. @allow_viewer_only
  36. def search(request):
  37. response = {}
  38. collection = json.loads(request.POST.get('collection', '{}'))
  39. query = json.loads(request.POST.get('query', '{}'))
  40. facet = json.loads(request.POST.get('facet', '{}'))
  41. query['download'] = 'download' in request.POST
  42. fetch_result = 'fetch_result' in request.POST
  43. if collection:
  44. try:
  45. if fetch_result:
  46. response = get_engine(request.user, collection, facet).fetch_result(collection, query, facet)
  47. else:
  48. response = get_engine(request.user, collection, facet).query(collection, query, facet)
  49. except RestException, e:
  50. response.update(extract_solr_exception_message(e))
  51. except Exception, e:
  52. raise PopupException(e, title=_('Error while accessing Solr'))
  53. response['error'] = force_unicode(e)
  54. else:
  55. response['error'] = _('There is no collection to search.')
  56. if 'error' in response:
  57. augment_solr_exception(response, collection)
  58. return JsonResponse(response)
  59. def query_suggest(request):
  60. if request.method != 'POST':
  61. raise PopupException(_('POST request required.'))
  62. collection = json.loads(request.POST.get('collection', '{}'))
  63. query = request.POST.get('query', '')
  64. result = {'status': -1, 'message': ''}
  65. solr_query = {}
  66. solr_query['q'] = query
  67. solr_query['dictionary'] = collection['suggest']['dictionary']
  68. try:
  69. response = SolrApi(SOLR_URL.get(), request.user).suggest(collection['name'], solr_query)
  70. result['response'] = response
  71. result['status'] = 0
  72. except Exception, e:
  73. result['message'] = force_unicode(e)
  74. return JsonResponse(result)
  75. def index_fields_dynamic(request):
  76. result = {'status': -1, 'message': 'Error'}
  77. try:
  78. name = request.POST.get('name')
  79. engine = request.POST.get('engine')
  80. source = request.POST.get('source')
  81. dynamic_fields = get_engine(request.user, engine, source=source).luke(name)
  82. result['message'] = ''
  83. result['fields'] = [
  84. Collection2._make_field(name, properties)
  85. for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties
  86. ]
  87. result['gridlayout_header_fields'] = [
  88. Collection2._make_gridlayout_header_field({'name': name, 'type': properties.get('type')}, True)
  89. for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties
  90. ]
  91. result['status'] = 0
  92. except Exception, e:
  93. result['message'] = force_unicode(e)
  94. return JsonResponse(result)
  95. def nested_documents(request):
  96. result = {'status': -1, 'message': 'Error'}
  97. response = {}
  98. collection = json.loads(request.POST.get('collection', '{}'))
  99. query = {'qs': [{'q': '_root_:*'}], 'fqs': [], 'start': 0, 'limit': 0}
  100. try:
  101. response = get_engine(request.user, collection).query(collection, query)
  102. result['has_nested_documents'] = response['response']['numFound'] > 0
  103. result['status'] = 0
  104. except Exception, e:
  105. LOG.exception('Failed to list nested documents')
  106. result['message'] = force_unicode(e)
  107. result['has_nested_documents'] = False
  108. return JsonResponse(result)
  109. @allow_viewer_only
  110. def get_document(request):
  111. result = {'status': -1, 'message': 'Error'}
  112. try:
  113. collection = json.loads(request.POST.get('collection', '{}'))
  114. doc_id = request.POST.get('id')
  115. if doc_id:
  116. result['doc'] = get_engine(request.user, collection).get(collection, doc_id)
  117. if result['doc']['doc']:
  118. result['status'] = 0
  119. result['message'] = ''
  120. else:
  121. result['status'] = 1
  122. result['message'] = _('No document was returned by Solr.')
  123. else:
  124. result['message'] = _('This document does not have any index id.')
  125. result['status'] = 1
  126. except Exception, e:
  127. result['message'] = force_unicode(e)
  128. return JsonResponse(result)
  129. @allow_viewer_only
  130. def update_document(request):
  131. result = {'status': -1, 'message': 'Error'}
  132. if not can_edit_index(request.user):
  133. result['message'] = _('Permission to edit the document denied')
  134. return JsonResponse(result)
  135. try:
  136. collection = json.loads(request.POST.get('collection', '{}'))
  137. document = json.loads(request.POST.get('document', '{}'))
  138. doc_id = request.POST.get('id')
  139. if document['hasChanged']:
  140. edits = {
  141. "id": doc_id,
  142. }
  143. version = None # If there is a version, use it to avoid potential concurrent update conflicts
  144. for field in document['details']:
  145. if field['hasChanged'] and field['key'] != '_version_':
  146. edits[field['key']] = {"set": field['value']}
  147. if field['key'] == '_version_':
  148. version = field['value']
  149. result['update'] = SolrApi(SOLR_URL.get(), request.user).update(collection['name'], json.dumps([edits]), content_type='json', version=version)
  150. result['message'] = _('Document successfully updated.')
  151. result['status'] = 0
  152. else:
  153. result['status'] = 0
  154. result['message'] = _('Document has no modifications to change.')
  155. except RestException, e:
  156. try:
  157. result['message'] = json.loads(e.message)['error']['msg']
  158. except:
  159. LOG.exception('Failed to parse json response')
  160. result['message'] = force_unicode(e)
  161. except Exception, e:
  162. result['message'] = force_unicode(e)
  163. return JsonResponse(result)
  164. @allow_viewer_only
  165. def get_stats(request):
  166. result = {'status': -1, 'message': 'Error'}
  167. try:
  168. collection = json.loads(request.POST.get('collection', '{}'))
  169. query = json.loads(request.POST.get('query', '{}'))
  170. analysis = json.loads(request.POST.get('analysis', '{}'))
  171. field = analysis['name']
  172. facet = analysis['stats']['facet']
  173. result['stats'] = get_engine(request.user, collection).stats(collection['name'], [field], query, facet)
  174. result['status'] = 0
  175. result['message'] = ''
  176. except Exception, e:
  177. LOG.exception('Failed to get stats for field')
  178. result['message'] = force_unicode(e)
  179. if 'not currently supported' in result['message']:
  180. result['status'] = 1
  181. result['message'] = _('This field type does not support stats')
  182. return JsonResponse(result)
  183. @allow_viewer_only
  184. def get_terms(request):
  185. result = {'status': -1, 'message': 'Error'}
  186. try:
  187. collection = json.loads(request.POST.get('collection', '{}'))
  188. analysis = json.loads(request.POST.get('analysis', '{}'))
  189. limit = json.loads(request.POST.get('limit', '25'))
  190. support_distributed = [engine for engine in get_engines(request.user) if engine['type'] == 'solr'][0]['analytics']
  191. field = analysis['name']
  192. properties = {
  193. 'terms.limit': limit,
  194. 'terms.distrib': str(support_distributed).lower(),
  195. # lower
  196. # mincount
  197. # maxcount
  198. }
  199. if analysis['terms']['prefix']:
  200. properties['terms.regex'] = '.*%(prefix)s.*' % analysis['terms'] # Use regexp instead of case sensitive 'terms.prefix'
  201. properties['terms.regex.flag'] = 'case_insensitive'
  202. result['terms'] = SolrApi(SOLR_URL.get(), request.user).terms(collection['name'], field, properties)
  203. result['terms'] = pairwise2(field, [], result['terms']['terms'][field])
  204. result['status'] = 0
  205. result['message'] = ''
  206. except Exception, e:
  207. result['message'] = force_unicode(e)
  208. if 'not currently supported' in result['message']:
  209. result['status'] = 1
  210. result['message'] = _('This field does not support stats')
  211. return JsonResponse(result)
  212. @allow_viewer_only
  213. def download(request):
  214. try:
  215. file_format = 'csv' if 'csv' == request.POST.get('type') else 'xls' if 'xls' == request.POST.get('type') else 'json'
  216. facet = json.loads(request.POST.get('facet', '{}'))
  217. json_response = search(request)
  218. response = json.loads(json_response.content)
  219. if facet:
  220. response['response']['docs'] = response['normalized_facets'][0]['docs']
  221. collection = facet
  222. if not collection['template']['fieldsSelected']:
  223. facet['fields'] = facet['template']['fieldsAttributes']
  224. else:
  225. collection = json.loads(request.POST.get('collection', '{}'))
  226. if file_format == 'json':
  227. docs = response['response']['docs']
  228. resp = JsonResponse(docs, safe=False)
  229. resp['Content-Disposition'] = 'attachment; filename="%s.%s"' % ('query_result', file_format)
  230. return resp
  231. else:
  232. return export_download(response, file_format, collection, user_agent=request.META.get('HTTP_USER_AGENT'))
  233. except Exception, e:
  234. raise PopupException(_("Could not download search results: %s") % e)
  235. @allow_viewer_only
  236. def get_timeline(request):
  237. result = {'status': -1, 'message': 'Error'}
  238. try:
  239. collection = json.loads(request.POST.get('collection', '{}'))
  240. query = json.loads(request.POST.get('query', '{}'))
  241. facet = json.loads(request.POST.get('facet', '{}'))
  242. qdata = json.loads(request.POST.get('qdata', '{}'))
  243. multiQ = request.POST.get('multiQ', 'query')
  244. if multiQ == 'query':
  245. label = qdata['q']
  246. query['qs'] = [qdata]
  247. elif facet['type'] == 'range':
  248. _prop = filter(lambda prop: prop['from'] == qdata, facet['properties'])[0]
  249. label = '%(from)s - %(to)s ' % _prop
  250. facet_id = facet['id']
  251. # Only care about our current field:value filter
  252. for fq in query['fqs']:
  253. if fq['id'] == facet_id:
  254. fq['properties'] = [_prop]
  255. else:
  256. label = qdata
  257. facet_id = facet['id']
  258. # Only care about our current field:value filter
  259. for fq in query['fqs']:
  260. if fq['id'] == facet_id:
  261. fq['filter'] = [{'value': qdata, 'exclude': False}]
  262. # Remove other facets from collection for speed
  263. collection['facets'] = filter(lambda f: f['widgetType'] == 'histogram-widget', collection['facets'])
  264. response = SolrApi(SOLR_URL.get(), request.user).query(collection, query)
  265. response = augment_solr_response(response, collection, query)
  266. label += ' (%s) ' % response['response']['numFound']
  267. result['series'] = {'label': label, 'counts': response['normalized_facets'][0]['counts']}
  268. result['status'] = 0
  269. result['message'] = ''
  270. except Exception, e:
  271. result['message'] = force_unicode(e)
  272. return JsonResponse(result)
  273. @allow_viewer_only
  274. def new_facet(request):
  275. result = {'status': -1, 'message': 'Error'}
  276. try:
  277. collection = json.loads(request.POST.get('collection', '{}'))
  278. facet_id = request.POST.get('id')
  279. facet_label = request.POST.get('label')
  280. facet_field = request.POST.get('field')
  281. widget_type = request.POST.get('widget_type')
  282. window_size = request.POST.get('window_size')
  283. result['message'] = ''
  284. result['facet'] = _create_facet(collection, request.user, facet_id, facet_label, facet_field, widget_type, window_size)
  285. result['status'] = 0
  286. except Exception, e:
  287. result['message'] = force_unicode(e)
  288. return JsonResponse(result)
  289. def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type, window_size):
  290. properties = {
  291. 'sort': 'desc',
  292. 'canRange': False,
  293. 'stacked': False,
  294. 'limit': 10,
  295. 'mincount': 0,
  296. 'missing': False,
  297. 'isDate': False,
  298. 'slot': 0,
  299. 'aggregate': {'function': 'unique', 'formula': '', 'plain_formula': '', 'percentile': 50}
  300. }
  301. if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'):
  302. facet_type = 'pivot'
  303. elif widget_type == 'document-widget':
  304. # SQL query, 1 solr widget
  305. properties['uuid'] = facet_field
  306. properties['engine'] = 'impala'
  307. properties['statement'] = 'select * from web_logs limit 50'
  308. properties['facets'] = [{'canRange': False, 'field': 'blank', 'limit': 10, 'mincount': 0, 'sort': 'desc', 'aggregate': {'function': 'count'}, 'isDate': False}]
  309. facet_type = 'statement'
  310. else:
  311. api = get_engine(user, collection)
  312. range_properties = _new_range_facet(api, collection, facet_field, widget_type, window_size)
  313. if range_properties:
  314. facet_type = 'range'
  315. properties.update(range_properties)
  316. properties['initial_gap'] = properties['gap']
  317. properties['initial_start'] = properties['start']
  318. properties['initial_end'] = properties['end']
  319. else:
  320. facet_type = 'field'
  321. if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget', 'tree2-widget', 'text-facet-widget', 'hit-widget', 'gradient-map-widget'):
  322. # properties = {'canRange': False, 'stacked': False, 'limit': 10} # TODO: Lighter weight top nested facet
  323. properties['facets_form'] = NESTED_FACET_FORM
  324. # Not supported on dim 2 currently
  325. properties['facets_form']['type'] = 'field'
  326. properties['facets_form']['canRange'] = False
  327. properties['facets_form']['isFacetForm'] = True
  328. facet = NESTED_FACET_FORM.copy()
  329. facet['field'] = facet_field
  330. facet['limit'] = 10
  331. facet['fieldLabel'] = facet_field
  332. facet['multiselect'] = True
  333. if widget_type == 'text-facet-widget':
  334. properties['type'] = facet_type
  335. if USE_GRIDSTER.get():
  336. properties['limit'] = facet['limit'] = 100
  337. if range_properties:
  338. # TODO: timeline still uses properties from top properties
  339. facet.update(range_properties)
  340. facet['initial_gap'] = facet['gap']
  341. facet['initial_start'] = facet['start']
  342. facet['initial_end'] = facet['end']
  343. facet['stacked'] = False
  344. facet['type'] = 'range'
  345. else:
  346. facet['type'] = facet_type
  347. if collection.get('engine', 'solr') != 'solr':
  348. facet['sort'] = 'default'
  349. properties['facets'] = [facet]
  350. properties['domain'] = {'blockParent': [], 'blockChildren': []}
  351. properties['compare'] = COMPARE_FACET
  352. properties['filter'] = QUERY_FACET
  353. if widget_type == 'hit-widget':
  354. facet_type = 'function'
  355. facet['aggregate']['function'] = 'unique'
  356. else:
  357. facet_type = 'nested'
  358. facet['aggregate']['function'] = 'count'
  359. if widget_type == 'pie2-widget':
  360. properties['scope'] = 'stack'
  361. properties['timelineChartType'] = 'bar'
  362. elif widget_type == 'tree2-widget':
  363. properties['scope'] = 'tree'
  364. properties['facets_form']['limit'] = 5
  365. properties['isOldPivot'] = True
  366. elif widget_type == 'gradient-map-widget':
  367. properties['scope'] = 'world'
  368. facet['limit'] = 100
  369. else:
  370. properties['scope'] = 'stack'
  371. properties['timelineChartType'] = 'bar'
  372. if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget') and widget_type != 'gradient-map-widget':
  373. properties['mincount'] = 1
  374. properties['facets'] = []
  375. properties['stacked'] = True
  376. properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
  377. if widget_type == 'map-widget':
  378. properties['scope'] = 'world'
  379. properties['limit'] = 100
  380. else:
  381. properties['scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree'
  382. if widget_type == 'histogram-widget':
  383. properties['enableSelection'] = True
  384. properties['timelineChartType'] = 'bar'
  385. properties['extraSeries'] = []
  386. return {
  387. 'id': facet_id,
  388. 'label': facet_label,
  389. 'field': facet_field,
  390. 'type': facet_type,
  391. 'widgetType': widget_type,
  392. 'properties': properties,
  393. # Hue 4+
  394. 'template': {
  395. "showFieldList": True,
  396. "showGrid": False,
  397. "showChart": True,
  398. "chartSettings" : {
  399. 'chartType': 'pie' if widget_type == 'pie2-widget' else ('timeline' if widget_type == 'timeline-widget' else ('gradientmap' if widget_type == 'gradient-map-widget' else 'bars')),
  400. 'chartSorting': 'none',
  401. 'chartScatterGroup': None,
  402. 'chartScatterSize': None,
  403. 'chartScope': 'world',
  404. 'chartX': None,
  405. 'chartYSingle': None,
  406. 'chartYMulti': [],
  407. 'chartData': [],
  408. 'chartMapLabel': None,
  409. },
  410. "fieldsAttributes": [],
  411. "fieldsAttributesFilter": "",
  412. "filteredAttributeFieldsAll": True,
  413. "fields": [],
  414. "fieldsSelected": [],
  415. "leafletmap": {'latitudeField': None, 'longitudeField': None, 'labelField': None}, # Use own?
  416. 'leafletmapOn': False,
  417. 'isGridLayout': False,
  418. "hasDataForChart": True,
  419. "rows": 25,
  420. },
  421. 'queryResult': {}
  422. }
  423. @allow_viewer_only
  424. def get_range_facet(request):
  425. result = {'status': -1, 'message': ''}
  426. try:
  427. collection = json.loads(request.POST.get('collection', '{}'))
  428. facet = json.loads(request.POST.get('facet', '{}'))
  429. action = request.POST.get('action', 'select')
  430. solr_api = SolrApi(SOLR_URL.get(), request.user)
  431. if action == 'select':
  432. properties = _guess_gap(solr_api, collection, facet, facet['properties']['start'], facet['properties']['end'])
  433. else:
  434. properties = _zoom_range_facet(solr_api, collection, facet) # Zoom out
  435. result['properties'] = properties
  436. result['status'] = 0
  437. except Exception, e:
  438. result['message'] = force_unicode(e)
  439. return JsonResponse(result)
  440. def get_collection(request):
  441. result = {'status': -1, 'message': ''}
  442. try:
  443. name = request.POST.get('name')
  444. engine = request.POST.get('engine')
  445. source = request.POST.get('source')
  446. collection = Collection2(request.user, name=name, engine=engine, source=source)
  447. collection_json = collection.get_json(request.user)
  448. result['collection'] = json.loads(collection_json)
  449. result['status'] = 0
  450. except Exception, e:
  451. result['message'] = force_unicode(e)
  452. return JsonResponse(result)
  453. def get_collections(request):
  454. result = {'status': -1, 'message': ''}
  455. try:
  456. collection = json.loads(request.POST.get('collection'))
  457. show_all = json.loads(request.POST.get('show_all'))
  458. result['collection'] = get_engine(request.user, collection).datasets(show_all=show_all)
  459. result['status'] = 0
  460. except Exception, e:
  461. if 'does not have privileges' in str(e):
  462. result['status'] = 0
  463. result['collection'] = [json.loads(request.POST.get('collection'))['name']]
  464. else:
  465. result['message'] = force_unicode(e)
  466. return JsonResponse(result)