views.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import json
  18. import logging
  19. from django.core.urlresolvers import reverse
  20. from django.utils.encoding import smart_str, force_unicode
  21. from django.utils.html import escape
  22. from django.utils.translation import ugettext as _
  23. from django.shortcuts import redirect
  24. from desktop.lib.django_util import JsonResponse, render
  25. from desktop.lib.exceptions_renderable import PopupException
  26. from desktop.lib.rest.http_client import RestException
  27. from libsolr.api import SolrApi
  28. from indexer.management.commands import indexer_setup
  29. from search.api import _guess_gap, _zoom_range_facet, _new_range_facet
  30. from search.conf import SOLR_URL
  31. from search.data_export import download as export_download
  32. from search.decorators import allow_admin_only
  33. from search.management.commands import search_setup
  34. from search.models import Collection, augment_solr_response, augment_solr_exception,\
  35. pairwise2
  36. from search.search_controller import SearchController
  37. LOG = logging.getLogger(__name__)
  38. def index(request):
  39. hue_collections = SearchController(request.user).get_search_collections()
  40. collection_id = request.GET.get('collection')
  41. if not hue_collections or not collection_id:
  42. if request.user.is_superuser:
  43. return admin_collections(request, True)
  44. else:
  45. return no_collections(request)
  46. try:
  47. collection = Collection.objects.get(id=collection_id) # TODO perms HUE-1987
  48. except Exception, e:
  49. raise PopupException(e, title=_('Error while accessing the collection'))
  50. query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
  51. return render('search.mako', request, {
  52. 'collection': collection,
  53. 'query': query,
  54. 'initial': json.dumps({'collections': [], 'layout': []}),
  55. })
  56. @allow_admin_only
  57. def new_search(request):
  58. collections = SearchController(request.user).get_all_indexes()
  59. if not collections:
  60. return no_collections(request)
  61. collection = Collection(name=collections[0], label=collections[0])
  62. query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
  63. return render('search.mako', request, {
  64. 'collection': collection,
  65. 'query': query,
  66. 'initial': json.dumps({
  67. 'collections': collections,
  68. 'layout': [
  69. {"size":2,"rows":[{"widgets":[]}],"drops":["temp"],"klass":"card card-home card-column span2"},
  70. {"size":10,"rows":[{"widgets":[
  71. {"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget",
  72. "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
  73. "drops":["temp"],"klass":"card card-home card-column span10"}
  74. ]
  75. }),
  76. })
  77. def browse(request, name):
  78. collections = SearchController(request.user).get_all_indexes()
  79. if not collections:
  80. return no_collections(request)
  81. collection = Collection(name=name, label=name)
  82. query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
  83. return render('search.mako', request, {
  84. 'collection': collection,
  85. 'query': query,
  86. 'initial': json.dumps({
  87. 'autoLoad': True,
  88. 'collections': collections,
  89. 'layout': [
  90. {"size":12,"rows":[{"widgets":[
  91. {"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget",
  92. "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
  93. "drops":["temp"],"klass":"card card-home card-column span10"}
  94. ]
  95. }),
  96. })
  97. def search(request):
  98. response = {}
  99. collection = json.loads(request.POST.get('collection', '{}'))
  100. query = json.loads(request.POST.get('query', '{}'))
  101. query['download'] = 'download' in request.POST
  102. # todo: remove the selected histo facet if multiq
  103. if collection['id']:
  104. hue_collection = Collection.objects.get(id=collection['id']) # TODO perms
  105. if collection:
  106. try:
  107. response = SolrApi(SOLR_URL.get(), request.user).query(collection, query)
  108. response = augment_solr_response(response, collection, query)
  109. except RestException, e:
  110. try:
  111. response['error'] = json.loads(e.message)['error']['msg']
  112. except:
  113. response['error'] = force_unicode(str(e))
  114. except Exception, e:
  115. raise PopupException(e, title=_('Error while accessing Solr'))
  116. response['error'] = force_unicode(str(e))
  117. else:
  118. response['error'] = _('There is no collection to search.')
  119. if 'error' in response:
  120. augment_solr_exception(response, collection)
  121. return JsonResponse(response)
  122. @allow_admin_only
  123. def save(request):
  124. response = {'status': -1}
  125. collection = json.loads(request.POST.get('collection', '{}')) # TODO perms
  126. layout = json.loads(request.POST.get('layout', '{}'))
  127. collection['template']['extracode'] = escape(collection['template']['extracode'])
  128. if collection:
  129. if collection['id']:
  130. hue_collection = Collection.objects.get(id=collection['id'])
  131. else:
  132. hue_collection = Collection.objects.create2(name=collection['name'], label=collection['label'])
  133. hue_collection.update_properties({'collection': collection})
  134. hue_collection.update_properties({'layout': layout})
  135. hue_collection.name = collection['name']
  136. hue_collection.label = collection['label']
  137. hue_collection.enabled = collection['enabled']
  138. hue_collection.save()
  139. response['status'] = 0
  140. response['id'] = hue_collection.id
  141. response['message'] = _('Page saved !')
  142. else:
  143. response['message'] = _('There is no collection to search.')
  144. return JsonResponse(response)
  145. def download(request):
  146. try:
  147. file_format = 'csv' if 'csv' in request.POST else 'xls' if 'xls' in request.POST else 'json'
  148. response = search(request)
  149. if file_format == 'json':
  150. docs = json.loads(response.content)['response']['docs']
  151. resp = JsonResponse(docs, safe=False)
  152. resp['Content-Disposition'] = 'attachment; filename=%s.%s' % ('query_result', file_format)
  153. return resp
  154. else:
  155. collection = json.loads(request.POST.get('collection', '{}'))
  156. return export_download(json.loads(response.content), file_format, collection)
  157. except Exception, e:
  158. raise PopupException(_("Could not download search results: %s") % e)
  159. def no_collections(request):
  160. return render('no_collections.mako', request, {})
  161. @allow_admin_only
  162. def admin_collections(request, is_redirect=False):
  163. existing_hue_collections = Collection.objects.all()
  164. if request.GET.get('format') == 'json':
  165. collections = []
  166. for collection in existing_hue_collections:
  167. massaged_collection = {
  168. 'id': collection.id,
  169. 'name': collection.name,
  170. 'label': collection.label,
  171. 'enabled': collection.enabled,
  172. 'isCoreOnly': collection.is_core_only,
  173. 'absoluteUrl': collection.get_absolute_url()
  174. }
  175. collections.append(massaged_collection)
  176. return JsonResponse(collections)
  177. return render('admin_collections.mako', request, {
  178. 'existing_hue_collections': existing_hue_collections,
  179. 'is_redirect': is_redirect
  180. })
  181. @allow_admin_only
  182. def admin_collection_delete(request):
  183. if request.method != 'POST':
  184. raise PopupException(_('POST request required.'))
  185. collections = json.loads(request.POST.get('collections'))
  186. searcher = SearchController(request.user)
  187. response = {
  188. 'result': searcher.delete_collections([collection['id'] for collection in collections])
  189. }
  190. return JsonResponse(response)
  191. @allow_admin_only
  192. def admin_collection_copy(request):
  193. if request.method != 'POST':
  194. raise PopupException(_('POST request required.'))
  195. collections = json.loads(request.POST.get('collections'))
  196. searcher = SearchController(request.user)
  197. response = {
  198. 'result': searcher.copy_collections([collection['id'] for collection in collections])
  199. }
  200. return JsonResponse(response)
  201. def query_suggest(request, collection_id, query=""):
  202. hue_collection = Collection.objects.get(id=collection_id)
  203. result = {'status': -1, 'message': 'Error'}
  204. solr_query = {}
  205. solr_query['collection'] = hue_collection.name
  206. solr_query['q'] = query
  207. try:
  208. response = SolrApi(SOLR_URL.get(), request.user).suggest(solr_query, hue_collection)
  209. result['message'] = response
  210. result['status'] = 0
  211. except Exception, e:
  212. result['message'] = unicode(str(e), "utf8")
  213. return JsonResponse(result)
  214. def index_fields_dynamic(request):
  215. result = {'status': -1, 'message': 'Error'}
  216. try:
  217. name = request.POST['name']
  218. hue_collection = Collection(name=name, label=name)
  219. dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(hue_collection.name)
  220. result['message'] = ''
  221. result['fields'] = [Collection._make_field(name, properties)
  222. for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties]
  223. result['gridlayout_header_fields'] = [Collection._make_gridlayout_header_field({'name': name}, True)
  224. for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties]
  225. result['status'] = 0
  226. except Exception, e:
  227. result['message'] = unicode(str(e), "utf8")
  228. return JsonResponse(result)
  229. def get_document(request):
  230. result = {'status': -1, 'message': 'Error'}
  231. try:
  232. collection = json.loads(request.POST.get('collection', '{}'))
  233. doc_id = request.POST.get('id')
  234. if doc_id:
  235. result['doc'] = SolrApi(SOLR_URL.get(), request.user).get(collection['name'], doc_id)
  236. if result['doc']['doc']:
  237. result['status'] = 0
  238. result['message'] = ''
  239. else:
  240. result['status'] = 1
  241. result['message'] = _('No document was returned by Solr.')
  242. else:
  243. result['message'] = _('This document does not have any index id.')
  244. result['status'] = 1
  245. except Exception, e:
  246. result['message'] = unicode(str(e), "utf8")
  247. return JsonResponse(result)
  248. def get_stats(request):
  249. result = {'status': -1, 'message': 'Error'}
  250. try:
  251. collection = json.loads(request.POST.get('collection', '{}'))
  252. query = json.loads(request.POST.get('query', '{}'))
  253. analysis = json.loads(request.POST.get('analysis', '{}'))
  254. field = analysis['name']
  255. facet = analysis['stats']['facet']
  256. result['stats'] = SolrApi(SOLR_URL.get(), request.user).stats(collection['name'], [field], query, facet)
  257. result['status'] = 0
  258. result['message'] = ''
  259. except Exception, e:
  260. result['message'] = unicode(str(e), "utf8")
  261. if 'not currently supported' in result['message']:
  262. result['status'] = 1
  263. result['message'] = _('This field does not support stats')
  264. return JsonResponse(result)
  265. def get_terms(request):
  266. result = {'status': -1, 'message': 'Error'}
  267. try:
  268. collection = json.loads(request.POST.get('collection', '{}'))
  269. analysis = json.loads(request.POST.get('analysis', '{}'))
  270. field = analysis['name']
  271. properties = {
  272. 'terms.limit': 25,
  273. 'terms.prefix': analysis['terms']['prefix']
  274. # lower
  275. # limit
  276. # mincount
  277. # maxcount
  278. }
  279. result['terms'] = SolrApi(SOLR_URL.get(), request.user).terms(collection['name'], field, properties)
  280. result['terms'] = pairwise2(field, [], result['terms']['terms'][field])
  281. result['status'] = 0
  282. result['message'] = ''
  283. except Exception, e:
  284. result['message'] = unicode(str(e), "utf8")
  285. if 'not currently supported' in result['message']:
  286. result['status'] = 1
  287. result['message'] = _('This field does not support stats')
  288. return JsonResponse(result)
  289. def get_timeline(request):
  290. result = {'status': -1, 'message': 'Error'}
  291. try:
  292. collection = json.loads(request.POST.get('collection', '{}'))
  293. query = json.loads(request.POST.get('query', '{}'))
  294. facet = json.loads(request.POST.get('facet', '{}'))
  295. qdata = json.loads(request.POST.get('qdata', '{}'))
  296. multiQ = request.POST.get('multiQ', 'query')
  297. if multiQ == 'query':
  298. label = qdata['q']
  299. query['qs'] = [qdata]
  300. elif facet['type'] == 'range':
  301. _prop = filter(lambda prop: prop['from'] == qdata, facet['properties'])[0]
  302. label = '%(from)s - %(to)s ' % _prop
  303. facet_id = facet['id']
  304. # Only care about our current field:value filter
  305. for fq in query['fqs']:
  306. if fq['id'] == facet_id:
  307. fq['properties'] = [_prop]
  308. else:
  309. label = qdata
  310. facet_id = facet['id']
  311. # Only care about our current field:value filter
  312. for fq in query['fqs']:
  313. if fq['id'] == facet_id:
  314. fq['filter'] = [{'value': qdata, 'exclude': False}]
  315. # Remove other facets from collection for speed
  316. collection['facets'] = filter(lambda f: f['widgetType'] == 'histogram-widget', collection['facets'])
  317. response = SolrApi(SOLR_URL.get(), request.user).query(collection, query)
  318. response = augment_solr_response(response, collection, query)
  319. label += ' (%s) ' % response['response']['numFound']
  320. result['series'] = {'label': label, 'counts': response['normalized_facets'][0]['counts']}
  321. result['status'] = 0
  322. result['message'] = ''
  323. except Exception, e:
  324. result['message'] = unicode(str(e), "utf8")
  325. return JsonResponse(result)
  326. def new_facet(request):
  327. result = {'status': -1, 'message': 'Error'}
  328. try:
  329. collection = json.loads(request.POST.get('collection', '{}')) # Perms
  330. facet_id = request.POST['id']
  331. facet_label = request.POST['label']
  332. facet_field = request.POST['field']
  333. widget_type = request.POST['widget_type']
  334. result['message'] = ''
  335. result['facet'] = _create_facet(collection, request.user, facet_id, facet_label, facet_field, widget_type)
  336. result['status'] = 0
  337. except Exception, e:
  338. result['message'] = unicode(str(e), "utf8")
  339. return JsonResponse(result)
  340. def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type):
  341. properties = {
  342. 'sort': 'desc',
  343. 'canRange': False,
  344. 'stacked': False,
  345. 'limit': 10,
  346. 'mincount': 0,
  347. 'isDate': False,
  348. 'andUp': False, # Not used yet
  349. }
  350. if widget_type in ('tree-widget', 'heatmap-widget'):
  351. facet_type = 'pivot'
  352. else:
  353. solr_api = SolrApi(SOLR_URL.get(), user)
  354. range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type)
  355. if range_properties:
  356. facet_type = 'range'
  357. properties.update(range_properties)
  358. properties['initial_gap'] = properties['gap']
  359. properties['initial_start'] = properties['start']
  360. properties['initial_end'] = properties['end']
  361. elif widget_type == 'hit-widget':
  362. facet_type = 'query'
  363. else:
  364. facet_type = 'field'
  365. if widget_type == 'map-widget':
  366. properties['scope'] = 'world'
  367. properties['mincount'] = 1
  368. properties['limit'] = 100
  369. elif widget_type in ('tree-widget', 'heatmap-widget'):
  370. properties['mincount'] = 1
  371. properties['facets'] = []
  372. properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
  373. properties['scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree'
  374. return {
  375. 'id': facet_id,
  376. 'label': facet_label,
  377. 'field': facet_field,
  378. 'type': facet_type,
  379. 'widgetType': widget_type,
  380. 'properties': properties
  381. }
  382. def get_range_facet(request):
  383. result = {'status': -1, 'message': ''}
  384. try:
  385. collection = json.loads(request.POST.get('collection', '{}')) # Perms
  386. facet = json.loads(request.POST.get('facet', '{}'))
  387. action = request.POST.get('action', 'select')
  388. solr_api = SolrApi(SOLR_URL.get(), request.user)
  389. if action == 'select':
  390. properties = _guess_gap(solr_api, collection, facet, facet['properties']['start'], facet['properties']['end'])
  391. else:
  392. properties = _zoom_range_facet(solr_api, collection, facet) # Zoom out
  393. result['properties'] = properties
  394. result['status'] = 0
  395. except Exception, e:
  396. result['message'] = unicode(str(e), "utf8")
  397. return JsonResponse(result)
  398. def get_collection(request):
  399. result = {'status': -1, 'message': ''}
  400. try:
  401. name = request.POST['name']
  402. collection = Collection(name=name, label=name)
  403. collection_json = collection.get_c(request.user)
  404. result['collection'] = json.loads(collection_json)
  405. result['status'] = 0
  406. except Exception, e:
  407. result['message'] = unicode(str(e), "utf8")
  408. return JsonResponse(result)
  409. def get_collections(request):
  410. result = {'status': -1, 'message': ''}
  411. try:
  412. show_all = json.loads(request.POST.get('show_all'))
  413. result['collection'] = SearchController(request.user).get_all_indexes(show_all=show_all)
  414. result['status'] = 0
  415. except Exception, e:
  416. if 'does not have privileges' in str(e):
  417. result['status'] = 0
  418. result['collection'] = [json.loads(request.POST.get('collection'))['name']]
  419. else:
  420. result['message'] = unicode(str(e), "utf8")
  421. return JsonResponse(result)
  422. def install_examples(request):
  423. result = {'status': -1, 'message': ''}
  424. if request.method != 'POST':
  425. result['message'] = _('A POST request is required.')
  426. else:
  427. try:
  428. search_setup.Command().handle_noargs()
  429. indexer_setup.Command().handle_noargs()
  430. result['status'] = 0
  431. except Exception, e:
  432. LOG.exception(e)
  433. result['message'] = str(e)
  434. return JsonResponse(result)