views.py 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. try:
  18. import json
  19. except ImportError:
  20. import simplejson as json
  21. import logging
  22. import re
  23. from django import forms
  24. from django.contrib import messages
  25. from django.db.models import Q
  26. from django.http import HttpResponse, QueryDict
  27. from django.shortcuts import redirect
  28. from django.utils.translation import ugettext as _
  29. from django.core.urlresolvers import reverse
  30. from desktop.context_processors import get_app_name
  31. from desktop.lib.paginator import Paginator
  32. from desktop.lib.django_util import copy_query_dict, format_preserving_redirect, render
  33. from desktop.lib.django_util import login_notrequired, get_desktop_uri_prefix
  34. from desktop.lib.exceptions_renderable import PopupException
  35. from jobsub.parameterization import find_variables, substitute_variables
  36. import beeswax.forms
  37. import beeswax.design
  38. import beeswax.management.commands.beeswax_install_examples
  39. from beeswax import common, data_export, models, conf
  40. from beeswax.forms import QueryForm
  41. from beeswax.design import HQLdesign
  42. from beeswax.models import SavedQuery, make_query_context, QueryHistory
  43. from beeswax.server import dbms
  44. from beeswax.server.dbms import expand_exception, get_query_server_config
  45. LOG = logging.getLogger(__name__)
  46. def index(request):
  47. return execute_query(request)
  48. """
  49. Design views
  50. """
  51. def save_design(request, form, type, design, explicit_save):
  52. """
  53. save_design(request, form, type, design, explicit_save) -> SavedQuery
  54. A helper method to save the design:
  55. * If ``explicit_save``, then we save the data in the current design.
  56. * If the user clicked the submit button, we do NOT overwrite the current
  57. design. Instead, we create a new "auto" design (iff the user modified
  58. the data). This new design is named after the current design, with the
  59. AUTO_DESIGN_SUFFIX to signify that it's different.
  60. Need to return a SavedQuery because we may end up with a different one.
  61. Assumes that form.saveform is the SaveForm, and that it is valid.
  62. """
  63. assert form.saveform.is_valid()
  64. if type == models.HQL:
  65. design_cls = beeswax.design.HQLdesign
  66. elif type == models.IMPALA:
  67. design_cls = beeswax.design.HQLdesign
  68. else:
  69. raise ValueError(_('Invalid design type %(type)s') % {'type': type})
  70. old_design = design
  71. design_obj = design_cls(form)
  72. new_data = design_obj.dumps()
  73. # Auto save if (1) the user didn't click "save", and (2) the data is different.
  74. # Don't generate an auto-saved design if the user didn't change anything
  75. if explicit_save:
  76. design.name = form.saveform.cleaned_data['name']
  77. design.desc = form.saveform.cleaned_data['desc']
  78. design.is_auto = False
  79. elif new_data != old_design.data:
  80. # Auto save iff the data is different
  81. if old_design.id is not None:
  82. # Clone iff the parent design isn't a new unsaved model
  83. design = old_design.clone()
  84. if not old_design.is_auto:
  85. design.name = old_design.name + models.SavedQuery.AUTO_DESIGN_SUFFIX
  86. else:
  87. design.name = models.SavedQuery.DEFAULT_NEW_DESIGN_NAME
  88. design.is_auto = True
  89. design.type = type
  90. design.data = new_data
  91. design.save()
  92. LOG.info('Saved %s design "%s" (id %s) for %s' %
  93. (explicit_save and '' or 'auto ', design.name, design.id, design.owner))
  94. if explicit_save:
  95. messages.info(request, _('Saved design "%(name)s"') % {'name': design.name})
  96. # Design may now have a new/different id
  97. return design
  98. def save_design_properties(request):
  99. response = {'status': -1, 'data': ''}
  100. try:
  101. if request.method != 'POST':
  102. raise PopupException(_('POST request required.'))
  103. design_id = request.POST.get('pk')
  104. design = authorized_get_design(request, design_id)
  105. field = request.POST.get('name')
  106. if field == 'name':
  107. design.name = request.POST.get('value')
  108. elif field == 'description':
  109. design.desc = request.POST.get('value')
  110. design.save()
  111. response['status'] = 0
  112. except Exception, e:
  113. response['data'] = str(e)
  114. return HttpResponse(json.dumps(response), mimetype="application/json")
  115. def delete_design(request):
  116. if request.method == 'POST':
  117. ids = request.POST.getlist('designs_selection')
  118. designs = dict([(design_id, authorized_get_design(request, design_id)) for design_id in ids])
  119. if None in designs.values():
  120. LOG.error('Cannot delete non-existent design(s) %s' % ','.join([key for key, name in designs.items() if name is None]))
  121. return list_designs(request)
  122. for design in designs.values():
  123. if request.POST.get('skipTrash', 'false') == 'false':
  124. design.is_trashed = True
  125. design.save()
  126. else:
  127. design.delete()
  128. return redirect(reverse(get_app_name(request) + ':list_designs'))
  129. else:
  130. return render('confirm.html', request, dict(url=request.path, title=_('Delete design(s)?')))
  131. def restore_design(request):
  132. if request.method == 'POST':
  133. ids = request.POST.getlist('designs_selection')
  134. designs = dict([(design_id, authorized_get_design(request, design_id)) for design_id in ids])
  135. if None in designs.values():
  136. LOG.error('Cannot restore non-existent design(s) %s' % ','.join([key for key, name in designs.items() if name is None]))
  137. return list_designs(request)
  138. for design in designs.values():
  139. design.is_trashed = False
  140. design.save()
  141. return redirect(reverse(get_app_name(request) + ':list_designs'))
  142. else:
  143. return render('confirm.html', request, dict(url=request.path, title=_('Restore design(s)?')))
  144. def clone_design(request, design_id):
  145. """Clone a design belonging to any user"""
  146. design = authorized_get_design(request, design_id)
  147. if design is None:
  148. LOG.error('Cannot clone non-existent design %s' % (design_id,))
  149. return list_designs(request)
  150. copy = design.clone()
  151. copy.name = design.name + ' (copy)'
  152. copy.owner = request.user
  153. copy.save()
  154. messages.info(request, _('Copied design: %(name)s') % {'name': design.name})
  155. return format_preserving_redirect(
  156. request, reverse(get_app_name(request) + ':execute_query', kwargs={'design_id': copy.id}))
  157. def list_designs(request):
  158. """
  159. View function for show all saved queries.
  160. We get here from /beeswax/list_designs?filterargs, with the options being:
  161. page=<n> - Controls pagination. Defaults to 1.
  162. user=<name> - Show design items belonging to a user. Default to all users.
  163. type=<type> - <type> is "hql", for saved query type. Default to show all.
  164. sort=<key> - Sort by the attribute <key>, which is one of:
  165. "date", "name", "desc", and "type" (design type)
  166. Accepts the form "-date", which sort in descending order.
  167. Default to "-date".
  168. text=<frag> - Search for fragment "frag" in names and descriptions.
  169. Depending on Beeswax configuration parameter ``SHOW_ONLY_PERSONAL_SAVED_QUERIES``,
  170. only the personal queries of the user will be returned (even if another user is
  171. specified in ``filterargs``).
  172. """
  173. DEFAULT_PAGE_SIZE = 20
  174. app_name= get_app_name(request)
  175. if conf.SHARE_SAVED_QUERIES.get() or request.user.is_superuser:
  176. user = None
  177. else:
  178. user = request.user
  179. # Extract the saved query list.
  180. prefix = 'q-'
  181. querydict_query = _copy_prefix(prefix, request.GET)
  182. # Manually limit up the user filter.
  183. querydict_query[ prefix + 'user' ] = user
  184. querydict_query[ prefix + 'type' ] = app_name
  185. page, filter_params = _list_designs(querydict_query, DEFAULT_PAGE_SIZE, prefix)
  186. return render('list_designs.mako', request, {
  187. 'page': page,
  188. 'filter_params': filter_params,
  189. 'user': request.user,
  190. 'designs_json': json.dumps([query.id for query in page.object_list])
  191. })
  192. def list_trashed_designs(request):
  193. DEFAULT_PAGE_SIZE = 20
  194. app_name= get_app_name(request)
  195. user = request.user
  196. # Extract the saved query list.
  197. prefix = 'q-'
  198. querydict_query = _copy_prefix(prefix, request.GET)
  199. # Manually limit up the user filter.
  200. querydict_query[ prefix + 'user' ] = user
  201. querydict_query[ prefix + 'type' ] = app_name
  202. page, filter_params = _list_designs(querydict_query, DEFAULT_PAGE_SIZE, prefix, is_trashed=True)
  203. return render('list_trashed_designs.mako', request, {
  204. 'page': page,
  205. 'filter_params': filter_params,
  206. 'user': request.user,
  207. 'designs_json': json.dumps([query.id for query in page.object_list])
  208. })
  209. def my_queries(request):
  210. """
  211. View a mix of history and saved queries.
  212. It understands all the GET params in ``list_query_history`` (with a ``h-`` prefix)
  213. and those in ``list_designs`` (with a ``q-`` prefix). The only thing it disallows
  214. is the ``user`` filter, since this view only shows what belongs to the user.
  215. """
  216. DEFAULT_PAGE_SIZE = 30
  217. app_name= get_app_name(request)
  218. # Extract the history list.
  219. prefix = 'h-'
  220. querydict_history = _copy_prefix(prefix, request.GET)
  221. # Manually limit up the user filter.
  222. querydict_history[ prefix + 'user' ] = request.user.username
  223. querydict_history[ prefix + 'type' ] = app_name
  224. hist_page, hist_filter = _list_query_history(request.user,
  225. querydict_history,
  226. DEFAULT_PAGE_SIZE,
  227. prefix)
  228. # Extract the saved query list.
  229. prefix = 'q-'
  230. querydict_query = _copy_prefix(prefix, request.GET)
  231. # Manually limit up the user filter.
  232. querydict_query[ prefix + 'user' ] = request.user.username
  233. querydict_query[ prefix + 'type' ] = app_name
  234. query_page, query_filter = _list_designs(querydict_query, DEFAULT_PAGE_SIZE, prefix)
  235. filter_params = hist_filter
  236. filter_params.update(query_filter)
  237. return render('my_queries.mako', request, {
  238. 'request': request,
  239. 'h_page': hist_page,
  240. 'q_page': query_page,
  241. 'filter_params': filter_params,
  242. 'designs_json': json.dumps([query.id for query in query_page.object_list])
  243. })
  244. def list_query_history(request):
  245. """
  246. View the history of query (for the current user).
  247. We get here from /beeswax/query_history?filterargs, with the options being:
  248. page=<n> - Controls pagination. Defaults to 1.
  249. user=<name> - Show history items from a user. Default to current user only.
  250. Also accepts ':all' to show all history items.
  251. type=<type> - <type> is "beeswax|impala", for design type. Default to show all.
  252. design_id=<id> - Show history for this particular design id.
  253. sort=<key> - Sort by the attribute <key>, which is one of:
  254. "date", "state", "name" (design name), and "type" (design type)
  255. Accepts the form "-date", which sort in descending order.
  256. Default to "-date".
  257. auto_query=<bool> - Show auto generated actions (drop table, read data, etc). Default False
  258. """
  259. DEFAULT_PAGE_SIZE = 30
  260. prefix = 'q-'
  261. share_queries = conf.SHARE_SAVED_QUERIES.get() or request.user.is_superuser
  262. querydict_query = request.GET.copy()
  263. if not share_queries:
  264. querydict_query[prefix + 'user'] = request.user.username
  265. app_name = get_app_name(request)
  266. querydict_query[prefix + 'type'] = app_name
  267. page, filter_params = _list_query_history(request.user, querydict_query, DEFAULT_PAGE_SIZE, prefix)
  268. filter = request.GET.get(prefix + 'search') and request.GET.get(prefix + 'search') or ''
  269. return render('list_history.mako', request, {
  270. 'request': request,
  271. 'page': page,
  272. 'filter_params': filter_params,
  273. 'share_queries': share_queries,
  274. 'prefix': prefix,
  275. 'filter': filter,
  276. })
  277. def download(request, id, format):
  278. assert format in common.DL_FORMATS
  279. query_history = authorized_get_history(request, id, must_exist=True)
  280. db = dbms.get(request.user, query_history.get_query_server_config())
  281. LOG.debug('Download results for query %s: [ %s ]' % (query_history.server_id, query_history.query))
  282. return data_export.download(query_history.get_handle(), format, db)
  283. """
  284. Queries Views
  285. """
  286. def execute_query(request, design_id=None):
  287. """
  288. View function for executing an arbitrary query.
  289. It understands the optional GET/POST params:
  290. on_success_url
  291. If given, it will be displayed when the query is successfully finished.
  292. Otherwise, it will display the view query results page by default.
  293. """
  294. authorized_get_design(request, design_id)
  295. error_message = None
  296. form = QueryForm()
  297. action = request.path
  298. log = None
  299. app_name = get_app_name(request)
  300. query_type = SavedQuery.TYPES_MAPPING[app_name]
  301. design = safe_get_design(request, query_type, design_id)
  302. on_success_url = request.REQUEST.get('on_success_url')
  303. query_server = get_query_server_config(app_name)
  304. db = dbms.get(request.user, query_server)
  305. databases = _get_db_choices(request)
  306. if request.method == 'POST':
  307. form.bind(request.POST)
  308. form.query.fields['database'].choices = databases # Could not do it in the form
  309. to_explain = request.POST.has_key('button-explain')
  310. to_submit = request.POST.has_key('button-submit')
  311. # Always validate the saveform, which will tell us whether it needs explicit saving
  312. if form.is_valid():
  313. to_save = form.saveform.cleaned_data['save']
  314. to_saveas = form.saveform.cleaned_data['saveas']
  315. if to_saveas and not design.is_auto:
  316. # Save As only affects a previously saved query
  317. design = design.clone()
  318. if to_submit or to_save or to_saveas or to_explain:
  319. explicit_save = to_save or to_saveas
  320. design = save_design(request, form, query_type, design, explicit_save)
  321. action = reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id))
  322. if to_explain or to_submit:
  323. query_str = form.query.cleaned_data["query"]
  324. # (Optional) Parameterization.
  325. parameterization = get_parameterization(request, query_str, form, design, to_explain)
  326. if parameterization:
  327. return parameterization
  328. try:
  329. query = HQLdesign(form, query_type=query_type)
  330. if to_explain:
  331. return explain_directly(request, query, design, query_server)
  332. else:
  333. download = request.POST.has_key('download')
  334. return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
  335. except Exception, ex:
  336. error_message, log = expand_exception(ex, db)
  337. else:
  338. if design.id is not None:
  339. data = HQLdesign.loads(design.data).get_query_dict()
  340. form.bind(data)
  341. form.saveform.set_data(design.name, design.desc)
  342. else:
  343. # New design
  344. form.bind()
  345. form.query.fields['database'].choices = databases # Could not do it in the form
  346. return render('execute.mako', request, {
  347. 'action': action,
  348. 'design': design,
  349. 'error_message': error_message,
  350. 'form': form,
  351. 'log': log,
  352. 'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
  353. 'on_success_url': on_success_url,
  354. 'can_edit_name': design and not design.is_auto and design.name,
  355. })
  356. def execute_parameterized_query(request, design_id):
  357. return _run_parameterized_query(request, design_id, False)
  358. def explain_parameterized_query(request, design_id):
  359. return _run_parameterized_query(request, design_id, True)
  360. def watch_query(request, id):
  361. """
  362. Wait for the query to finish and (by default) displays the results of query id.
  363. It understands the optional GET params:
  364. on_success_url
  365. If given, it will be displayed when the query is successfully finished.
  366. Otherwise, it will display the view query results page by default.
  367. context
  368. A string of "name:data" that describes the context
  369. that generated this query result. It may be:
  370. - "table":"<table_name>"
  371. - "design":<design_id>
  372. All other GET params will be passed to on_success_url (if present).
  373. """
  374. # Coerce types; manage arguments
  375. query_history = authorized_get_history(request, id, must_exist=True)
  376. db = dbms.get(request.user, query_history.get_query_server_config())
  377. # GET param: context.
  378. context_param = request.GET.get('context', '')
  379. # GET param: on_success_url. Default to view_results
  380. results_url = reverse(get_app_name(request) + ':view_results', kwargs={'id': id, 'first_row': 0})
  381. if request.GET.get('download', ''):
  382. results_url += '?download=true'
  383. on_success_url = request.GET.get('on_success_url')
  384. if not on_success_url:
  385. on_success_url = results_url
  386. # Go to next statement if asked to continue or when a statement with no dataset finished.
  387. if request.method == 'POST' or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
  388. try:
  389. query_history = db.execute_next_statement(query_history)
  390. except Exception, ex:
  391. pass
  392. # Check query state
  393. handle, state = _get_query_handle_and_state(query_history)
  394. query_history.save_state(state)
  395. if query_history.is_failure():
  396. # When we fetch, Beeswax server will throw us a Exception, which has the
  397. # log we want to display.
  398. return format_preserving_redirect(request, results_url, request.GET)
  399. elif query_history.is_finished() or (query_history.is_success() and query_history.has_results):
  400. return format_preserving_redirect(request, on_success_url, request.GET)
  401. # Still running
  402. log = db.get_log(handle)
  403. # Keep waiting
  404. # - Translate context into something more meaningful (type, data)
  405. query_context = _parse_query_context(context_param)
  406. return render('watch_wait.mako', request, {
  407. 'query': query_history,
  408. 'fwd_params': request.GET.urlencode(),
  409. 'log': log,
  410. 'hadoop_jobs': _parse_out_hadoop_jobs(log),
  411. 'query_context': query_context,
  412. })
  413. def watch_query_refresh_json(request, id):
  414. query_history = authorized_get_history(request, id, must_exist=True)
  415. db = dbms.get(request.user, query_history.get_query_server_config())
  416. handle, state = _get_query_handle_and_state(query_history)
  417. query_history.save_state(state)
  418. try:
  419. if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
  420. db.execute_next_statement(query_history)
  421. handle, state = _get_query_handle_and_state(query_history)
  422. except Exception, ex:
  423. LOG.exception(ex)
  424. handle, state = _get_query_handle_and_state(query_history)
  425. try:
  426. log = db.get_log(handle)
  427. except Exception, ex:
  428. log = str(ex)
  429. jobs = _parse_out_hadoop_jobs(log)
  430. job_urls = dict([(job, reverse('jobbrowser.views.single_job', kwargs=dict(job=job))) for job in jobs])
  431. result = {
  432. 'log': log,
  433. 'jobs': jobs,
  434. 'jobUrls': job_urls,
  435. 'isSuccess': query_history.is_finished() or (query_history.is_success() and query_history.has_results),
  436. 'isFailure': query_history.is_failure()
  437. }
  438. return HttpResponse(json.dumps(result), mimetype="application/json")
  439. def cancel_operation(request, query_id):
  440. response = {'status': -1, 'message': ''}
  441. if request.method != 'POST':
  442. response['message'] = _('A POST request is required.')
  443. else:
  444. try:
  445. query_history = authorized_get_history(request, query_id, must_exist=True)
  446. db = dbms.get(request.user, query_history.get_query_server_config())
  447. db.cancel_operation(query_history.get_handle())
  448. _get_query_handle_and_state(query_history)
  449. response = {'status': 0}
  450. except Exception, e:
  451. response = {'message': unicode(e)}
  452. return HttpResponse(json.dumps(response), mimetype="application/json")
  453. def view_results(request, id, first_row=0):
  454. """
  455. Returns the view for the results of the QueryHistory with the given id.
  456. The query results MUST be ready.
  457. To display query results, one should always go through the watch_query view.
  458. If the result set has has_result_set=False, display an empty result.
  459. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
  460. spits out a warning if first_row doesn't match the servers conception.
  461. Multiple readers will produce a confusing interaction here, and that's known.
  462. It understands the ``context`` GET parameter. (See watch_query().)
  463. """
  464. first_row = long(first_row)
  465. start_over = (first_row == 0)
  466. results = type('Result', (object,), {
  467. 'rows': 0,
  468. 'columns': [],
  469. 'has_more': False,
  470. 'start_row': 0, })
  471. data = []
  472. fetch_error = False
  473. error_message = ''
  474. log = ''
  475. app_name = get_app_name(request)
  476. query_history = authorized_get_history(request, id, must_exist=True)
  477. query_server = query_history.get_query_server_config()
  478. db = dbms.get(request.user, query_server)
  479. handle, state = _get_query_handle_and_state(query_history)
  480. context_param = request.GET.get('context', '')
  481. query_context = _parse_query_context(context_param)
  482. # To remove in Hue 2.4
  483. download = request.GET.get('download', '')
  484. # Update the status as expired should not be accessible
  485. # Impala does not support startover for now
  486. expired = state == models.QueryHistory.STATE.expired
  487. if expired or app_name == 'impala':
  488. state = models.QueryHistory.STATE.expired
  489. query_history.save_state(state)
  490. # Retrieve query results or use empty result if no result set
  491. try:
  492. if query_server['server_name'] == 'impala' and not handle.has_result_set:
  493. downloadable = False
  494. elif not download:
  495. results = db.fetch(handle, start_over, 100)
  496. data = list(results.rows()) # Materialize results
  497. # We display the "Download" button only when we know that there are results:
  498. downloadable = first_row > 0 or data
  499. log = db.get_log(handle)
  500. else:
  501. downloadable = True
  502. except Exception, ex:
  503. fetch_error = True
  504. error_message, log = expand_exception(ex, db, handle)
  505. # Handle errors
  506. error = fetch_error or results is None or expired
  507. context = {
  508. 'error': error,
  509. 'error_message': error_message,
  510. 'query': query_history,
  511. 'results': data,
  512. 'expected_first_row': first_row,
  513. 'log': log,
  514. 'hadoop_jobs': app_name != 'impala' and _parse_out_hadoop_jobs(log),
  515. 'query_context': query_context,
  516. 'can_save': False,
  517. 'context_param': context_param,
  518. 'expired': expired,
  519. 'app_name': app_name,
  520. 'download': download,
  521. 'next_json_set': None
  522. }
  523. if not error:
  524. download_urls = {}
  525. if downloadable:
  526. for format in common.DL_FORMATS:
  527. download_urls[format] = reverse(app_name + ':download', kwargs=dict(id=str(id), format=format))
  528. save_form = beeswax.forms.SaveResultsForm()
  529. results.start_row = first_row
  530. context.update({
  531. 'results': data,
  532. 'has_more': results.has_more,
  533. 'next_row': results.start_row + len(data),
  534. 'start_row': results.start_row,
  535. 'expected_first_row': first_row,
  536. 'columns': results.columns,
  537. 'download_urls': download_urls,
  538. 'save_form': save_form,
  539. 'can_save': query_history.owner == request.user and not download,
  540. 'next_json_set': reverse(get_app_name(request) + ':view_results', kwargs={
  541. 'id': str(id),
  542. 'first_row': results.start_row + len(data)
  543. }) + ('?context=' + context_param or '') + '&format=json'
  544. })
  545. if request.GET.get('format') == 'json':
  546. context = {
  547. 'results': data,
  548. 'has_more': results.has_more,
  549. 'next_row': results.start_row + len(data),
  550. 'start_row': results.start_row,
  551. 'next_json_set': reverse(get_app_name(request) + ':view_results', kwargs={
  552. 'id': str(id),
  553. 'first_row': results.start_row + len(data)
  554. }) + ('?context=' + context_param or '') + '&format=json'
  555. }
  556. return HttpResponse(json.dumps(context), mimetype="application/json")
  557. return render('watch_results.mako', request, context)
  558. def save_results(request, id):
  559. """
  560. Save the results of a query to an HDFS directory or Hive table.
  561. """
  562. query_history = authorized_get_history(request, id, must_exist=True)
  563. app_name = get_app_name(request)
  564. server_id, state = _get_query_handle_and_state(query_history)
  565. query_history.save_state(state)
  566. error_msg, log = None, None
  567. if request.method == 'POST':
  568. if not query_history.is_success():
  569. msg = _('This query is %(state)s. Results unavailable.') % {'state': state}
  570. raise PopupException(msg)
  571. db = dbms.get(request.user, query_history.get_query_server_config())
  572. form = beeswax.forms.SaveResultsForm(request.POST, db=db)
  573. if request.POST.get('cancel'):
  574. return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id))
  575. if form.is_valid():
  576. try:
  577. handle, state = _get_query_handle_and_state(query_history)
  578. result_meta = db.get_results_metadata(handle)
  579. except Exception, ex:
  580. raise PopupException(_('Cannot find query: %s') % ex)
  581. try:
  582. if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
  583. target_dir = form.cleaned_data['target_dir']
  584. query_history = db.insert_query_into_directory(query_history, target_dir)
  585. redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \
  586. + '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir}))
  587. elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
  588. redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
  589. except Exception, ex:
  590. error_msg, log = expand_exception(ex, db)
  591. raise PopupException(_('The result could not be saved: %s') % log, detail=ex)
  592. return redirected
  593. else:
  594. form = beeswax.forms.SaveResultsForm()
  595. if error_msg:
  596. error_msg = _('Failed to save results from query: %(error)s') % {'error': error_msg}
  597. return render('save_results.mako', request, {
  598. 'action': reverse(get_app_name(request) + ':save_results', kwargs={'id': str(id)}),
  599. 'form': form,
  600. 'error_msg': error_msg,
  601. 'log': log,
  602. })
  603. def confirm_query(request, query, on_success_url=None):
  604. """
  605. Used by other forms to confirm a query before it's executed.
  606. The form is the same as execute_query below.
  607. query - The HQL about to be executed
  608. on_success_url - The page to go to upon successful execution
  609. """
  610. mform = QueryForm()
  611. mform.bind()
  612. mform.query.initial = dict(query=query)
  613. return render('execute.mako', request, {
  614. 'form': mform,
  615. 'action': reverse(get_app_name(request) + ':execute_query'),
  616. 'error_message': None,
  617. 'design': None,
  618. 'on_success_url': on_success_url,
  619. 'design': None,
  620. 'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
  621. })
  622. def explain_directly(request, query, design, query_server):
  623. explanation = dbms.get(request.user, query_server).explain(query)
  624. context = ("design", design)
  625. return render('explain.mako', request, dict(query=query, explanation=explanation.textual, query_context=context))
  626. def configuration(request):
  627. app_name = get_app_name(request)
  628. query_server = get_query_server_config(app_name)
  629. config_values = dbms.get(request.user, query_server).get_default_configuration(
  630. bool(request.REQUEST.get("include_hadoop", False)))
  631. for value in config_values:
  632. if 'password' in value.key.lower():
  633. value.value = "*" * 10
  634. return render("configuration.mako", request, {'config_values': config_values})
  635. """
  636. Other views
  637. """
  638. def install_examples(request):
  639. response = {'status': -1, 'message': ''}
  640. if request.method == 'POST':
  641. try:
  642. app_name = get_app_name(request)
  643. beeswax.management.commands.beeswax_install_examples.Command().handle_noargs(app_name=app_name)
  644. response['status'] = 0 # Always return 0 currently
  645. except Exception, err:
  646. LOG.exception(err)
  647. response['message'] = str(err)
  648. else:
  649. response['message'] = _('A POST request is required.')
  650. return HttpResponse(json.dumps(response), mimetype="application/json")
  651. @login_notrequired
  652. def query_done_cb(request, server_id):
  653. """
  654. A callback for query completion notification. When the query is done,
  655. BeeswaxServer notifies us by sending a GET request to this view.
  656. This view should always return a 200 response, to reflect that the
  657. notification is delivered to the right view.
  658. """
  659. message_template = '<html><head></head>%(message)s<body></body></html>'
  660. message = {'message': 'error'}
  661. try:
  662. query_history = models.QueryHistory.objects.get(server_id=server_id)
  663. # Update the query status
  664. query_history.set_to_available()
  665. # Find out details about the query
  666. if not query_history.notify:
  667. message['message'] = 'email_notify is false'
  668. return HttpResponse(message_template % message)
  669. design = query_history.design
  670. user = query_history.owner
  671. subject = _("Beeswax query completed")
  672. if design:
  673. subject += ": %s" % (design.name,)
  674. link = "%s%s" % \
  675. (get_desktop_uri_prefix(),
  676. reverse(get_app_name(request) + ':watch_query', kwargs={'id': query_history.id}))
  677. body = _("%(subject)s. You may see the results here: %(link)s\n\nQuery:\n%(query)s") % {
  678. 'subject': subject, 'link': link, 'query': query_history.query
  679. }
  680. user.email_user(subject, body)
  681. message['message'] = 'sent'
  682. except Exception, ex:
  683. msg = "Failed to send query completion notification via e-mail: %s" % (ex)
  684. LOG.error(msg)
  685. message['message'] = msg
  686. return HttpResponse(message_template % message)
  687. def autocomplete(request, database=None, table=None):
  688. app_name = get_app_name(request)
  689. query_server = get_query_server_config(app_name)
  690. db = dbms.get(request.user, query_server)
  691. response = {}
  692. try:
  693. if database is None:
  694. response['databases'] = db.get_databases()
  695. elif table is None:
  696. response['tables'] = db.get_tables(database=database)
  697. else:
  698. t = db.get_table(database, table)
  699. response['columns'] = [column.name for column in t.cols]
  700. except Exception, e:
  701. LOG.warn('Autocomplete data fetching error %s.%s: %s' % (database, table, e))
  702. response['error'] = e.message
  703. return HttpResponse(json.dumps(response), mimetype="application/json")
  704. """
  705. Utils
  706. """
  707. def authorized_get_design(request, design_id, owner_only=False, must_exist=False):
  708. if design_id is None and not must_exist:
  709. return None
  710. try:
  711. design = models.SavedQuery.objects.get(id=design_id)
  712. except models.SavedQuery.DoesNotExist:
  713. if must_exist:
  714. raise PopupException(_('Design %(id)s does not exist.') % {'id': design_id})
  715. else:
  716. return None
  717. if not conf.SHARE_SAVED_QUERIES.get() and (not request.user.is_superuser or owner_only) \
  718. and design.owner != request.user:
  719. raise PopupException(_('Cannot access design %(id)s') % {'id': design_id})
  720. else:
  721. return design
  722. def authorized_get_history(request, query_history_id, owner_only=False, must_exist=False):
  723. if query_history_id is None and not must_exist:
  724. return None
  725. try:
  726. query_history = models.QueryHistory.get(id=query_history_id)
  727. except models.QueryHistory.DoesNotExist:
  728. if must_exist:
  729. raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id})
  730. else:
  731. return None
  732. if not conf.SHARE_SAVED_QUERIES.get() and (not request.user.is_superuser or owner_only) \
  733. and query_history.owner != request.user:
  734. raise PopupException(_('Cannot access QueryHistory %(id)s') % {'id': query_history_id})
  735. else:
  736. return query_history
  737. def safe_get_design(request, design_type, design_id=None):
  738. """
  739. Return a new design, if design_id is None,
  740. Return the design with the given id and type. If the design is not found,
  741. display a notification and return a new design.
  742. """
  743. design = None
  744. if design_id is not None:
  745. try:
  746. design = models.SavedQuery.get(design_id, request.user, design_type)
  747. except models.SavedQuery.DoesNotExist:
  748. messages.error(request, _('Design does not exist'))
  749. if design is None:
  750. design = models.SavedQuery(owner=request.user, type=design_type)
  751. return design
  752. def get_parameterization(request, query_str, form, design, is_explain):
  753. """
  754. Figures out whether a design is parameterizable, and, if so,
  755. returns a form to fill out. Returns None if there's no parameterization
  756. to do.
  757. """
  758. if form.query.cleaned_data["is_parameterized"]:
  759. parameters_form = make_parameterization_form(query_str)
  760. if parameters_form:
  761. return render("parameterization.mako", request, dict(
  762. form=parameters_form(prefix="parameterization"),
  763. design=design,
  764. explain=is_explain))
  765. return None
  766. def make_parameterization_form(query_str):
  767. """
  768. Creates a django form on the fly with arguments from the
  769. query.
  770. """
  771. variables = find_variables(query_str)
  772. if len(variables) > 0:
  773. class Form(forms.Form):
  774. for name in sorted(variables):
  775. locals()[name] = forms.CharField(required=True)
  776. return Form
  777. else:
  778. return None
  779. def _run_parameterized_query(request, design_id, explain):
  780. """
  781. Given a design and arguments to parameterize that design, runs the query.
  782. - explain is a boolean to determine whether to run as an explain or as an
  783. execute.
  784. This is an extra "step" in the flow from execute_query.
  785. """
  786. design = authorized_get_design(request, design_id, must_exist=True)
  787. # Reconstitute the form
  788. design_obj = beeswax.design.HQLdesign.loads(design.data)
  789. query_form = QueryForm()
  790. params = design_obj.get_query_dict()
  791. params.update(request.POST)
  792. databases = _get_db_choices(request)
  793. query_form.bind(params)
  794. query_form.query.fields['database'].choices = databases # Could not do it in the form
  795. if not query_form.is_valid():
  796. raise PopupException(_("Query form is invalid: %s") % query_form.errors)
  797. query_str = query_form.query.cleaned_data["query"]
  798. app_name = get_app_name(request)
  799. query_server = get_query_server_config(app_name)
  800. query_type = SavedQuery.TYPES_MAPPING[app_name]
  801. parameterization_form_cls = make_parameterization_form(query_str)
  802. if not parameterization_form_cls:
  803. raise PopupException(_("Query is not parameterizable."))
  804. parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
  805. if parameterization_form.is_valid():
  806. real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
  807. query = HQLdesign(query_form, query_type=query_type)
  808. query._data_dict['query']['query'] = real_query
  809. try:
  810. if explain:
  811. return explain_directly(request, query, design, query_server)
  812. else:
  813. return execute_directly(request, query, query_server, design)
  814. except Exception, ex:
  815. db = dbms.get(request.user, query_server)
  816. error_message, log = expand_exception(ex, db)
  817. return render('execute.mako', request, {
  818. 'action': reverse(get_app_name(request) + ':execute_query'),
  819. 'design': design,
  820. 'error_message': error_message,
  821. 'form': query_form,
  822. 'log': log,
  823. 'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
  824. })
  825. else:
  826. return render("parameterization.mako", request, dict(form=parameterization_form, design=design, explain=explain))
  827. def execute_directly(request, query, query_server=None, design=None, tablename=None,
  828. on_success_url=None, on_success_params=None, **kwargs):
  829. """
  830. execute_directly(request, query_msg, tablename, design) -> HTTP response for execution
  831. This method wraps around dbms.execute_query() to take care of the HTTP response
  832. after the execution.
  833. query
  834. The HQL model Query object.
  835. query_server
  836. To which Query Server to submit the query.
  837. Dictionary with keys: ['server_name', 'server_host', 'server_port'].
  838. design
  839. The design associated with the query.
  840. tablename
  841. The associated table name for the context.
  842. on_success_url
  843. Where to go after the query is done. The URL handler may expect an option "context" GET
  844. param. (See ``watch_query``.) For advanced usage, on_success_url can be a function, in
  845. which case the on complete URL is the return of:
  846. on_success_url(history_obj) -> URL string
  847. Defaults to the view results page.
  848. on_success_params
  849. Optional params to pass to the on_success_url (in additional to "context").
  850. Note that this may throw a Beeswax exception.
  851. """
  852. if design is not None:
  853. authorized_get_design(request, design.id)
  854. db = dbms.get(request.user, query_server)
  855. database = query.query.get('database', 'default')
  856. db.use(database)
  857. history_obj = db.execute_query(query, design)
  858. watch_url = reverse(get_app_name(request) + ':watch_query', kwargs={'id': history_obj.id})
  859. if 'download' in kwargs and kwargs['download']:
  860. watch_url += '?download=true'
  861. # Prepare the GET params for the watch_url
  862. get_dict = QueryDict(None, mutable=True)
  863. # (1) context
  864. if design:
  865. get_dict['context'] = make_query_context('design', design.id)
  866. elif tablename:
  867. get_dict['context'] = make_query_context('table', '%s:%s' % (tablename, database))
  868. # (2) on_success_url
  869. if on_success_url:
  870. if callable(on_success_url):
  871. on_success_url = on_success_url(history_obj)
  872. get_dict['on_success_url'] = on_success_url
  873. # (3) misc
  874. if on_success_params:
  875. get_dict.update(on_success_params)
  876. return format_preserving_redirect(request, watch_url, get_dict)
  877. def _list_designs(querydict, page_size, prefix="", is_trashed=False):
  878. """
  879. _list_designs(querydict, page_size, prefix, user) -> (page, filter_param)
  880. A helper to gather the designs page. It understands all the GET params in
  881. ``list_designs``, by reading keys from the ``querydict`` with the given ``prefix``.
  882. If a ``user`` is specified, only the saved queries of this user will be returned.
  883. This has priority over the ``user`` in the ``querydict`` parameter.
  884. """
  885. DEFAULT_SORT = ('-', 'date') # Descending date
  886. SORT_ATTR_TRANSLATION = dict(
  887. date='mtime',
  888. name='name',
  889. desc='desc',
  890. type='type',
  891. )
  892. # Filtering. Only display designs explicitly saved.
  893. db_queryset = models.SavedQuery.objects.filter(is_auto=False, is_trashed=is_trashed)
  894. user = querydict.get(prefix + 'user')
  895. if user is not None:
  896. db_queryset = db_queryset.filter(owner__username=user)
  897. # Design type
  898. d_type = querydict.get(prefix + 'type')
  899. if d_type:
  900. d_type = str(d_type)
  901. if d_type not in SavedQuery.TYPES_MAPPING.keys():
  902. LOG.warn('Bad parameter to list_designs: type=%s' % (d_type,))
  903. else:
  904. db_queryset = db_queryset.filter(type=SavedQuery.TYPES_MAPPING[d_type])
  905. # Text search
  906. frag = querydict.get(prefix + 'text')
  907. if frag:
  908. db_queryset = db_queryset.filter(Q(name__icontains=frag) | Q(desc__icontains=frag))
  909. # Ordering
  910. sort_key = querydict.get(prefix + 'sort')
  911. if sort_key:
  912. if sort_key[0] == '-':
  913. sort_dir, sort_attr = '-', sort_key[1:]
  914. else:
  915. sort_dir, sort_attr = '', sort_key
  916. if not SORT_ATTR_TRANSLATION.has_key(sort_attr):
  917. LOG.warn('Bad parameter to list_designs: sort=%s' % (sort_key,))
  918. sort_dir, sort_attr = DEFAULT_SORT
  919. else:
  920. sort_dir, sort_attr = DEFAULT_SORT
  921. db_queryset = db_queryset.order_by(sort_dir + SORT_ATTR_TRANSLATION[sort_attr])
  922. pagenum = int(querydict.get(prefix + 'page', 1))
  923. paginator = Paginator(db_queryset, page_size)
  924. page = paginator.page(pagenum)
  925. # We need to pass the parameters back to the template to generate links
  926. keys_to_copy = [ prefix + key for key in ('user', 'type', 'sort') ]
  927. filter_params = copy_query_dict(querydict, keys_to_copy)
  928. return page, filter_params
  929. def _get_query_handle_and_state(query_history):
  930. """
  931. Front-end wrapper to handle exceptions. Expects the query to be submitted.
  932. """
  933. handle = query_history.get_handle()
  934. if handle is None:
  935. raise PopupException(_("Failed to retrieve query state from the Query Server."))
  936. query_server = query_history.get_query_server_config()
  937. if query_server['server_name'] == 'impala' and not handle.has_result_set:
  938. state = QueryHistory.STATE.available
  939. else:
  940. state = dbms.get(query_history.owner, query_history.get_query_server_config()).get_state(handle)
  941. if state is None:
  942. raise PopupException(_("Failed to contact Server to check query status."))
  943. return (handle, state)
  944. def _parse_query_context(context):
  945. """
  946. _parse_query_context(context) -> ('table', <table_name>) -or- ('design', <design_obj>)
  947. """
  948. if not context:
  949. return None
  950. pair = context.split(':', 1)
  951. if len(pair) != 2 or pair[0] not in ('table', 'design'):
  952. LOG.error("Invalid query context data: %s" % (context,))
  953. return None
  954. if pair[0] == 'design': # Translate design id to design obj
  955. pair[1] = models.SavedQuery.get(int(pair[1]))
  956. return pair
  957. HADOOP_JOBS_RE = re.compile("(http[^\s]*/jobdetails.jsp\?jobid=([a-z0-9_]*))")
  958. HADOOP_YARN_JOBS_RE = re.compile("(http[^\s]*/proxy/([a-z0-9_]+?)/)")
  959. def _parse_out_hadoop_jobs(log):
  960. """
  961. Ideally, Hive would tell us what jobs it has run directly
  962. from the Thrift interface. For now, we parse the logs
  963. to look for URLs to those jobs.
  964. """
  965. ret = []
  966. for match in HADOOP_JOBS_RE.finditer(log):
  967. full_job_url, job_id = match.groups()
  968. # We ignore full_job_url for now, but it may
  969. # come in handy if we support multiple MR clusters
  970. # correctly.
  971. # Ignore duplicates
  972. if job_id not in ret:
  973. ret.append(job_id)
  974. for match in HADOOP_YARN_JOBS_RE.finditer(log):
  975. full_job_url, job_id = match.groups()
  976. if job_id not in ret:
  977. ret.append(job_id)
  978. return ret
  979. def _copy_prefix(prefix, base_dict):
  980. """Copy keys starting with ``prefix``"""
  981. querydict = QueryDict(None, mutable=True)
  982. for key, val in base_dict.iteritems():
  983. if key.startswith(prefix):
  984. querydict[key] = val
  985. return querydict
  986. def _list_query_history(user, querydict, page_size, prefix=""):
  987. """
  988. _list_query_history(user, querydict, page_size, prefix) -> (page, filter_param)
  989. A helper to gather the history page. It understands all the GET params in
  990. ``list_query_history``, by reading keys from the ``querydict`` with the
  991. given ``prefix``.
  992. """
  993. DEFAULT_SORT = ('-', 'date') # Descending date
  994. SORT_ATTR_TRANSLATION = dict(
  995. date='submission_date',
  996. state='last_state',
  997. name='design__name',
  998. type='design__type',
  999. )
  1000. db_queryset = models.QueryHistory.objects.select_related()
  1001. # Filtering
  1002. #
  1003. # Queries without designs are the ones we submitted on behalf of the user,
  1004. # (e.g. view table data). Exclude those when returning query history.
  1005. if not querydict.get(prefix + 'auto_query', False):
  1006. db_queryset = db_queryset.filter(design__isnull=False)
  1007. user_filter = querydict.get(prefix + 'user', user.username)
  1008. if user_filter != ':all':
  1009. db_queryset = db_queryset.filter(owner__username=user_filter)
  1010. # Design id
  1011. design_id = querydict.get(prefix + 'design_id')
  1012. if design_id:
  1013. db_queryset = db_queryset.filter(design__id=int(design_id))
  1014. # Search
  1015. search_filter = querydict.get(prefix + 'search')
  1016. if search_filter:
  1017. db_queryset = db_queryset.filter(Q(design__name__icontains=search_filter) | Q(query__icontains=search_filter) | Q(owner__username__icontains=search_filter))
  1018. # Design type
  1019. d_type = querydict.get(prefix + 'type')
  1020. if d_type:
  1021. if d_type not in SavedQuery.TYPES_MAPPING.keys():
  1022. LOG.warn('Bad parameter to list_query_history: type=%s' % (d_type,))
  1023. else:
  1024. db_queryset = db_queryset.filter(design__type=SavedQuery.TYPES_MAPPING[d_type])
  1025. # Ordering
  1026. sort_key = querydict.get(prefix + 'sort')
  1027. if sort_key:
  1028. sort_dir, sort_attr = '', sort_key
  1029. if sort_key[0] == '-':
  1030. sort_dir, sort_attr = '-', sort_key[1:]
  1031. if not SORT_ATTR_TRANSLATION.has_key(sort_attr):
  1032. LOG.warn('Bad parameter to list_query_history: sort=%s' % (sort_key,))
  1033. sort_dir, sort_attr = DEFAULT_SORT
  1034. else:
  1035. sort_dir, sort_attr = DEFAULT_SORT
  1036. db_queryset = db_queryset.order_by(sort_dir + SORT_ATTR_TRANSLATION[sort_attr])
  1037. # Get the total return count before slicing
  1038. total_count = db_queryset.count()
  1039. # Slicing (must be the last filter applied)
  1040. pagenum = int(querydict.get(prefix + 'page', 1))
  1041. if pagenum < 1:
  1042. pagenum = 1
  1043. db_queryset = db_queryset[ page_size * (pagenum - 1) : page_size * pagenum ]
  1044. paginator = Paginator(db_queryset, page_size, total=total_count)
  1045. page = paginator.page(pagenum)
  1046. # We do slicing ourselves, rather than letting the Paginator handle it, in order to
  1047. # update the last_state on the running queries
  1048. for history in page.object_list:
  1049. _update_query_state(history.get_full_object())
  1050. # We need to pass the parameters back to the template to generate links
  1051. keys_to_copy = [ prefix + key for key in ('user', 'type', 'sort', 'design_id', 'auto_query') ]
  1052. filter_params = copy_query_dict(querydict, keys_to_copy)
  1053. return page, filter_params
  1054. def _update_query_state(query_history):
  1055. """
  1056. Update the last_state for a QueryHistory object. Returns success as True/False.
  1057. This only occurs iff the current last_state is submitted or running, since the other
  1058. states are stable, more-or-less.
  1059. Note that there is a transition from available/failed to expired. That occurs lazily
  1060. when the user attempts to view results that have expired.
  1061. """
  1062. if query_history.last_state <= models.QueryHistory.STATE.running.index:
  1063. try:
  1064. state_enum = dbms.get(query_history.owner, query_history.get_query_server_config()).get_state(query_history.get_handle())
  1065. if state_enum is None:
  1066. # Error was logged at the source
  1067. return False
  1068. except Exception, e:
  1069. LOG.error(e)
  1070. state_enum = models.QueryHistory.STATE.failed
  1071. query_history.save_state(state_enum)
  1072. return True
  1073. def _get_db_choices(request):
  1074. app_name = get_app_name(request)
  1075. query_server = get_query_server_config(app_name)
  1076. db = dbms.get(request.user, query_server)
  1077. dbs = db.get_databases()
  1078. return ((db, db) for db in dbs)
  1079. WHITESPACE = re.compile("\s+", re.MULTILINE)
  1080. def collapse_whitespace(s):
  1081. return WHITESPACE.sub(" ", s).strip()