views.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from builtins import object
  18. import json
  19. import logging
  20. from django.urls import reverse
  21. from django.db.models import Q
  22. from django.shortcuts import redirect
  23. from django.utils.translation import ugettext as _
  24. from django.views.decorators.clickjacking import xframe_options_exempt
  25. from django.views.decorators.http import require_POST
  26. from beeswax.data_export import DOWNLOAD_COOKIE_AGE
  27. from beeswax.management.commands import beeswax_install_examples
  28. from desktop.auth.decorators import admin_required
  29. from desktop.conf import ENABLE_DOWNLOAD, USE_NEW_EDITOR
  30. from desktop.lib import export_csvxls
  31. from desktop.lib.connectors.models import Connector
  32. from desktop.lib.django_util import render, JsonResponse
  33. from desktop.lib.exceptions_renderable import PopupException
  34. from desktop.lib.json_utils import JSONEncoderForHTML
  35. from desktop.models import Document2, Document, FilesystemException, _get_gist_document
  36. from desktop.views import serve_403_error
  37. from metadata.conf import has_optimizer, has_catalog, has_workload_analytics
  38. from notebook.conf import get_ordered_interpreters, SHOW_NOTEBOOKS, EXAMPLES
  39. from notebook.connectors.base import Notebook, _get_snippet_name, get_interpreter
  40. from notebook.connectors.spark_shell import SparkApi
  41. from notebook.decorators import check_editor_access_permission, check_document_access_permission, check_document_modify_permission
  42. from notebook.management.commands.notebook_setup import Command
  43. from notebook.models import make_notebook, _get_editor_type, get_api, _get_dialect_example
  44. LOG = logging.getLogger(__name__)
  45. def notebooks(request):
  46. editor_type = request.GET.get('type', 'notebook')
  47. if editor_type != 'notebook':
  48. if USE_NEW_EDITOR.get():
  49. notebooks = [doc.to_dict() for doc in Document2.objects.documents(
  50. user=request.user).search_documents(types=['query-%s' % editor_type])]
  51. else:
  52. notebooks = [
  53. d.content_object.to_dict()
  54. for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra__startswith='query'))
  55. if not d.content_object.is_history and d.content_object.type == 'query-' + editor_type
  56. ]
  57. else:
  58. if USE_NEW_EDITOR.get():
  59. notebooks = [doc.to_dict() for doc in Document2.objects.documents(user=request.user).search_documents(types=['notebook'])]
  60. else:
  61. notebooks = [
  62. d.content_object.to_dict()
  63. for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra='notebook'))
  64. if not d.content_object.is_history
  65. ]
  66. return render('notebooks.mako', request, {
  67. 'notebooks_json': json.dumps(notebooks, cls=JSONEncoderForHTML),
  68. 'editor_type': editor_type
  69. })
  70. @check_document_access_permission
  71. def notebook(request, is_embeddable=False):
  72. if not SHOW_NOTEBOOKS.get() or not request.user.has_hue_permission(action="access", app='notebook'):
  73. return serve_403_error(request)
  74. notebook_id = request.GET.get('notebook', request.GET.get('editor'))
  75. is_yarn_mode = False
  76. try:
  77. from spark.conf import LIVY_SERVER_SESSION_KIND
  78. is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
  79. except:
  80. LOG.exception('Spark is not enabled')
  81. return render('notebook.mako', request, {
  82. 'editor_id': notebook_id or None,
  83. 'notebooks_json': '{}',
  84. 'is_embeddable': request.GET.get('is_embeddable', False),
  85. 'options_json': json.dumps({
  86. 'languages': get_ordered_interpreters(request.user),
  87. 'session_properties': SparkApi.to_properties(),
  88. 'is_optimizer_enabled': has_optimizer(),
  89. 'is_wa_enabled': has_workload_analytics(),
  90. 'is_navigator_enabled': has_catalog(request.user),
  91. 'editor_type': 'notebook'
  92. }),
  93. 'is_yarn_mode': is_yarn_mode,
  94. })
  95. @check_document_access_permission
  96. def notebook_embeddable(request):
  97. return notebook(request, True)
  98. @check_editor_access_permission()
  99. @check_document_access_permission
  100. def editor(request, is_mobile=False, is_embeddable=False):
  101. editor_id = request.GET.get('editor')
  102. editor_type = request.GET.get('type', 'hive')
  103. gist_id = request.GET.get('gist')
  104. if editor_type == 'notebook' or request.GET.get('notebook'):
  105. return notebook(request)
  106. if editor_type == 'gist':
  107. gist_doc = _get_gist_document(uuid=gist_id)
  108. editor_type = gist_doc.extra
  109. if EXAMPLES.AUTO_OPEN.get() and not editor_id:
  110. sample_query = _get_dialect_example(dialect=editor_type)
  111. if sample_query:
  112. editor_id = sample_query.id
  113. if editor_id and not gist_id: # Open existing saved editor document
  114. editor_type = _get_editor_type(editor_id)
  115. template = 'editor.mako'
  116. if is_mobile:
  117. template = 'editor_m.mako'
  118. return render(template, request, {
  119. 'editor_id': editor_id or None,
  120. 'notebooks_json': '{}',
  121. 'is_embeddable': request.GET.get('is_embeddable', False),
  122. 'editor_type': editor_type,
  123. 'options_json': json.dumps({
  124. 'languages': get_ordered_interpreters(request.user),
  125. 'mode': 'editor',
  126. 'is_optimizer_enabled': has_optimizer(),
  127. 'is_wa_enabled': has_workload_analytics(),
  128. 'is_navigator_enabled': has_catalog(request.user),
  129. 'editor_type': editor_type,
  130. 'mobile': is_mobile
  131. })
  132. })
  133. @check_document_access_permission
  134. def editor_embeddable(request):
  135. return editor(request, False, True)
  136. @check_document_access_permission
  137. def editor_m(request):
  138. return editor(request, True)
  139. def new(request):
  140. return notebook(request)
  141. def browse(request, database, table, partition_spec=None):
  142. snippet = {'type': request.POST.get('sourceType', 'hive')}
  143. statement = get_api(request, snippet).get_browse_query(snippet, database, table, partition_spec)
  144. editor_type = snippet['type']
  145. namespace = request.POST.get('namespace', 'default')
  146. compute = json.loads(request.POST.get('cluster', '{}'))
  147. if request.method == 'POST':
  148. notebook = make_notebook(
  149. name='Execute and watch',
  150. editor_type=editor_type,
  151. statement=statement,
  152. database=database,
  153. status='ready-execute',
  154. is_task=True,
  155. namespace=namespace,
  156. compute=compute
  157. )
  158. return JsonResponse(notebook.execute(request, batch=False))
  159. else:
  160. editor = make_notebook(
  161. name='Browse',
  162. editor_type=editor_type,
  163. statement=statement,
  164. status='ready-execute',
  165. namespace=namespace,
  166. compute=compute
  167. )
  168. return render('editor.mako', request, {
  169. 'notebooks_json': json.dumps([editor.get_data()]),
  170. 'options_json': json.dumps({
  171. 'languages': get_ordered_interpreters(request.user),
  172. 'mode': 'editor',
  173. 'editor_type': editor_type
  174. }),
  175. 'editor_type': editor_type,
  176. })
  177. # Deprecated in Hue 4
  178. @check_document_access_permission
  179. def execute_and_watch(request):
  180. notebook_id = request.GET.get('editor', request.GET.get('notebook'))
  181. snippet_id = int(request.GET['snippet'])
  182. action = request.GET['action']
  183. destination = request.GET['destination']
  184. notebook = Notebook(document=Document2.objects.get(id=notebook_id)).get_data()
  185. snippet = notebook['snippets'][snippet_id]
  186. editor_type = snippet['type']
  187. api = get_api(request, snippet)
  188. if action == 'save_as_table':
  189. sql, success_url = api.export_data_as_table(notebook, snippet, destination)
  190. editor = make_notebook(
  191. name='Execute and watch',
  192. editor_type=editor_type,
  193. statement=sql,
  194. status='ready-execute',
  195. database=snippet['database']
  196. )
  197. elif action == 'insert_as_query':
  198. # TODO: checks/workarounds in case of non impersonation or Sentry
  199. # TODO: keep older simpler way in case of known not many rows?
  200. sql, success_url = api.export_large_data_to_hdfs(notebook, snippet, destination)
  201. editor = make_notebook(
  202. name='Execute and watch',
  203. editor_type=editor_type,
  204. statement=sql,
  205. status='ready-execute',
  206. database=snippet['database'],
  207. on_success_url=success_url
  208. )
  209. elif action == 'index_query':
  210. if destination == '__hue__':
  211. destination = _get_snippet_name(notebook, unique=True, table_format=True)
  212. live_indexing = True
  213. else:
  214. live_indexing = False
  215. sql, success_url = api.export_data_as_table(notebook, snippet, destination, is_temporary=True, location='')
  216. editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
  217. sample = get_api(request, snippet).fetch_result(notebook, snippet, 0, start_over=True)
  218. from indexer.api3 import _index # Will ve moved to the lib
  219. from indexer.file_format import HiveFormat
  220. from indexer.fields import Field
  221. file_format = {
  222. 'name': 'col',
  223. 'inputFormat': 'query',
  224. 'format': {'quoteChar': '"', 'recordSeparator': '\n', 'type': 'csv', 'hasHeader': False, 'fieldSeparator': '\u0001'},
  225. "sample": '',
  226. "columns": [
  227. Field(col['name'].rsplit('.')[-1], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
  228. for col in sample['meta']
  229. ]
  230. }
  231. if live_indexing:
  232. file_format['inputFormat'] = 'hs2_handle'
  233. file_format['fetch_handle'] = lambda rows, start_over: get_api(
  234. request, snippet).fetch_result(notebook, snippet, rows=rows, start_over=start_over)
  235. job_handle = _index(request, file_format, destination, query=notebook['uuid'])
  236. if live_indexing:
  237. return redirect(reverse('search:browse', kwargs={'name': destination}))
  238. else:
  239. return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_handle['handle']['id']}))
  240. else:
  241. raise PopupException(_('Action %s is unknown') % action)
  242. return render('editor.mako', request, {
  243. 'notebooks_json': json.dumps([editor.get_data()]),
  244. 'options_json': json.dumps({
  245. 'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
  246. 'mode': 'editor',
  247. 'editor_type': editor_type,
  248. 'success_url': success_url
  249. }),
  250. 'editor_type': editor_type,
  251. })
  252. @check_document_modify_permission()
  253. def delete(request):
  254. response = {'status': -1}
  255. notebooks = json.loads(request.POST.get('notebooks', '[]'))
  256. if not notebooks:
  257. response['message'] = _('No notebooks have been selected for deletion.')
  258. else:
  259. ctr = 0
  260. failures = []
  261. for notebook in notebooks:
  262. try:
  263. doc2 = Document2.objects.get_by_uuid(user=request.user, uuid=notebook['uuid'], perm_type='write')
  264. doc = doc2._get_doc1()
  265. doc.can_write_or_exception(request.user)
  266. doc2.trash()
  267. ctr += 1
  268. except FilesystemException as e:
  269. failures.append(notebook['uuid'])
  270. LOG.exception("Failed to delete document with UUID %s that is writable by user %s, skipping." % (
  271. notebook['uuid'], request.user.username))
  272. response['status'] = 0
  273. if failures:
  274. response['errors'] = failures
  275. response['message'] = _('Trashed %d notebook(s) and failed to delete %d notebook(s).') % (ctr, len(failures))
  276. else:
  277. response['message'] = _('Trashed %d notebook(s)') % ctr
  278. return JsonResponse(response)
  279. @check_document_access_permission
  280. def copy(request):
  281. response = {'status': -1}
  282. notebooks = json.loads(request.POST.get('notebooks', '[]'))
  283. if not notebooks:
  284. response['message'] = _('No notebooks have been selected for copying.')
  285. else:
  286. ctr = 0
  287. failures = []
  288. for notebook in notebooks:
  289. try:
  290. doc2 = Document2.objects.get_by_uuid(user=request.user, uuid=notebook['uuid'])
  291. doc = doc2._get_doc1()
  292. name = doc2.name + '-copy'
  293. doc2 = doc2.copy(name=name, owner=request.user)
  294. doc.copy(content_object=doc2, name=name, owner=request.user)
  295. except FilesystemException as e:
  296. failures.append(notebook['uuid'])
  297. LOG.exception("Failed to copy document with UUID %s accessible by user %s, skipping." % (notebook['uuid'], request.user.username))
  298. response['status'] = 0
  299. if failures:
  300. response['errors'] = failures
  301. response['message'] = _('Copied %d notebook(s) and failed to copy %d notebook(s).') % (ctr, len(failures))
  302. else:
  303. response['message'] = _('Copied %d notebook(s)') % ctr
  304. return JsonResponse(response)
  305. @check_document_access_permission
  306. def download(request):
  307. if not ENABLE_DOWNLOAD.get():
  308. return serve_403_error(request)
  309. notebook = json.loads(request.POST.get('notebook', '{}'))
  310. snippet = json.loads(request.POST.get('snippet', '{}'))
  311. file_format = request.POST.get('format', 'csv')
  312. user_agent = request.META.get('HTTP_USER_AGENT')
  313. file_name = _get_snippet_name(notebook)
  314. content_generator = get_api(request, snippet).download(notebook, snippet, file_format=file_format)
  315. response = export_csvxls.make_response(content_generator, file_format, file_name, user_agent=user_agent)
  316. if snippet['id']:
  317. response.set_cookie(
  318. 'download-%s' % snippet['id'],
  319. json.dumps({
  320. 'truncated': 'false',
  321. 'row_counter': '0'
  322. }),
  323. max_age=DOWNLOAD_COOKIE_AGE
  324. )
  325. if response:
  326. request.audit = {
  327. 'operation': 'DOWNLOAD',
  328. 'operationText': 'User %s downloaded results from %s as %s' % (request.user.username, _get_snippet_name(notebook), file_format),
  329. 'allowed': True
  330. }
  331. return response
  332. @require_POST
  333. @admin_required
  334. def install_examples(request):
  335. response = {'status': -1, 'message': '', 'errorMessage': ''}
  336. try:
  337. connector = Connector.objects.get(id=request.POST.get('connector'))
  338. if connector:
  339. dialect = connector.dialect
  340. db_name = request.POST.get('db_name', 'default')
  341. interpreter = get_interpreter(connector_type=connector.to_dict()['type'], user=request.user)
  342. successes, errors = beeswax_install_examples.Command().handle(
  343. dialect=dialect, db_name=db_name, user=request.user, interpreter=interpreter, request=request
  344. )
  345. response['message'] = ' '.join(successes)
  346. response['errorMessage'] = ' '.join(errors)
  347. response['status'] = len(errors)
  348. else:
  349. Command().handle(user=request.user, dialect=request.POST.get('dialect', 'hive'))
  350. response['status'] = 0
  351. response['message'] = _('Examples refreshed')
  352. except Exception as e:
  353. msg = 'Error during Editor samples installation'
  354. LOG.exception(msg)
  355. response['errorMessage'] = msg + ': ' + str(e)
  356. return JsonResponse(response)