api2.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from future import standard_library
  18. standard_library.install_aliases()
  19. from builtins import map
  20. import logging
  21. import os
  22. import json
  23. import sys
  24. import tempfile
  25. import zipfile
  26. from datetime import datetime
  27. from django.core import management
  28. from django.db import transaction
  29. from django.http import HttpResponse
  30. from django.shortcuts import redirect
  31. from django.utils.html import escape
  32. from django.utils.translation import ugettext as _
  33. from django.views.decorators.csrf import ensure_csrf_cookie
  34. from django.views.decorators.http import require_POST
  35. from metadata.conf import has_catalog
  36. from metadata.catalog_api import search_entities as metadata_search_entities, _highlight, search_entities_interactive as metadata_search_entities_interactive
  37. from notebook.connectors.altus import SdxApi, AnalyticDbApi, DataEngApi, DataWarehouse2Api
  38. from notebook.connectors.base import Notebook, get_interpreter
  39. from notebook.models import Analytics
  40. from useradmin.models import User, Group
  41. from desktop import appmanager
  42. from desktop.auth.backend import is_admin
  43. from desktop.conf import ENABLE_CONNECTORS, ENABLE_GIST_PREVIEW, get_clusters, IS_K8S_ONLY, ENABLE_SHARING
  44. from desktop.lib.conf import BoundContainer, GLOBAL_CONFIG, is_anonymous
  45. from desktop.lib.django_util import JsonResponse, login_notrequired, render
  46. from desktop.lib.exceptions_renderable import PopupException
  47. from desktop.lib.export_csvxls import make_response
  48. from desktop.lib.i18n import smart_str, force_unicode
  49. from desktop.lib.paths import get_desktop_root
  50. from desktop.models import Document2, Document, Directory, FilesystemException, uuid_default, \
  51. UserPreferences, get_user_preferences, set_user_preferences, get_cluster_config, __paginate, _get_gist_document
  52. from desktop.views import serve_403_error
  53. if sys.version_info[0] > 2:
  54. from io import StringIO as string_io
  55. else:
  56. from StringIO import StringIO as string_io
  57. LOG = logging.getLogger(__name__)
  58. def api_error_handler(func):
  59. def decorator(*args, **kwargs):
  60. response = {}
  61. try:
  62. return func(*args, **kwargs)
  63. except Exception as e:
  64. LOG.exception('Error running %s' % func)
  65. response['status'] = -1
  66. response['message'] = force_unicode(str(e))
  67. finally:
  68. if response:
  69. return JsonResponse(response)
  70. return decorator
  71. @api_error_handler
  72. def get_config(request):
  73. config = get_cluster_config(request.user)
  74. config['hue_config']['is_admin'] = is_admin(request.user);
  75. config['clusters'] = list(get_clusters(request.user).values())
  76. config['documents'] = {
  77. 'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct())
  78. }
  79. config['status'] = 0
  80. return JsonResponse(config)
  81. @api_error_handler
  82. def get_hue_config(request):
  83. if not is_admin(request.user):
  84. raise PopupException(_('You must be a superuser.'))
  85. show_private = request.GET.get('private', False)
  86. app_modules = appmanager.DESKTOP_MODULES
  87. config_modules = GLOBAL_CONFIG.get().values()
  88. if ENABLE_CONNECTORS.get():
  89. app_modules = [app_module for app_module in app_modules if app_module.name == 'desktop']
  90. config_modules = [config_module for config_module in config_modules if config_module.config.key == 'desktop']
  91. apps = [{
  92. 'name': app.name,
  93. 'has_ui': app.menu_index != 999,
  94. 'display_name': app.display_name
  95. } for app in sorted(app_modules, key=lambda app: app.name)]
  96. def recurse_conf(modules):
  97. attrs = []
  98. for module in modules:
  99. if not show_private and module.config.private:
  100. continue
  101. conf = {
  102. 'help': module.config.help or _('No help available.'),
  103. 'key': module.config.key,
  104. 'is_anonymous': is_anonymous(module.config.key)
  105. }
  106. if isinstance(module, BoundContainer):
  107. conf['values'] = recurse_conf(module.get().values())
  108. else:
  109. conf['default'] = str(module.config.default)
  110. if 'password' in module.config.key:
  111. conf['value'] = '*' * 10
  112. elif sys.version_info[0] > 2:
  113. conf['value'] = str(module.get_raw())
  114. else:
  115. conf['value'] = str(module.get_raw()).decode('utf-8', 'replace')
  116. attrs.append(conf)
  117. return attrs
  118. return JsonResponse({
  119. 'config': sorted(recurse_conf(config_modules), key=lambda conf: conf.get('key')),
  120. 'conf_dir': os.path.realpath(os.getenv('HUE_CONF_DIR', get_desktop_root('conf'))),
  121. 'apps': apps
  122. })
  123. @api_error_handler
  124. def get_context_namespaces(request, interface):
  125. '''
  126. Namespaces are node cluster contexts (e.g. Hive + Ranger) that can be queried by computes.
  127. '''
  128. response = {}
  129. namespaces = []
  130. clusters = list(get_clusters(request.user).values())
  131. # Currently broken if not sent
  132. namespaces.extend([{
  133. 'id': cluster['id'],
  134. 'name': cluster['name'],
  135. 'status': 'CREATED',
  136. 'computes': [cluster]
  137. } for cluster in clusters if cluster.get('type') == 'direct'
  138. ])
  139. if interface == 'hive' or interface == 'impala' or interface == 'report':
  140. if get_cluster_config(request.user)['has_computes']:
  141. # Note: attaching computes to namespaces might be done via the frontend in the future
  142. if interface == 'impala':
  143. if IS_K8S_ONLY.get():
  144. adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
  145. else:
  146. adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters']
  147. for _cluster in adb_clusters: # Add "fake" namespace if needed
  148. if not _cluster.get('namespaceCrn'):
  149. _cluster['namespaceCrn'] = _cluster['crn']
  150. _cluster['id'] = _cluster['crn']
  151. _cluster['namespaceName'] = _cluster['clusterName']
  152. _cluster['name'] = _cluster['clusterName']
  153. _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '',
  154. else:
  155. adb_clusters = []
  156. if IS_K8S_ONLY.get():
  157. sdx_namespaces = []
  158. else:
  159. sdx_namespaces = SdxApi(request.user).list_namespaces()
  160. # Adding "fake" namespace for cluster without one
  161. sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])])
  162. namespaces.extend([{
  163. 'id': namespace.get('crn', 'None'),
  164. 'name': namespace.get('namespaceName'),
  165. 'status': namespace.get('status'),
  166. 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')]
  167. } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get()
  168. ])
  169. response[interface] = namespaces
  170. response['status'] = 0
  171. return JsonResponse(response)
  172. @api_error_handler
  173. def get_context_computes(request, interface):
  174. '''
  175. Some clusters like Snowball can have multiple computes for a certain languages (Hive, Impala...).
  176. '''
  177. response = {}
  178. computes = []
  179. clusters = list(get_clusters(request.user).values())
  180. if get_cluster_config(request.user)['has_computes']: # TODO: only based on interface selected?
  181. interpreter = get_interpreter(connector_type=interface, user=request.user)
  182. if interpreter['dialect'] == 'impala':
  183. # dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
  184. dw_clusters = [
  185. {'crn': 'c1', 'clusterName': 'c1', 'status': 'created', 'options': {'server_host': 'c1.gethue.com', 'server_port': 10000}},
  186. {'crn': 'c2', 'clusterName': 'c2', 'status': 'created', 'options': {'server_host': 'c2.gethue.com', 'server_port': 10000}},
  187. ]
  188. computes.extend([{
  189. 'id': cluster.get('crn'),
  190. 'name': cluster.get('clusterName'),
  191. 'status': cluster.get('status'),
  192. 'namespace': cluster.get('namespaceCrn', cluster.get('crn')),
  193. 'type': interpreter['dialect'],
  194. 'options': cluster['options'],
  195. } for cluster in dw_clusters]
  196. )
  197. else:
  198. # Currently broken if not sent
  199. computes.extend([{
  200. 'id': cluster['id'],
  201. 'name': cluster['name'],
  202. 'namespace': cluster['id'],
  203. 'interface': interface,
  204. 'type': cluster['type'],
  205. 'options': {}
  206. } for cluster in clusters if cluster.get('type') == 'direct'
  207. ])
  208. response[interface] = computes
  209. response['status'] = 0
  210. return JsonResponse(response)
  211. # Deprecated, not used.
  212. @api_error_handler
  213. def get_context_clusters(request, interface):
  214. response = {}
  215. clusters = []
  216. cluster_configs = list(get_clusters(request.user).values())
  217. for cluster in cluster_configs:
  218. cluster = {
  219. 'id': cluster.get('id'),
  220. 'name': cluster.get('name'),
  221. 'status': 'CREATED',
  222. 'environmentType': cluster.get('type'),
  223. 'serviceType': cluster.get('interface'),
  224. 'namespace': '',
  225. 'type': cluster.get('type')
  226. }
  227. if cluster.get('type') == 'altus':
  228. cluster['name'] = 'Altus DE'
  229. cluster['type'] = 'altus-de'
  230. clusters.append(cluster)
  231. cluster = cluster.copy()
  232. cluster['name'] = 'Altus Data Warehouse'
  233. cluster['type'] = 'altus-dw'
  234. elif cluster.get('type') == 'altusv2':
  235. cluster['name'] = 'Data Warehouse'
  236. cluster['type'] = 'altus-dw2'
  237. clusters.append(cluster)
  238. response[interface] = clusters
  239. response['status'] = 0
  240. return JsonResponse(response)
  241. @api_error_handler
  242. def search_documents(request):
  243. """
  244. Returns the directories and documents based on given params that are accessible by the current user
  245. Optional params:
  246. perms=<mode> - Controls whether to retrieve owned, shared, or both. Defaults to both.
  247. include_history=<bool> - Controls whether to retrieve history docs. Defaults to false.
  248. include_trashed=<bool> - Controls whether to retrieve docs in the trash. Defaults to true.
  249. include_managed=<bool> - Controls whether to retrieve docs generated by Hue. Defaults to false.
  250. flatten=<bool> - Controls whether to return documents in a flat list, or roll up documents to a common directory
  251. if possible. Defaults to true.
  252. page=<n> - Controls pagination. Defaults to 1.
  253. limit=<n> - Controls limit per page. Defaults to all.
  254. type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc).
  255. Defaults to all. Can appear multiple times.
  256. sort=<key> - Sort by the attribute <key>, which is one of: "name", "type", "owner", "last_modified"
  257. Accepts the form "-last_modified", which sorts in descending order.
  258. Defaults to "-last_modified".
  259. text=<frag> - Search for fragment "frag" in names and descriptions.
  260. """
  261. response = {
  262. 'documents': []
  263. }
  264. perms = request.GET.get('perms', 'both').lower()
  265. include_history = json.loads(request.GET.get('include_history', 'false'))
  266. include_trashed = json.loads(request.GET.get('include_trashed', 'true'))
  267. include_managed = json.loads(request.GET.get('include_managed', 'false'))
  268. flatten = json.loads(request.GET.get('flatten', 'true'))
  269. if perms not in ['owned', 'shared', 'both']:
  270. raise PopupException(_('Invalid value for perms, acceptable values are: owned, shared, both.'))
  271. documents = Document2.objects.documents(
  272. user=request.user,
  273. perms=perms,
  274. include_history=include_history,
  275. include_trashed=include_trashed,
  276. include_managed=include_managed
  277. )
  278. # Refine results
  279. response.update(_filter_documents(request, queryset=documents, flatten=flatten))
  280. # Paginate
  281. response.update(_paginate(request, queryset=response['documents']))
  282. # Serialize results
  283. response['documents'] = [doc.to_dict() for doc in response.get('documents', [])]
  284. return JsonResponse(response)
  285. def _search(user, perms='both', include_history=False, include_trashed=False, include_managed=False, search_text=None, limit=25):
  286. response = {
  287. 'documents': []
  288. }
  289. documents = Document2.objects.documents(
  290. user=user,
  291. perms=perms,
  292. include_history=include_history,
  293. include_trashed=include_trashed,
  294. include_managed=include_managed
  295. )
  296. type_filters = None
  297. sort = '-last_modified'
  298. search_text = search_text
  299. flatten = True
  300. page = 1
  301. # Refine results
  302. response.update(__filter_documents(type_filters, sort, search_text, queryset=documents, flatten=flatten))
  303. # Paginate
  304. response.update(__paginate(page, limit, queryset=response['documents']))
  305. return response
  306. @api_error_handler
  307. def get_document(request):
  308. """
  309. Returns the document or directory found for the given uuid or path and current user.
  310. If a directory is found, return any children documents too.
  311. Optional params:
  312. page=<n> - Controls pagination. Defaults to 1.
  313. limit=<n> - Controls limit per page. Defaults to all.
  314. type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc). Default to all.
  315. sort=<key> - Sort by the attribute <key>, which is one of:
  316. "name", "type", "owner", "last_modified"
  317. Accepts the form "-last_modified", which sorts in descending order.
  318. Default to "-last_modified".
  319. text=<frag> - Search for fragment "frag" in names and descriptions.
  320. data=<false|true> - Return all the data of the document. Default to false.
  321. dependencies=<false|true> - Return all the dependencies and dependents of the document. Default to false.
  322. """
  323. path = request.GET.get('path', '/')
  324. uuid = request.GET.get('uuid')
  325. uuids = request.GET.get('uuids')
  326. with_data = request.GET.get('data', 'false').lower() == 'true'
  327. with_dependencies = request.GET.get('dependencies', 'false').lower() == 'true'
  328. if uuids:
  329. response = {
  330. 'data_list': [_get_document_helper(request, uuid, with_data, with_dependencies, path) for uuid in uuids.split(',')],
  331. 'status': 0
  332. }
  333. else:
  334. response = _get_document_helper(request, uuid, with_data, with_dependencies, path)
  335. return JsonResponse(response)
  336. def _get_document_helper(request, uuid, with_data, with_dependencies, path):
  337. if uuid:
  338. if uuid.isdigit():
  339. document = Document2.objects.document(user=request.user, doc_id=uuid)
  340. else:
  341. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  342. else: # Find by path
  343. document = Document2.objects.get_by_path(user=request.user, path=path)
  344. response = {
  345. 'document': document.to_dict(),
  346. 'parent': document.parent_directory.to_dict() if document.parent_directory else None,
  347. 'children': [],
  348. 'dependencies': [],
  349. 'dependents': [],
  350. 'data': '',
  351. 'status': 0
  352. }
  353. response['user_perms'] = {
  354. 'can_read': document.can_read(request.user),
  355. 'can_write': document.can_write(request.user)
  356. }
  357. if with_data:
  358. data = json.loads(document.data)
  359. # Upgrade session properties for Hive and Impala
  360. if document.type.startswith('query'):
  361. from notebook.models import upgrade_session_properties
  362. notebook = Notebook(document=document)
  363. notebook = upgrade_session_properties(request, notebook)
  364. data = json.loads(notebook.data)
  365. if document.type == 'query-pig': # Import correctly from before Hue 4.0
  366. properties = data['snippets'][0]['properties']
  367. if 'hadoopProperties' not in properties:
  368. properties['hadoopProperties'] = []
  369. if 'parameters' not in properties:
  370. properties['parameters'] = []
  371. if 'resources' not in properties:
  372. properties['resources'] = []
  373. if data.get('uuid') != document.uuid: # Old format < 3.11
  374. data['uuid'] = document.uuid
  375. response['data'] = data
  376. if with_dependencies:
  377. response['dependencies'] = [dependency.to_dict() for dependency in document.dependencies.all()]
  378. response['dependents'] = [dependent.to_dict() for dependent in document.dependents.exclude(is_history=True).all()]
  379. # Get children documents if this is a directory
  380. if document.is_directory:
  381. directory = Directory.objects.get(id=document.id)
  382. # If this is the user's home directory, fetch shared docs too
  383. if document.is_home_directory:
  384. children = directory.get_children_and_shared_documents(user=request.user)
  385. response.update(_filter_documents(request, queryset=children, flatten=True))
  386. else:
  387. children = directory.get_children_documents()
  388. response.update(_filter_documents(request, queryset=children, flatten=False))
  389. # Paginate and serialize Results
  390. if 'documents' in response:
  391. response.update(_paginate(request, queryset=response['documents']))
  392. # Rename documents to children
  393. response['children'] = response.pop('documents')
  394. response['children'] = [doc.to_dict() for doc in response['children']]
  395. return response
  396. @api_error_handler
  397. def open_document(request):
  398. doc_id = request.GET.get('id')
  399. if doc_id.isdigit():
  400. document = Document2.objects.document(user=request.user, doc_id=doc_id)
  401. else:
  402. document = Document2.objects.get_by_uuid(user=request.user, uuid=doc_id)
  403. return redirect(document.get_absolute_url())
  404. @api_error_handler
  405. @require_POST
  406. def move_document(request):
  407. source_doc_uuid = json.loads(request.POST.get('source_doc_uuid'))
  408. destination_doc_uuid = json.loads(request.POST.get('destination_doc_uuid'))
  409. if not source_doc_uuid or not destination_doc_uuid:
  410. raise PopupException(_('move_document requires source_doc_uuid and destination_doc_uuid'))
  411. source = Document2.objects.get_by_uuid(user=request.user, uuid=source_doc_uuid, perm_type='write')
  412. destination = Directory.objects.get_by_uuid(user=request.user, uuid=destination_doc_uuid, perm_type='write')
  413. doc = source.move(destination, request.user)
  414. return JsonResponse({
  415. 'status': 0,
  416. 'document': doc.to_dict()
  417. })
  418. @api_error_handler
  419. @require_POST
  420. def create_directory(request):
  421. parent_uuid = json.loads(request.POST.get('parent_uuid'))
  422. name = json.loads(request.POST.get('name'))
  423. if not parent_uuid or not name:
  424. raise PopupException(_('create_directory requires parent_uuid and name'))
  425. parent_dir = Directory.objects.get_by_uuid(user=request.user, uuid=parent_uuid, perm_type='write')
  426. directory = Directory.objects.create(name=name, owner=request.user, parent_directory=parent_dir)
  427. return JsonResponse({
  428. 'status': 0,
  429. 'directory': directory.to_dict()
  430. })
  431. @api_error_handler
  432. @require_POST
  433. def update_document(request):
  434. uuid = json.loads(request.POST.get('uuid'))
  435. if not uuid:
  436. raise PopupException(_('update_document requires uuid'))
  437. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type='write')
  438. whitelisted_attrs = ['name', 'description']
  439. for attr in whitelisted_attrs:
  440. if request.POST.get(attr):
  441. setattr(document, attr, request.POST.get(attr))
  442. document.save(update_fields=whitelisted_attrs)
  443. return JsonResponse({
  444. 'status': 0,
  445. 'document': document.to_dict()
  446. })
  447. @api_error_handler
  448. @require_POST
  449. def delete_document(request):
  450. """
  451. Accepts a uuid and optional skip_trash parameter
  452. (Default) skip_trash=false, flags a document as trashed
  453. skip_trash=true, deletes it permanently along with any history dependencies
  454. If directory and skip_trash=false, all dependencies will also be flagged as trash
  455. If directory and skip_trash=true, directory must be empty (no dependencies)
  456. """
  457. uuid = json.loads(request.POST.get('uuid'))
  458. skip_trash = json.loads(request.POST.get('skip_trash', 'false'))
  459. if not uuid:
  460. raise PopupException(_('delete_document requires uuid'))
  461. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type='write')
  462. if skip_trash:
  463. document.delete()
  464. else:
  465. document.trash()
  466. return JsonResponse({
  467. 'status': 0,
  468. })
  469. @api_error_handler
  470. @require_POST
  471. def copy_document(request):
  472. uuid = json.loads(request.POST.get('uuid', '""'))
  473. if not uuid:
  474. raise PopupException(_('copy_document requires uuid'))
  475. # Document2 and Document model objects are linked and both are saved when saving
  476. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  477. # Document model object
  478. document1 = document.doc.get()
  479. if document.type == 'directory':
  480. raise PopupException(_('Directory copy is not supported'))
  481. name = document.name + '-copy'
  482. # Make the copy of the Document2 model object
  483. copy_document = document.copy(name=name, owner=request.user)
  484. # Make the copy of Document model object too
  485. document1.copy(content_object=copy_document, name=name, owner=request.user)
  486. # Import workspace for all oozie jobs
  487. if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
  488. from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace
  489. # Update the name field in the json 'data' field
  490. if document.type == 'oozie-workflow2':
  491. workflow = Workflow(document=document)
  492. workflow.update_name(name)
  493. workflow.update_uuid(copy_document.uuid)
  494. _import_workspace(request.fs, request.user, workflow)
  495. copy_document.update_data({'workflow': workflow.get_data()['workflow']})
  496. copy_document.save()
  497. if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
  498. if document.type == 'oozie-bundle2':
  499. bundle_or_coordinator = Bundle(document=document)
  500. else:
  501. bundle_or_coordinator = Coordinator(document=document)
  502. json_data = bundle_or_coordinator.get_data_for_json()
  503. json_data['name'] = name
  504. json_data['uuid'] = copy_document.uuid
  505. copy_document.update_data(json_data)
  506. copy_document.save()
  507. _import_workspace(request.fs, request.user, bundle_or_coordinator)
  508. elif document.type == 'search-dashboard':
  509. from dashboard.models import Collection2
  510. collection = Collection2(request.user, document=document)
  511. collection.data['collection']['label'] = name
  512. collection.data['collection']['uuid'] = copy_document.uuid
  513. copy_document.update_data({'collection': collection.data['collection']})
  514. copy_document.save()
  515. # Keep the document and data in sync
  516. else:
  517. copy_data = copy_document.data_dict
  518. if 'name' in copy_data:
  519. copy_data['name'] = name
  520. if 'uuid' in copy_data:
  521. copy_data['uuid'] = copy_document.uuid
  522. copy_document.update_data(copy_data)
  523. copy_document.save()
  524. return JsonResponse({
  525. 'status': 0,
  526. 'document': copy_document.to_dict()
  527. })
  528. @api_error_handler
  529. @require_POST
  530. def restore_document(request):
  531. """
  532. Accepts a uuid
  533. Restores the document to /home
  534. """
  535. uuids = json.loads(request.POST.get('uuids'))
  536. if not uuids:
  537. raise PopupException(_('restore_document requires comma separated uuids'))
  538. for uuid in uuids.split(','):
  539. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type='write')
  540. document.restore()
  541. return JsonResponse({
  542. 'status': 0,
  543. })
  544. @api_error_handler
  545. @require_POST
  546. def share_document(request):
  547. """
  548. Set who else or which other group can interact with the document.
  549. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  550. """
  551. if not is_admin(request.user) and not ENABLE_SHARING.get():
  552. return serve_403_error(request)
  553. uuid = request.POST.get('uuid')
  554. perms_dict = request.POST.get('data')
  555. if not uuid or not perms_dict:
  556. raise PopupException(_('share_document requires uuid and perms_dict'))
  557. else:
  558. perms_dict = json.loads(perms_dict)
  559. uuid = json.loads(uuid)
  560. doc = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  561. for name, perm in perms_dict.items():
  562. users = groups = None
  563. if perm.get('user_ids'):
  564. users = User.objects.in_bulk(perm.get('user_ids'))
  565. else:
  566. users = []
  567. if perm.get('group_ids'):
  568. groups = Group.objects.in_bulk(perm.get('group_ids'))
  569. else:
  570. groups = []
  571. doc = doc.share(request.user, name=name, users=users, groups=groups)
  572. return JsonResponse({
  573. 'status': 0,
  574. 'document': doc.to_dict()
  575. })
  576. @api_error_handler
  577. @require_POST
  578. def share_document_link(request):
  579. """
  580. Globally activate of de-activate access to a document for logged-in users.
  581. Example of input: {"uuid": "xxxx", "perm": "read" / "write" / "off"}
  582. """
  583. if not is_admin(request.user) and not ENABLE_SHARING.get():
  584. return serve_403_error(request)
  585. uuid = request.POST.get('uuid')
  586. perm = request.POST.get('perm')
  587. if not uuid or not perm:
  588. raise PopupException(_('share_document_link requires uuid and permission data'))
  589. else:
  590. uuid = json.loads(uuid)
  591. perm = json.loads(perm)
  592. doc = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  593. doc = doc.share_link(request.user, perm=perm)
  594. return JsonResponse({
  595. 'status': 0,
  596. 'document': doc.to_dict()
  597. })
  598. @ensure_csrf_cookie
  599. def export_documents(request):
  600. if request.GET.get('documents'):
  601. selection = json.loads(request.GET.get('documents'))
  602. else:
  603. selection = json.loads(request.POST.get('documents'))
  604. include_history = request.GET.get('history', 'false') == 'true'
  605. # Only export documents the user has permissions to read
  606. docs = Document2.objects.documents(user=request.user, perms='both', include_history=True, include_trashed=True).\
  607. filter(id__in=selection).order_by('-id')
  608. # Add any dependencies to the set of exported documents
  609. export_doc_set = _get_dependencies(docs, include_history=include_history)
  610. # For directories, add any children docs to the set of exported documents
  611. export_doc_set.update(_get_dependencies(docs, deps_mode=False))
  612. # Get PKs of documents to export
  613. doc_ids = [doc.pk for doc in export_doc_set]
  614. num_docs = len(doc_ids)
  615. if len(selection) == 1 and num_docs >= len(selection) and docs[0].name:
  616. filename = docs[0].name
  617. else:
  618. filename = 'hue-documents-%s-(%s)' % (datetime.today().strftime('%Y-%m-%d'), num_docs)
  619. f = string_io()
  620. if doc_ids:
  621. doc_ids = ','.join(map(str, doc_ids))
  622. management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_foreign_keys=True, verbosity=2, stdout=f)
  623. if request.GET.get('format') == 'json':
  624. return JsonResponse(f.getvalue(), safe=False)
  625. elif request.GET.get('format') == 'zip':
  626. zfile = zipfile.ZipFile(f, 'w')
  627. zfile.writestr("hue.json", f.getvalue())
  628. for doc in docs:
  629. if doc.type == 'notebook':
  630. try:
  631. from spark.models import Notebook
  632. zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
  633. except Exception as e:
  634. LOG.exception(e)
  635. zfile.close()
  636. response = HttpResponse(content_type="application/zip")
  637. response["Content-Length"] = len(f.getvalue())
  638. response['Content-Disposition'] = b'attachment; filename="%s".zip' % filename
  639. response.write(f.getvalue())
  640. return response
  641. else:
  642. return make_response(f.getvalue(), 'json', filename)
  643. @ensure_csrf_cookie
  644. def import_documents(request):
  645. def is_reserved_directory(doc):
  646. return doc['fields']['type'] == 'directory' and doc['fields']['name'] in (Document2.HOME_DIR, Document2.TRASH_DIR)
  647. try:
  648. if request.FILES.get('documents'):
  649. documents = request.FILES['documents'].read()
  650. else:
  651. documents = json.loads(request.POST.get('documents'))
  652. documents = json.loads(documents)
  653. except ValueError as e:
  654. raise PopupException(_('Failed to import documents, the file does not contain valid JSON.'))
  655. # Validate documents
  656. if not _is_import_valid(documents):
  657. raise PopupException(_('Failed to import documents, the file does not contain the expected JSON schema for Hue documents.'))
  658. docs = []
  659. uuids_map = dict((doc['fields']['uuid'], None) for doc in documents if not is_reserved_directory(doc))
  660. for doc in documents:
  661. # Filter docs to import, ignoring reserved directories (home and Trash) and history docs
  662. if not is_reserved_directory(doc):
  663. # Remove any deprecated fields
  664. if 'tags' in doc['fields']:
  665. doc['fields'].pop('tags')
  666. # If doc is not owned by current user, make a copy of the document with current user as owner
  667. if doc['fields']['owner'][0] != request.user.username:
  668. doc = _copy_document_with_owner(doc, request.user, uuids_map)
  669. else: # Update existing doc or create new
  670. doc = _create_or_update_document_with_owner(doc, request.user, uuids_map)
  671. # For oozie docs replace dependent uuids with the newly created ones
  672. if doc['fields']['type'].startswith('oozie-'):
  673. doc = _update_imported_oozie_document(doc, uuids_map)
  674. # If the doc contains any history dependencies, ignore them
  675. # NOTE: this assumes that each dependency is exported as an array using the natural PK [uuid, version, is_history]
  676. deps_minus_history = [dep for dep in doc['fields'].get('dependencies', []) if len(dep) >= 3 and not dep[2]]
  677. doc['fields']['dependencies'] = deps_minus_history
  678. # Replace illegal characters
  679. if '/' in doc['fields']['name']:
  680. new_name = doc['fields']['name'].replace('/', '-')
  681. LOG.warn("Found illegal slash in document named: %s, renaming to: %s." % (doc['fields']['name'], new_name))
  682. doc['fields']['name'] = new_name
  683. # Set last modified date to now
  684. doc['fields']['last_modified'] = datetime.now().replace(microsecond=0).isoformat()
  685. docs.append(doc)
  686. f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
  687. f.write(json.dumps(docs))
  688. f.flush()
  689. stdout = string_io()
  690. try:
  691. with transaction.atomic(): # We wrap both commands to commit loaddata & sync
  692. management.call_command('loaddata', f.name, verbosity=3, traceback=True, stdout=stdout, commit=False) # We need to use commit=False because commit=True will close the connection and make Document.objects.sync fail.
  693. Document.objects.sync()
  694. if request.POST.get('redirect'):
  695. return redirect(request.POST.get('redirect'))
  696. else:
  697. return JsonResponse({
  698. 'status': 0,
  699. 'message': stdout.getvalue(),
  700. 'count': len(documents),
  701. 'created_count': len([doc for doc in documents if doc['pk'] is None]),
  702. 'updated_count': len([doc for doc in documents if doc['pk'] is not None]),
  703. 'username': request.user.username,
  704. 'documents': [
  705. dict([
  706. ('name', doc['fields']['name']),
  707. ('uuid', doc['fields']['uuid']),
  708. ('type', doc['fields']['type']),
  709. ('owner', doc['fields']['owner'][0])
  710. ]) for doc in docs]
  711. })
  712. except Exception as e:
  713. LOG.error('Failed to run loaddata command in import_documents:\n %s' % stdout.getvalue())
  714. return JsonResponse({'status': -1, 'message': smart_str(e)})
  715. finally:
  716. stdout.close()
  717. def _update_imported_oozie_document(doc, uuids_map):
  718. for key, value in uuids_map.items():
  719. if value:
  720. doc['fields']['data'] = doc['fields']['data'].replace(key, value)
  721. return doc
  722. def user_preferences(request, key=None):
  723. response = {'status': 0, 'data': {}}
  724. if request.method != "POST":
  725. response['data'] = get_user_preferences(request.user, key)
  726. else:
  727. if "set" in request.POST:
  728. x = set_user_preferences(request.user, key, request.POST["set"])
  729. response['data'] = {key: x.value}
  730. elif "delete" in request.POST:
  731. try:
  732. x = UserPreferences.objects.get(user=request.user, key=key)
  733. x.delete()
  734. except UserPreferences.DoesNotExist:
  735. pass
  736. return JsonResponse(response)
  737. @api_error_handler
  738. def gist_create(request):
  739. '''
  740. Only supporting Editor App currently.
  741. '''
  742. response = {'status': 0}
  743. statement = request.POST.get('statement', '')
  744. gist_type = request.POST.get('doc_type', 'hive')
  745. name = request.POST.get('name', '')
  746. description = request.POST.get('description', '')
  747. if not name:
  748. name = _('%s Query') % gist_type.capitalize()
  749. statement_raw = statement
  750. if not statement.strip().startswith('--'):
  751. statement = '-- Created by %s\n\n%s' % (request.user.get_full_name() or request.user.username, statement)
  752. gist_doc = Document2.objects.create(
  753. name=name,
  754. type='gist',
  755. owner=request.user,
  756. data=json.dumps({'statement': statement, 'statement_raw': statement_raw}),
  757. extra=gist_type,
  758. parent_directory=Document2.objects.get_gist_directory(request.user)
  759. )
  760. response['id'] = gist_doc.id
  761. response['uuid'] = gist_doc.uuid
  762. response['link'] = '%(scheme)s://%(host)s/hue/gist?uuid=%(uuid)s' % {
  763. 'scheme': 'https' if request.is_secure() else 'http',
  764. 'host': request.get_host(),
  765. 'uuid': gist_doc.uuid,
  766. }
  767. return JsonResponse(response)
  768. @login_notrequired
  769. @api_error_handler
  770. def gist_get(request):
  771. gist_uuid = request.GET.get('uuid')
  772. gist_doc = _get_gist_document(uuid=gist_uuid)
  773. if ENABLE_GIST_PREVIEW.get() and 'Slackbot-LinkExpanding' in request.META.get('HTTP_USER_AGENT', ''):
  774. statement = json.loads(gist_doc.data)['statement_raw']
  775. return render(
  776. 'unfurl_link.mako',
  777. request, {
  778. 'title': _('SQL gist from %s') % (gist_doc.owner.get_full_name() or gist_doc.owner.username),
  779. 'description': statement if len(statement) < 150 else (statement[:150] + '...'),
  780. 'image_link': None
  781. }
  782. )
  783. else:
  784. return redirect('/hue/editor?gist=%(uuid)s&type=%(type)s' % {
  785. 'uuid': gist_doc.uuid,
  786. 'type': gist_doc.extra
  787. })
  788. def search_entities(request):
  789. sources = json.loads(request.POST.get('sources')) or []
  790. if 'documents' in sources:
  791. search_text = json.loads(request.POST.get('query_s', ''))
  792. entities = _search(user=request.user, search_text=search_text)
  793. response = {
  794. 'entities': [{
  795. 'hue_name': _highlight(search_text, escape(e.name)),
  796. 'hue_description': _highlight(search_text, escape(e.description)),
  797. 'type': 'HUE',
  798. 'last_modified': e.last_modified,
  799. 'owner': escape(e.owner),
  800. 'doc_type': escape(e.type),
  801. 'originalName': escape(e.name),
  802. 'link': e.get_absolute_url()
  803. } for e in entities['documents']
  804. ],
  805. 'count': len(entities['documents']),
  806. 'status': 0
  807. }
  808. return JsonResponse(response)
  809. else:
  810. if has_catalog(request.user):
  811. return metadata_search_entities(request)
  812. else:
  813. return JsonResponse({'status': 1, 'message': _('Navigator not enabled')})
  814. def search_entities_interactive(request):
  815. sources = json.loads(request.POST.get('sources')) or []
  816. if 'documents' in sources:
  817. search_text = json.loads(request.POST.get('query_s', ''))
  818. limit = int(request.POST.get('limit', 25))
  819. entities = _search(user=request.user, search_text=search_text, limit=limit)
  820. response = {
  821. 'results': [{
  822. 'hue_name': _highlight(search_text, escape(e.name)),
  823. 'hue_description': _highlight(search_text, escape(e.description)),
  824. 'link': e.get_absolute_url(),
  825. 'doc_type': escape(e.type),
  826. 'last_modified': e.last_modified,
  827. 'owner': escape(e.owner),
  828. 'type': 'HUE',
  829. 'uuid': e.uuid,
  830. 'parentUuid': e.parent_directory.uuid,
  831. 'originalName': escape(e.name)
  832. }
  833. for e in entities['documents']
  834. ],
  835. 'count': len(entities['documents']),
  836. 'status': 0
  837. }
  838. return JsonResponse(response)
  839. else:
  840. if has_catalog(request.user):
  841. return metadata_search_entities_interactive(request)
  842. else:
  843. return JsonResponse({'status': 1, 'message': _('Navigator not enabled')})
  844. def _is_import_valid(documents):
  845. """
  846. Validates the JSON file to be imported for schema correctness
  847. :param documents: object loaded from JSON file
  848. :return: True if schema seems valid, False otherwise
  849. """
  850. return isinstance(documents, list) and \
  851. all(isinstance(d, dict) for d in documents) and \
  852. all(all(k in d for k in ('pk', 'model', 'fields')) for d in documents) and \
  853. all(all(k in d['fields'] for k in ('uuid', 'owner')) for d in documents)
  854. def _get_dependencies(documents, deps_mode=True, include_history=False):
  855. """
  856. Given a list of Document2 objects, perform a depth-first search and return a set of documents with all
  857. dependencies (excluding history docs) included
  858. :param doc_set: set of Document2 objects to include
  859. :param deps_mode: traverse dependencies relationship, otherwise traverse children relationship
  860. """
  861. doc_set = set()
  862. for doc in documents:
  863. stack = [doc]
  864. while stack:
  865. curr_doc = stack.pop()
  866. if curr_doc not in doc_set and (include_history or not curr_doc.is_history):
  867. doc_set.add(curr_doc)
  868. if deps_mode:
  869. deps_set = set(curr_doc.dependencies.all())
  870. else:
  871. deps_set = set(curr_doc.children.all())
  872. stack.extend(deps_set - doc_set)
  873. return doc_set
  874. def _copy_document_with_owner(doc, owner, uuids_map):
  875. home_dir = Directory.objects.get_home_directory(owner)
  876. doc['fields']['owner'] = [owner.username]
  877. doc['pk'] = None
  878. doc['fields']['version'] = 1
  879. # Retrieve from the import_uuids_map if it's already been reassigned, or assign a new UUID and map it
  880. old_uuid = doc['fields']['uuid']
  881. if uuids_map[old_uuid] is None:
  882. uuids_map[old_uuid] = uuid_default()
  883. doc['fields']['uuid'] = uuids_map[old_uuid]
  884. # Update UUID in data if needed
  885. if 'data' in doc['fields']:
  886. data = json.loads(doc['fields']['data'])
  887. if 'uuid' in data:
  888. data['uuid'] = uuids_map[old_uuid]
  889. doc['fields']['data'] = json.dumps(data)
  890. # Remap parent directory if needed
  891. parent_uuid = None
  892. if doc['fields'].get('parent_directory'):
  893. parent_uuid = doc['fields']['parent_directory'][0]
  894. if parent_uuid is not None and parent_uuid in list(uuids_map.keys()):
  895. if uuids_map[parent_uuid] is None:
  896. uuids_map[parent_uuid] = uuid_default()
  897. doc['fields']['parent_directory'] = [uuids_map[parent_uuid], 1, False]
  898. else:
  899. if parent_uuid is not None:
  900. LOG.warn('Could not find parent directory with UUID: %s in JSON import, will set parent to home directory' %
  901. parent_uuid)
  902. doc['fields']['parent_directory'] = [home_dir.uuid, home_dir.version, home_dir.is_history]
  903. # Remap dependencies if needed
  904. idx = 0
  905. for dep_uuid, dep_version, dep_is_history in doc['fields']['dependencies']:
  906. if dep_uuid not in list(uuids_map.keys()):
  907. LOG.warn('Could not find dependency UUID: %s in JSON import, may cause integrity errors if not found.' % dep_uuid)
  908. else:
  909. if uuids_map[dep_uuid] is None:
  910. uuids_map[dep_uuid] = uuid_default()
  911. doc['fields']['dependencies'][idx][0] = uuids_map[dep_uuid]
  912. idx += 1
  913. return doc
  914. def _create_or_update_document_with_owner(doc, owner, uuids_map):
  915. home_dir = Directory.objects.get_home_directory(owner)
  916. create_new = False
  917. try:
  918. owned_docs = Document2.objects.filter(uuid=doc['fields']['uuid'], owner=owner).order_by('-last_modified')
  919. if owned_docs.exists():
  920. existing_doc = owned_docs[0]
  921. doc['pk'] = existing_doc.pk
  922. else:
  923. create_new = True
  924. except FilesystemException as e:
  925. create_new = True
  926. if create_new:
  927. LOG.warn('Could not find document with UUID: %s, will create a new document on import.', doc['fields']['uuid'])
  928. doc['pk'] = None
  929. doc['fields']['version'] = 1
  930. # Verify that parent exists, log warning and set parent to user's home directory if not found
  931. if doc['fields']['parent_directory']:
  932. uuid, version, is_history = doc['fields']['parent_directory']
  933. if uuid not in list(uuids_map.keys()) and \
  934. not Document2.objects.filter(uuid=uuid, version=version, is_history=is_history).exists():
  935. LOG.warn('Could not find parent document with UUID: %s, will set parent to home directory' % uuid)
  936. doc['fields']['parent_directory'] = [home_dir.uuid, home_dir.version, home_dir.is_history]
  937. # Verify that dependencies exist, raise critical error if any dependency not found
  938. # Ignore history dependencies
  939. if doc['fields']['dependencies']:
  940. history_deps_list = []
  941. for index, (uuid, version, is_history) in enumerate(doc['fields']['dependencies']):
  942. if not uuid in list(uuids_map.keys()) and not is_history and \
  943. not Document2.objects.filter(uuid=uuid, version=version).exists():
  944. raise PopupException(_('Cannot import document, dependency with UUID: %s not found.') % uuid)
  945. elif is_history:
  946. history_deps_list.insert(0, index) # Insert in decreasing order to facilitate delete
  947. LOG.warn('History dependency with UUID: %s ignored while importing document %s' % (uuid, doc['fields']['name']))
  948. # Delete history dependencies not found in the DB
  949. for index in history_deps_list:
  950. del doc['fields']['dependencies'][index]
  951. return doc
  952. def _filter_documents(request, queryset, flatten=True):
  953. """
  954. Given optional querystring params extracted from the request, filter the given queryset of documents and return a
  955. dictionary with the refined queryset and filter params
  956. :param request: request object with params
  957. :param queryset: Document2 queryset
  958. :param flatten: Return all results in a flat list if true, otherwise roll up to common directory
  959. """
  960. type_filters = request.GET.getlist('type', None)
  961. sort = request.GET.get('sort', '-last_modified')
  962. search_text = request.GET.get('text', None)
  963. return __filter_documents(type_filters, sort, search_text, queryset, flatten)
  964. def __filter_documents(type_filters, sort, search_text, queryset, flatten=True):
  965. documents = queryset.search_documents(
  966. types=type_filters,
  967. search_text=search_text,
  968. order_by=sort
  969. )
  970. # Roll up documents to common directory
  971. if not flatten:
  972. documents = documents.exclude(parent_directory__in=documents)
  973. count = documents.count()
  974. return {
  975. 'documents': documents,
  976. 'count': count,
  977. 'types': type_filters,
  978. 'text': search_text,
  979. 'sort': sort
  980. }
  981. def _paginate(request, queryset):
  982. """
  983. Given optional querystring params extracted from the request, slice the given queryset of documents for the given page
  984. and limit, and return the updated queryset along with pagination params used.
  985. :param request: request object with params
  986. :param queryset: queryset
  987. """
  988. page = int(request.GET.get('page', 1))
  989. limit = int(request.GET.get('limit', 0))
  990. return __paginate(page, limit, queryset)