api2.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. import json
  19. import tempfile
  20. import StringIO
  21. import zipfile
  22. from datetime import datetime
  23. from django.contrib.auth.models import Group, User
  24. from django.core import management
  25. from django.http import HttpResponse
  26. from django.shortcuts import redirect
  27. from django.utils.translation import ugettext as _
  28. from django.views.decorators.csrf import ensure_csrf_cookie
  29. from django.views.decorators.http import require_POST
  30. from desktop.lib.django_util import JsonResponse
  31. from desktop.lib.exceptions_renderable import PopupException
  32. from desktop.lib.export_csvxls import make_response
  33. from desktop.lib.i18n import smart_str, force_unicode
  34. from desktop.models import Document2, Document, Directory, FilesystemException, uuid_default
  35. from notebook.connectors.base import Notebook
  36. from notebook.views import upgrade_session_properties
  37. LOG = logging.getLogger(__name__)
  38. def api_error_handler(func):
  39. def decorator(*args, **kwargs):
  40. response = {}
  41. try:
  42. return func(*args, **kwargs)
  43. except Exception, e:
  44. LOG.exception('Error running %s' % func)
  45. response['status'] = -1
  46. response['message'] = force_unicode(str(e))
  47. finally:
  48. if response:
  49. return JsonResponse(response)
  50. return decorator
  51. @api_error_handler
  52. def search_documents(request):
  53. """
  54. Returns the directories and documents based on given params that are accessible by the current user
  55. Optional params:
  56. perms=<mode> - Controls whether to retrieve owned, shared, or both. Defaults to both.
  57. include_history=<bool> - Controls whether to retrieve history docs. Defaults to false.
  58. flatten=<bool> - Controls whether to return documents in a flat list, or roll up documents to a common directory
  59. if possible. Defaults to true.
  60. page=<n> - Controls pagination. Defaults to 1.
  61. limit=<n> - Controls limit per page. Defaults to all.
  62. type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc).
  63. Defaults to all.
  64. sort=<key> - Sort by the attribute <key>, which is one of: "name", "type", "owner", "last_modified"
  65. Accepts the form "-last_modified", which sorts in descending order.
  66. Defaults to "-last_modified".
  67. text=<frag> - Search for fragment "frag" in names and descriptions.
  68. """
  69. response = {
  70. 'documents': []
  71. }
  72. perms = request.GET.get('perms', 'both').lower()
  73. include_history = json.loads(request.GET.get('include_history', 'false'))
  74. include_trashed = json.loads(request.GET.get('include_trashed', 'true'))
  75. flatten = json.loads(request.GET.get('flatten', 'true'))
  76. if perms not in ['owned', 'shared', 'both']:
  77. raise PopupException(_('Invalid value for perms, acceptable values are: owned, shared, both.'))
  78. documents = Document2.objects.documents(
  79. user=request.user,
  80. perms=perms,
  81. include_history=include_history,
  82. include_trashed=include_trashed
  83. )
  84. # Refine results
  85. response.update(_filter_documents(request, queryset=documents, flatten=flatten))
  86. # Paginate
  87. response.update(_paginate(request, queryset=response['documents']))
  88. # Serialize results
  89. response['documents'] = [doc.to_dict() for doc in response.get('documents', [])]
  90. return JsonResponse(response)
  91. @api_error_handler
  92. def get_document(request):
  93. """
  94. Returns the document or directory found for the given uuid or path and current user.
  95. If a directory is found, return any children documents too.
  96. Optional params:
  97. page=<n> - Controls pagination. Defaults to 1.
  98. limit=<n> - Controls limit per page. Defaults to all.
  99. type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc). Default to all.
  100. sort=<key> - Sort by the attribute <key>, which is one of:
  101. "name", "type", "owner", "last_modified"
  102. Accepts the form "-last_modified", which sorts in descending order.
  103. Default to "-last_modified".
  104. text=<frag> - Search for fragment "frag" in names and descriptions.
  105. data=<false|true> - Return all the data of the document. Default to false.
  106. dependencies=<false|true> - Return all the dependencies and dependents of the document. Default to false.
  107. """
  108. path = request.GET.get('path', '/')
  109. uuid = request.GET.get('uuid')
  110. with_data = request.GET.get('data', 'false').lower() == 'true'
  111. with_dependencies = request.GET.get('dependencies', 'false').lower() == 'true'
  112. if uuid:
  113. if uuid.isdigit():
  114. document = Document2.objects.document(user=request.user, doc_id=uuid)
  115. else:
  116. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  117. else: # Find by path
  118. document = Document2.objects.get_by_path(user=request.user, path=path)
  119. response = {
  120. 'document': document.to_dict(),
  121. 'parent': document.parent_directory.to_dict() if document.parent_directory else None,
  122. 'children': [],
  123. 'dependencies': [],
  124. 'dependents': [],
  125. 'data': ''
  126. }
  127. response['user_perms'] = {
  128. 'can_read': document.can_read(request.user),
  129. 'can_write': document.can_write(request.user)
  130. }
  131. if with_data:
  132. data = json.loads(document.data)
  133. # Upgrade session properties for Hive and Impala
  134. if document.type.startswith('query'):
  135. notebook = Notebook(document=document)
  136. notebook = upgrade_session_properties(request, notebook)
  137. data = json.loads(notebook.data)
  138. response['data'] = data
  139. if with_dependencies:
  140. response['dependencies'] = [dependency.to_dict() for dependency in document.dependencies.all()]
  141. response['dependents'] = [dependent.to_dict() for dependent in document.dependents.all()]
  142. # Get children documents if this is a directory
  143. if document.is_directory:
  144. directory = Directory.objects.get(id=document.id)
  145. # If this is the user's home directory, fetch shared docs too
  146. if document.is_home_directory:
  147. children = directory.get_children_and_shared_documents(user=request.user)
  148. else:
  149. children = directory.get_children_documents()
  150. # Filter and order results
  151. response.update(_filter_documents(request, queryset=children, flatten=False))
  152. # Paginate and serialize Results
  153. if 'documents' in response:
  154. response.update(_paginate(request, queryset=response['documents']))
  155. # Rename documents to children
  156. response['children'] = response.pop('documents')
  157. response['children'] = [doc.to_dict() for doc in response['children']]
  158. return JsonResponse(response)
  159. @api_error_handler
  160. def open_document(request):
  161. doc_id = request.GET.get('id')
  162. if doc_id.isdigit():
  163. document = Document2.objects.document(user=request.user, doc_id=doc_id)
  164. else:
  165. document = Document2.objects.get_by_uuid(user=request.user, uuid=doc_id)
  166. return redirect(document.get_absolute_url())
  167. @api_error_handler
  168. @require_POST
  169. def move_document(request):
  170. source_doc_uuid = json.loads(request.POST.get('source_doc_uuid'))
  171. destination_doc_uuid = json.loads(request.POST.get('destination_doc_uuid'))
  172. if not source_doc_uuid or not destination_doc_uuid:
  173. raise PopupException(_('move_document requires source_doc_uuid and destination_doc_uuid'))
  174. source = Document2.objects.get_by_uuid(user=request.user, uuid=source_doc_uuid, perm_type='write')
  175. destination = Directory.objects.get_by_uuid(user=request.user, uuid=destination_doc_uuid, perm_type='write')
  176. doc = source.move(destination, request.user)
  177. return JsonResponse({
  178. 'status': 0,
  179. 'document': doc.to_dict()
  180. })
  181. @api_error_handler
  182. @require_POST
  183. def create_directory(request):
  184. parent_uuid = json.loads(request.POST.get('parent_uuid'))
  185. name = json.loads(request.POST.get('name'))
  186. if not parent_uuid or not name:
  187. raise PopupException(_('create_directory requires parent_uuid and name'))
  188. parent_dir = Directory.objects.get_by_uuid(user=request.user, uuid=parent_uuid, perm_type='write')
  189. directory = Directory.objects.create(name=name, owner=request.user, parent_directory=parent_dir)
  190. return JsonResponse({
  191. 'status': 0,
  192. 'directory': directory.to_dict()
  193. })
  194. @api_error_handler
  195. @require_POST
  196. def update_document(request):
  197. uuid = json.loads(request.POST.get('uuid'))
  198. if not uuid:
  199. raise PopupException(_('update_document requires uuid'))
  200. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type='write')
  201. whitelisted_attrs = ['name', 'description']
  202. for attr in whitelisted_attrs:
  203. if request.POST.get(attr):
  204. setattr(document, attr, request.POST.get(attr))
  205. document.save(update_fields=whitelisted_attrs)
  206. return JsonResponse({
  207. 'status': 0,
  208. 'document': document.to_dict()
  209. })
  210. @api_error_handler
  211. @require_POST
  212. def delete_document(request):
  213. """
  214. Accepts a uuid and optional skip_trash parameter
  215. (Default) skip_trash=false, flags a document as trashed
  216. skip_trash=true, deletes it permanently along with any history dependencies
  217. If directory and skip_trash=false, all dependencies will also be flagged as trash
  218. If directory and skip_trash=true, directory must be empty (no dependencies)
  219. """
  220. uuid = json.loads(request.POST.get('uuid'))
  221. skip_trash = json.loads(request.POST.get('skip_trash', 'false'))
  222. if not uuid:
  223. raise PopupException(_('delete_document requires uuid'))
  224. document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type='write')
  225. if skip_trash:
  226. document.delete()
  227. else:
  228. document.trash()
  229. return JsonResponse({
  230. 'status': 0,
  231. })
  232. @api_error_handler
  233. @require_POST
  234. def share_document(request):
  235. """
  236. Set who else or which other group can interact with the document.
  237. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  238. """
  239. perms_dict = json.loads(request.POST.get('data'))
  240. uuid = json.loads(request.POST.get('uuid'))
  241. if not uuid or not perms_dict:
  242. raise PopupException(_('share_document requires uuid and perms_dict'))
  243. doc = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
  244. for name, perm in perms_dict.iteritems():
  245. users = groups = None
  246. if perm.get('user_ids'):
  247. users = User.objects.in_bulk(perm.get('user_ids'))
  248. else:
  249. users = []
  250. if perm.get('group_ids'):
  251. groups = Group.objects.in_bulk(perm.get('group_ids'))
  252. else:
  253. groups = []
  254. doc = doc.share(request.user, name=name, users=users, groups=groups)
  255. return JsonResponse({
  256. 'status': 0,
  257. 'document': doc.to_dict()
  258. })
  259. @ensure_csrf_cookie
  260. def export_documents(request):
  261. if request.GET.get('documents'):
  262. selection = json.loads(request.GET.get('documents'))
  263. else:
  264. selection = json.loads(request.POST.get('documents'))
  265. # Only export documents the user has permissions to read
  266. docs = Document2.objects.documents(user=request.user, perms='both', include_history=True, include_trashed=True).\
  267. filter(id__in=selection).order_by('-id')
  268. # Add any dependencies to the set of exported documents
  269. export_doc_set = _get_dependencies(docs)
  270. # For directories, add any children docs to the set of exported documents
  271. export_doc_set.update(_get_dependencies(docs, deps_mode=False))
  272. # Get PKs of documents to export
  273. doc_ids = [doc.pk for doc in export_doc_set]
  274. f = StringIO.StringIO()
  275. if doc_ids:
  276. doc_ids = ','.join(map(str, doc_ids))
  277. management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_keys=True, verbosity=2, stdout=f)
  278. if request.GET.get('format') == 'json':
  279. return JsonResponse(f.getvalue(), safe=False)
  280. elif request.GET.get('format') == 'zip':
  281. zfile = zipfile.ZipFile(f, 'w')
  282. zfile.writestr("hue.json", f.getvalue())
  283. for doc in docs:
  284. if doc.type == 'notebook':
  285. try:
  286. from spark.models import Notebook
  287. zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
  288. except Exception, e:
  289. LOG.exception(e)
  290. zfile.close()
  291. response = HttpResponse(content_type="application/zip")
  292. response["Content-Length"] = len(f.getvalue())
  293. response['Content-Disposition'] = 'attachment; filename="hue-documents.zip"'
  294. response.write(f.getvalue())
  295. return response
  296. else:
  297. return make_response(f.getvalue(), 'json', 'hue-documents')
  298. @ensure_csrf_cookie
  299. def import_documents(request):
  300. if request.FILES.get('documents'):
  301. documents = request.FILES['documents'].read()
  302. else:
  303. documents = json.loads(request.POST.get('documents'))
  304. documents = json.loads(documents)
  305. docs = []
  306. uuids_map = dict((doc['fields']['uuid'], None) for doc in documents)
  307. for doc in documents:
  308. # If doc is not owned by current user, make a copy of the document with current user as owner
  309. if doc['fields']['owner'][0] != request.user.username:
  310. doc = _copy_document_with_owner(doc, request.user, uuids_map)
  311. else: # Update existing doc or create new
  312. doc = _create_or_update_document_with_owner(doc, request.user, uuids_map)
  313. # Set last modified date to now
  314. doc['fields']['last_modified'] = datetime.now().replace(microsecond=0).isoformat()
  315. docs.append(doc)
  316. f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
  317. f.write(json.dumps(docs))
  318. f.flush()
  319. stdout = StringIO.StringIO()
  320. try:
  321. management.call_command('loaddata', f.name, stdout=stdout)
  322. except Exception, e:
  323. return JsonResponse({'message': smart_str(e)})
  324. Document.objects.sync()
  325. if request.POST.get('redirect'):
  326. return redirect(request.POST.get('redirect'))
  327. else:
  328. return JsonResponse({'message': stdout.getvalue()})
  329. def _get_dependencies(documents, deps_mode=True):
  330. """
  331. Given a list of Document2 objects, perform a depth-first search and return a set of documents with all
  332. dependencies included
  333. :param doc_set: set of Document2 objects to include
  334. :param deps_mode: traverse dependencies relationship, otherwise traverse children relationship
  335. """
  336. doc_set = set()
  337. for doc in documents:
  338. stack = [doc]
  339. while stack:
  340. curr_doc = stack.pop()
  341. if curr_doc not in doc_set:
  342. doc_set.add(curr_doc)
  343. if deps_mode:
  344. deps_set = set(curr_doc.dependencies.all())
  345. else:
  346. deps_set = set(curr_doc.children.all())
  347. stack.extend(deps_set - doc_set)
  348. return doc_set
  349. def _copy_document_with_owner(doc, owner, uuids_map):
  350. home_dir = Directory.objects.get_home_directory(owner)
  351. doc['fields']['owner'] = [owner.username]
  352. doc['pk'] = None
  353. doc['fields']['version'] = 1
  354. # Retrieve from the import_uuids_map if it's already been reassigned, or assign a new UUID and map it
  355. old_uuid = doc['fields']['uuid']
  356. if uuids_map[old_uuid] is None:
  357. uuids_map[old_uuid] = uuid_default()
  358. doc['fields']['uuid'] = uuids_map[old_uuid]
  359. # Remap parent directory if needed
  360. parent_uuid = doc['fields']['parent_directory'][0]
  361. if parent_uuid not in uuids_map.keys():
  362. LOG.warn('Could not find parent directory with UUID: %s in JSON import, will set parent to home directory' %
  363. parent_uuid)
  364. doc['fields']['parent_directory'] = [home_dir.uuid, home_dir.version, home_dir.is_history]
  365. else:
  366. if uuids_map[parent_uuid] is None:
  367. uuids_map[parent_uuid] = uuid_default()
  368. doc['fields']['parent_directory'] = [uuids_map[parent_uuid], 1, False]
  369. # Remap dependencies if needed
  370. idx = 0
  371. for dep_uuid, dep_version, dep_is_history in doc['fields']['dependencies']:
  372. if dep_uuid not in uuids_map.keys():
  373. LOG.warn('Could not find dependency UUID: %s in JSON import, may cause integrity errors if not found.' % dep_uuid)
  374. else:
  375. if uuids_map[dep_uuid] is None:
  376. uuids_map[dep_uuid] = uuid_default()
  377. doc['fields']['dependencies'][idx][0] = uuids_map[dep_uuid]
  378. idx += 1
  379. return doc
  380. def _create_or_update_document_with_owner(doc, owner, uuids_map):
  381. home_dir = Directory.objects.get_home_directory(owner)
  382. create_new = False
  383. try:
  384. owned_docs = Document2.objects.filter(uuid=doc['fields']['uuid'], owner=owner).order_by('-last_modified')
  385. if owned_docs.exists():
  386. existing_doc = owned_docs[0]
  387. doc['pk'] = existing_doc.pk
  388. else:
  389. create_new = True
  390. except FilesystemException, e:
  391. create_new = True
  392. if create_new:
  393. LOG.warn('Could not find document with UUID: %s, will create a new document on import.', doc['fields']['uuid'])
  394. doc['pk'] = None
  395. doc['fields']['version'] = 1
  396. # Verify that parent exists, log warning and set parent to user's home directory if not found
  397. if doc['fields']['parent_directory']:
  398. uuid, version, is_history = doc['fields']['parent_directory']
  399. if uuid not in uuids_map.keys() and \
  400. not Document2.objects.filter(uuid=uuid, version=version, is_history=is_history).exists():
  401. LOG.warn('Could not find parent document with UUID: %s, will set parent to home directory' % uuid)
  402. doc['fields']['parent_directory'] = [home_dir.uuid, home_dir.version, home_dir.is_history]
  403. # Verify that dependencies exist, raise critical error if any dependency not found
  404. if doc['fields']['dependencies']:
  405. for uuid, version, is_history in doc['fields']['dependencies']:
  406. if not uuid in uuids_map.keys() and \
  407. not Document2.objects.filter(uuid=uuid, version=version, is_history=is_history).exists():
  408. raise PopupException(_('Cannot import document, dependency with UUID: %s not found.') % uuid)
  409. return doc
  410. def _filter_documents(request, queryset, flatten=True):
  411. """
  412. Given optional querystring params extracted from the request, filter the given queryset of documents and return a
  413. dictionary with the refined queryset and filter params
  414. :param request: request object with params
  415. :param queryset: Document2 queryset
  416. :param flatten: Return all results in a flat list if true, otherwise roll up to common directory
  417. """
  418. type_filters = request.GET.getlist('type', None)
  419. sort = request.GET.get('sort', '-last_modified')
  420. search_text = request.GET.get('text', None)
  421. documents = queryset.search_documents(
  422. types=type_filters,
  423. search_text=search_text,
  424. order_by=sort)
  425. # Roll up documents to common directory
  426. if not flatten:
  427. documents = documents.exclude(parent_directory__in=documents)
  428. count = documents.count()
  429. return {
  430. 'documents': documents,
  431. 'count': count,
  432. 'types': type_filters,
  433. 'text': search_text,
  434. 'sort': sort
  435. }
  436. def _paginate(request, queryset):
  437. """
  438. Given optional querystring params extracted from the request, slice the given queryset of documents for the given page
  439. and limit, and return the updated queryset along with pagination params used.
  440. :param request: request object with params
  441. :param queryset: queryset
  442. """
  443. page = int(request.GET.get('page', 1))
  444. limit = int(request.GET.get('limit', 0))
  445. if limit > 0:
  446. offset = (page - 1) * limit
  447. last = offset + limit
  448. queryset = queryset.all()[offset:last]
  449. return {
  450. 'documents': queryset,
  451. 'page': page,
  452. 'limit': limit
  453. }