api2.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. import json
  19. import tempfile
  20. import StringIO
  21. import zipfile
  22. from django.contrib.auth.models import Group, User
  23. from django.core import management
  24. from django.db import transaction
  25. from django.http import HttpResponse
  26. from django.shortcuts import redirect
  27. from django.utils.translation import ugettext as _
  28. from django.views.decorators.http import require_POST
  29. from beeswax.models import SavedQuery
  30. from desktop.lib.django_util import JsonResponse
  31. from desktop.lib.exceptions_renderable import PopupException
  32. from desktop.lib.export_csvxls import make_response
  33. from desktop.lib.i18n import smart_str, force_unicode
  34. from desktop.models import Document2, Document, Directory, DocumentTag, FilesystemException, import_saved_beeswax_query
  35. LOG = logging.getLogger(__name__)
  36. def api_error_handler(func):
  37. def decorator(*args, **kwargs):
  38. response = {}
  39. try:
  40. return func(*args, **kwargs)
  41. except Exception, e:
  42. LOG.exception('Error running %s' % func)
  43. response['status'] = -1
  44. response['message'] = force_unicode(str(e))
  45. finally:
  46. if response:
  47. return JsonResponse(response)
  48. return decorator
  49. @api_error_handler
  50. def get_documents(request):
  51. """
  52. Returns all documents and directories found for the given uuid or path and current user.
  53. Optional params:
  54. page=<n> - Controls pagination. Defaults to 1.
  55. limit=<n> - Controls limit per page. Defaults to all.
  56. type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc). Default to all.
  57. sort=<key> - Sort by the attribute <key>, which is one of:
  58. "name", "type", "owner", "last_modified"
  59. Accepts the form "-last_modified", which sorts in descending order.
  60. Default to "-last_modified".
  61. text=<frag> - Search for fragment "frag" in names and descriptions.
  62. """
  63. path = request.GET.get('path', '/')
  64. uuid = request.GET.get('uuid')
  65. if uuid:
  66. document = Document2.objects.get_by_uuid(uuid)
  67. else: # Find by path
  68. document = Document2.objects.get_by_path(user=request.user, path=path)
  69. # TODO perms
  70. # Get querystring filters if any
  71. page = int(request.GET.get('page', 1))
  72. limit = int(request.GET.get('limit', 0))
  73. type_filters = request.GET.getlist('type', None)
  74. sort = request.GET.get('sort', '-last_modified')
  75. search_text = request.GET.get('text', None)
  76. # Get children documents if this is a directory
  77. children = None
  78. count = 0
  79. if document.is_directory:
  80. directory = Directory.objects.get(id=document.id)
  81. children = directory.documents(types=type_filters, search_text=search_text, order_by=sort)
  82. count = children.count()
  83. # Paginate
  84. if children and limit > 0:
  85. offset = (page - 1) * limit
  86. last = offset + limit
  87. children = children.all()[offset:last]
  88. return JsonResponse({
  89. 'document': document.to_dict(),
  90. 'parent': document.parent_directory.to_dict() if document.parent_directory else None,
  91. 'children': [doc.to_dict() for doc in children] if children else [],
  92. 'page': page,
  93. 'limit': limit,
  94. 'count': count,
  95. 'types': type_filters,
  96. 'sort': sort,
  97. 'text': search_text
  98. })
  99. @api_error_handler
  100. def get_document(request):
  101. if request.GET.get('id'):
  102. doc = Document2.objects.get(id=request.GET['id'])
  103. else:
  104. doc = Document2.objects.get_by_uuid(uuid=request.GET['uuid'])
  105. doc_info = doc.to_dict()
  106. return JsonResponse(doc_info)
  107. @api_error_handler
  108. @require_POST
  109. def move_document(request):
  110. source_doc_uuid = json.loads(request.POST.get('source_doc_uuid'))
  111. destination_doc_uuid = json.loads(request.POST.get('destination_doc_uuid'))
  112. if not source_doc_uuid or not destination_doc_uuid:
  113. raise PopupException(_('move_document requires source_doc_uuid and destination_doc_uuid'))
  114. source = Directory.objects.get_by_uuid(uuid=source_doc_uuid)
  115. destination = Directory.objects.get_by_uuid(uuid=destination_doc_uuid)
  116. source.move(destination, request.user)
  117. return JsonResponse({'status': 0})
  118. @api_error_handler
  119. @require_POST
  120. def create_directory(request):
  121. parent_uuid = json.loads(request.POST.get('parent_uuid'))
  122. name = json.loads(request.POST.get('name'))
  123. if not parent_uuid or not name:
  124. raise PopupException(_('create_directory requires parent_uuid and name'))
  125. parent_dir = Directory.objects.get_by_uuid(uuid=parent_uuid)
  126. # TODO: Check permissions and move to manager
  127. directory = Directory.objects.create(name=name, owner=request.user, parent_directory=parent_dir)
  128. return JsonResponse({
  129. 'status': 0,
  130. 'directory': directory.to_dict()
  131. })
  132. @api_error_handler
  133. @require_POST
  134. def delete_document(request):
  135. """
  136. Accepts a uuid and optional skip_trash parameter
  137. (Default) skip_trash=false, flags a document as trashed
  138. skip_trash=true, deletes it permanently along with any history dependencies
  139. If directory and skip_trash=false, all dependencies will also be flagged as trash
  140. If directory and skip_trash=true, directory must be empty (no dependencies)
  141. """
  142. uuid = json.loads(request.POST.get('uuid'))
  143. skip_trash = json.loads(request.POST.get('skip_trash', 'false'))
  144. if not uuid:
  145. raise PopupException(_('delete_document requires uuid'))
  146. document = Document2.objects.get_by_uuid(uuid=uuid)
  147. if skip_trash:
  148. # TODO: check if document is in the .Trash folder, if not raise exception
  149. if document.is_directory and document.has_children:
  150. raise PopupException(_('Directory is not empty'))
  151. document.delete()
  152. else:
  153. document.trash() # TODO: get number of docs trashed
  154. return JsonResponse({
  155. 'status': 0,
  156. })
  157. @api_error_handler
  158. @require_POST
  159. def share_document(request):
  160. """
  161. Set who else or which other group can interact with the document.
  162. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  163. """
  164. perms_dict = json.loads(request.POST.get('data'))
  165. uuid = json.loads(request.POST.get('uuid'))
  166. if not uuid:
  167. raise PopupException(_('share_document requires uuid'))
  168. doc = Document2.objects.get_by_uuid(uuid=uuid)
  169. for name, perm in perms_dict.iteritems():
  170. users = groups = None
  171. if perm.get('user_ids'):
  172. users = User.objects.in_bulk(perm.get('user_ids'))
  173. else:
  174. users = []
  175. if perm.get('group_ids'):
  176. groups = Group.objects.in_bulk(perm.get('group_ids'))
  177. else:
  178. groups = []
  179. doc.share(request.user, name=name, users=users, groups=groups)
  180. return JsonResponse({
  181. 'status': 0,
  182. })
  183. def export_documents(request):
  184. if request.GET.get('documents'):
  185. selection = json.loads(request.GET.get('documents'))
  186. else:
  187. selection = json.loads(request.POST.get('documents'))
  188. # If non admin, only export documents the user owns
  189. docs = Document2.objects
  190. if not request.user.is_superuser:
  191. docs = docs.filter(owner=request.user)
  192. docs = docs.filter(id__in=selection).order_by('-id')
  193. doc_ids = docs.values_list('id', flat=True)
  194. f = StringIO.StringIO()
  195. if doc_ids:
  196. doc_ids = ','.join(map(str, doc_ids))
  197. management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_keys=True, verbosity=2, stdout=f)
  198. if request.GET.get('format') == 'json':
  199. return JsonResponse(f.getvalue(), safe=False)
  200. elif request.GET.get('format') == 'zip':
  201. zfile = zipfile.ZipFile(f, 'w')
  202. zfile.writestr("hue.json", f.getvalue())
  203. for doc in docs:
  204. if doc.type == 'notebook':
  205. try:
  206. from spark.models import Notebook
  207. zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
  208. except Exception, e:
  209. print e
  210. LOG.exception(e)
  211. zfile.close()
  212. response = HttpResponse(content_type="application/zip")
  213. response["Content-Length"] = len(f.getvalue())
  214. response['Content-Disposition'] = 'attachment; filename="hue-documents.zip"'
  215. response.write(f.getvalue())
  216. return response
  217. else:
  218. return make_response(f.getvalue(), 'json', 'hue-documents')
  219. def import_documents(request):
  220. if request.FILES.get('documents'):
  221. documents = request.FILES['documents'].read()
  222. else:
  223. documents = json.loads(request.POST.get('documents'))
  224. documents = json.loads(documents)
  225. docs = []
  226. for doc in documents:
  227. if not request.user.is_superuser:
  228. doc['fields']['owner'] = [request.user.username]
  229. owner = doc['fields']['owner'][0]
  230. doc['fields']['tags'] = []
  231. # TODO: Check if this should be replaced by get_by_uuid
  232. if Document2.objects.filter(uuid=doc['fields']['uuid'], owner__username=owner).exists():
  233. doc['pk'] = Document2.objects.get(uuid=doc['fields']['uuid'], owner__username=owner).pk
  234. else:
  235. doc['pk'] = None
  236. docs.append(doc)
  237. f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
  238. f.write(json.dumps(docs))
  239. f.flush()
  240. stdout = StringIO.StringIO()
  241. try:
  242. management.call_command('loaddata', f.name, stdout=stdout)
  243. except Exception, e:
  244. return JsonResponse({'message': smart_str(e)})
  245. Document.objects.sync()
  246. if request.POST.get('redirect'):
  247. return redirect(request.POST.get('redirect'))
  248. else:
  249. return JsonResponse({'message': stdout.getvalue()})
  250. def _convert_documents(user):
  251. """
  252. Given a user, converts any existing Document objects to Document2 objects
  253. """
  254. from beeswax.models import HQL, IMPALA, RDBMS
  255. with transaction.atomic():
  256. # If user does not have a home directory, we need to create one and import any orphan documents to it
  257. Document2.objects.create_user_directories(user)
  258. docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
  259. imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
  260. docs = docs.exclude(tags__in=[
  261. DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
  262. DocumentTag.objects.get_history_tag(user=user), # No history yet
  263. DocumentTag.objects.get_example_tag(user=user), # No examples
  264. imported_tag # No already imported docs
  265. ])
  266. root_doc, created = Directory.objects.get_or_create(name='', owner=user)
  267. imported_docs = []
  268. for doc in docs:
  269. if doc.content_object:
  270. try:
  271. notebook = import_saved_beeswax_query(doc.content_object)
  272. data = notebook.get_data()
  273. notebook_doc = Document2.objects.create(name=data['name'], type=data['type'], owner=user, data=notebook.get_json())
  274. doc.add_tag(imported_tag)
  275. doc.save()
  276. imported_docs.append(notebook_doc)
  277. except Exception, e:
  278. raise e
  279. if imported_docs:
  280. root_doc.children.add(*imported_docs)