converters.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import json
  18. import logging
  19. import time
  20. from django.db import transaction
  21. from django.utils.translation import ugettext as _
  22. from desktop.lib.exceptions_renderable import PopupException
  23. from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission
  24. from notebook.api import _historify
  25. from notebook.models import import_saved_beeswax_query
  26. LOG = logging.getLogger(__name__)
  27. class DocumentConverter(object):
  28. """
  29. Given a user, converts any existing Document objects to Document2 objects
  30. """
  31. def __init__(self, user):
  32. self.user = user
  33. # If user does not have a home directory, we need to create one and import any orphan documents to it
  34. self.home_dir = Document2.objects.create_user_directories(self.user)
  35. self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
  36. self.imported_doc_count = 0
  37. self.failed_doc_ids = []
  38. def convert(self):
  39. # Convert SavedQuery documents
  40. try:
  41. from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
  42. docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
  43. for doc in docs:
  44. try:
  45. if doc.content_object:
  46. notebook = import_saved_beeswax_query(doc.content_object)
  47. data = notebook.get_data()
  48. doc2 = self._create_doc2(
  49. document=doc,
  50. doctype=data['type'],
  51. name=data['name'],
  52. description=data['description'],
  53. data=notebook.get_json()
  54. )
  55. self.imported_doc_count += 1
  56. except Exception, e:
  57. self.failed_doc_ids.append(doc.id)
  58. LOG.exception('Failed to import SavedQuery document id: %d' % doc.id)
  59. except ImportError:
  60. LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
  61. # Convert SQL Query history documents
  62. try:
  63. from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
  64. docs = self._get_unconverted_docs(SavedQuery, only_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
  65. for doc in docs:
  66. try:
  67. if doc.content_object:
  68. notebook = import_saved_beeswax_query(doc.content_object)
  69. data = notebook.get_data()
  70. data['isSaved'] = False
  71. data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
  72. with transaction.atomic():
  73. doc2 = _historify(data, self.user)
  74. doc2.last_modified = doc.last_modified
  75. # save() updates the last_modified to current time. Resetting it using update()
  76. doc2.save()
  77. Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
  78. self.imported_doc_count += 1
  79. # Tag for not re-importing
  80. Document.objects.link(
  81. doc2,
  82. owner=doc2.owner,
  83. name=doc2.name,
  84. description=doc2.description,
  85. extra=doc.extra
  86. )
  87. doc.add_tag(self.imported_tag)
  88. doc.save()
  89. except Exception, e:
  90. self.failed_doc_ids.append(doc.id)
  91. LOG.exception('Failed to import history document id: %d' % doc.id)
  92. except ImportError, e:
  93. LOG.warn('Cannot convert history documents: beeswax app is not installed')
  94. # Convert Job Designer documents
  95. try:
  96. from oozie.models import Workflow
  97. # TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
  98. docs = self._get_unconverted_docs(Workflow)
  99. for doc in docs:
  100. try:
  101. if doc.content_object:
  102. data = doc.content_object.data_dict
  103. data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
  104. doc2 = self._create_doc2(
  105. document=doc,
  106. doctype='link-workflow',
  107. description=doc.description,
  108. data=json.dumps(data)
  109. )
  110. self.imported_doc_count += 1
  111. except Exception, e:
  112. self.failed_doc_ids.append(doc.id)
  113. LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
  114. except ImportError, e:
  115. LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
  116. # Convert PigScript documents
  117. try:
  118. from pig.models import PigScript
  119. # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
  120. docs = self._get_unconverted_docs(PigScript)
  121. for doc in docs:
  122. try:
  123. if doc.content_object:
  124. data = doc.content_object.dict
  125. data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
  126. doc2 = self._create_doc2(
  127. document=doc,
  128. doctype='link-pigscript',
  129. description=doc.description,
  130. data=json.dumps(data)
  131. )
  132. self.imported_doc_count += 1
  133. except Exception, e:
  134. self.failed_doc_ids.append(doc.id)
  135. LOG.exception('Failed to import Pig document id: %d' % doc.id)
  136. except ImportError, e:
  137. LOG.warn('Cannot convert Pig documents: pig app is not installed')
  138. # Add converted docs to root directory
  139. if self.imported_doc_count:
  140. LOG.info('Successfully imported %d documents for user: %s' % (self.imported_doc_count, self.user.username))
  141. # Log docs that failed to import
  142. if self.failed_doc_ids:
  143. LOG.error('Failed to import %d document(s) for user: %s - %s' % (len(self.failed_doc_ids), self.user.username, self.failed_doc_ids))
  144. # Set is_trashed field for old documents with is_trashed=None
  145. docs = Document2.objects.filter(owner=self.user, is_trashed=None).exclude(is_history=True)
  146. for doc in docs:
  147. try:
  148. if doc.path and doc.path != '/.Trash':
  149. doc_last_modified = doc.last_modified
  150. doc.is_trashed = doc.path.startswith('/.Trash')
  151. doc.save()
  152. # save() updates the last_modified to current time. Resetting it using update()
  153. Document2.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
  154. except Exception, e:
  155. LOG.exception("Failed to set is_trashed field with exception: %s" % e)
  156. def _get_unconverted_docs(self, content_type, only_history=False):
  157. docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
  158. tags = [
  159. DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
  160. DocumentTag.objects.get_example_tag(user=self.user), # No examples
  161. self.imported_tag # No already imported docs
  162. ]
  163. if only_history:
  164. docs = docs.filter(tags__in=[DocumentTag.objects.get_history_tag(user=self.user)])
  165. else: # Exclude history docs by default
  166. tags.append(DocumentTag.objects.get_history_tag(user=self.user))
  167. return docs.exclude(tags__in=tags)
  168. def _get_parent_directory(self, document):
  169. """
  170. Returns the parent directory object that should be used for a given document. If the document is tagged with a
  171. project name (non-RESERVED DocumentTag), a Directory object with the first project tag found is returned.
  172. Otherwise, the owner's home directory is returned.
  173. """
  174. parent_dir = self.home_dir
  175. project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
  176. if project_tags.exists():
  177. first_tag = project_tags[0]
  178. parent_dir, created = Directory.objects.get_or_create(
  179. owner=self.user,
  180. name=first_tag.tag,
  181. parent_directory=self.home_dir
  182. )
  183. return parent_dir
  184. def _sync_permissions(self, document, document2):
  185. """
  186. Syncs (creates) Document2Permissions based on the DocumentPermissions found for a given document.
  187. """
  188. doc_permissions = DocumentPermission.objects.filter(doc=document)
  189. for perm in doc_permissions:
  190. doc2_permission, created = Document2Permission.objects.get_or_create(doc=document2, perms=perm.perms)
  191. if perm.users:
  192. doc2_permission.users.add(*perm.users.all())
  193. if perm.groups:
  194. doc2_permission.groups.add(*perm.groups.all())
  195. def _create_doc2(self, document, doctype, name=None, description=None, data=None):
  196. try:
  197. document2 = None
  198. with transaction.atomic():
  199. name = name if name else document.name
  200. document2 = Document2.objects.create(
  201. owner=self.user,
  202. parent_directory=self._get_parent_directory(document),
  203. name=name,
  204. type=doctype,
  205. description=description,
  206. data=data
  207. )
  208. self._sync_permissions(document, document2)
  209. # Create a doc1 copy and link it for backwards compatibility
  210. Document.objects.link(
  211. document2,
  212. owner=document2.owner,
  213. name=document2.name,
  214. description=document2.description,
  215. extra=document.extra
  216. )
  217. # save() updates the last_modified to current time. Resetting it using update()
  218. Document2.objects.filter(id=document2.id).update(last_modified=document.last_modified)
  219. document.add_tag(self.imported_tag)
  220. document.save()
  221. return document2
  222. except Exception, e:
  223. # Just to be sure we delete Doc2 object incase of exception.
  224. # Possible when there are mixed InnoDB and MyISAM tables
  225. if document2 and Document2.objects.filter(id=document2.id).exists():
  226. document2.delete()
  227. raise PopupException(_("Failed to convert Document object: %s") % e)