hue_converters.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import json
  17. import logging
  18. import time
  19. import re
  20. from django.db import transaction
  21. from desktop.lib.exceptions_renderable import PopupException
  22. from django.core.exceptions import FieldError
  23. from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission, FilesystemException
  24. from notebook.models import import_saved_beeswax_query
  25. from doc2_utils import findMatchingQuery, removeInvalidChars
  26. LOG = logging.getLogger(__name__)
  27. class DocumentConverterHueScripts(object):
  28. """
  29. Given a user, converts any existing Document objects to Document2 objects
  30. """
  31. def __init__(self, user, allowdupes=False, startqueryname=None, startuser=None, processdocs=None):
  32. self.user = user
  33. self.allowdupes = allowdupes
  34. self.startqueryname = startqueryname
  35. self.startuser = startuser
  36. if (self.startqueryname or self.startuser) and not processdocs:
  37. self.processdocs = False
  38. else:
  39. self.processdocs = True
  40. # If user does not have a home directory, we need to create one and import any orphan documents to it
  41. try:
  42. self.home_dir = Document2.objects.create_user_directories(self.user)
  43. except FilesystemException, e:
  44. LOG.warn("User: %s failed: Exception: %s" % (self.user, e))
  45. raise
  46. self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
  47. self.imported_docs = []
  48. def convertfailed(self):
  49. # Convert SavedQuery documents
  50. try:
  51. from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
  52. docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
  53. for doc in docs:
  54. if doc.content_object:
  55. id_temp = doc.to_dict()
  56. id = id_temp['id']
  57. notebook = import_saved_beeswax_query(doc.content_object)
  58. data = notebook.get_data()
  59. name = data['name']
  60. query = data['snippets'][0]['statement_raw']
  61. if re.match(self.startqueryname, name) and not self.startuser:
  62. self.processdocs = True
  63. if self.processdocs:
  64. matchdocs = findMatchingQuery(user=self.user, id=id, name=name, query=query, include_history=False)
  65. if not matchdocs or self.allowdupes:
  66. try:
  67. if doc.is_historic():
  68. data['isSaved'] = False
  69. doc2 = self._create_doc2(
  70. document=doc,
  71. doctype=data['type'],
  72. name=data['name'],
  73. description=data['description'],
  74. data=notebook.get_json()
  75. )
  76. if doc.is_historic():
  77. doc2.is_history = False
  78. self.imported_docs.append(doc2)
  79. except:
  80. pass
  81. except ImportError:
  82. LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
  83. pass
  84. # Convert SQL Query history documents
  85. try:
  86. from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
  87. docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
  88. for doc in docs:
  89. if not doc.content_object:
  90. LOG.error("Content object is missing")
  91. elif doc.content_object:
  92. id_temp = doc.to_dict()
  93. id = id_temp['id']
  94. notebook = import_saved_beeswax_query(doc.content_object)
  95. data = notebook.get_data()
  96. name = data['name']
  97. query = data['snippets'][0]['statement_raw']
  98. if re.match(self.startqueryname, name) and not self.startuser:
  99. self.processdocs = True
  100. if self.processdocs:
  101. try:
  102. data['isSaved'] = False
  103. data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
  104. doc2 = self._historify(data, self.user)
  105. doc2.last_modified = doc.last_modified
  106. # save() updates the last_modified to current time. Resetting it using update()
  107. doc2.save()
  108. Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
  109. self.imported_docs.append(doc2)
  110. # Tag for not re-importing
  111. Document.objects.link(
  112. doc2,
  113. owner=doc2.owner,
  114. name=doc2.name,
  115. description=doc2.description,
  116. extra=doc.extra
  117. )
  118. try:
  119. doc.add_tag(self.imported_tag)
  120. except IntegrityError, e:
  121. LOG.exception("Failed to add imported_tag to doc %s with error %s" % (doc2.name, e))
  122. pass
  123. doc.save()
  124. except:
  125. LOG.exception("Doc name: %s" % (doc.name))
  126. pass
  127. except ImportError, e:
  128. LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
  129. pass
  130. # Convert Job Designer documents
  131. try:
  132. from oozie.models import Workflow
  133. # TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
  134. docs = self._get_unconverted_docs(Workflow)
  135. for doc in docs:
  136. try:
  137. if doc.content_object:
  138. data = doc.content_object.data_dict
  139. data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
  140. doc2 = self._create_doc2(
  141. document=doc,
  142. doctype='link-workflow',
  143. description=doc.description,
  144. data=json.dumps(data)
  145. )
  146. self.imported_docs.append(doc2)
  147. except Exception, e:
  148. self.failed_docs.append(doc)
  149. LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
  150. except ImportError, e:
  151. LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
  152. # Convert PigScript documents
  153. try:
  154. from pig.models import PigScript
  155. # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
  156. docs = self._get_unconverted_docs(PigScript)
  157. for doc in docs:
  158. try:
  159. if doc.content_object:
  160. data = doc.content_object.dict
  161. data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
  162. doc2 = self._create_doc2(
  163. document=doc,
  164. doctype='link-pigscript',
  165. description=doc.description,
  166. data=json.dumps(data)
  167. )
  168. self.imported_docs.append(doc2)
  169. except Exception, e:
  170. self.failed_docs.append(doc)
  171. LOG.exception('Failed to import Pig document id: %d' % doc.id)
  172. except ImportError, e:
  173. LOG.warn('Cannot convert Pig documents: pig app is not installed')
  174. # Add converted docs to root directory
  175. if self.imported_docs:
  176. LOG.info('Successfully imported %d documents' % len(self.imported_docs))
  177. # Set is_trashed field for old documents with is_trashed=None
  178. try:
  179. docs = Document2.objects.filter(owner=self.user, is_trashed=None)
  180. for doc in docs:
  181. try:
  182. if doc.path and doc.path != '/.Trash':
  183. doc_last_modified = doc.last_modified
  184. doc.is_trashed = doc.path.startswith('/.Trash')
  185. doc.save()
  186. # save() updates the last_modified to current time. Resetting it using update()
  187. Document2.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
  188. except Exception, e:
  189. LOG.exception("Failed to set is_trashed field with exception: %s" % e)
  190. except FieldError, e:
  191. LOG.info("Skipping is_trashed as does not exist in this version")
  192. return self.processdocs
  193. def _get_unconverted_docs(self, content_type, with_history=False):
  194. docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
  195. tags = [
  196. DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
  197. DocumentTag.objects.get_example_tag(user=self.user), # No examples
  198. # self.imported_tag # No already imported docs
  199. ]
  200. if not with_history:
  201. tags.append(DocumentTag.objects.get_history_tag(user=self.user)) # No history yet
  202. return docs.exclude(tags__in=tags)
  203. def _get_parent_directory(self, document):
  204. """
  205. Returns the parent directory object that should be used for a given document. If the document is tagged with a
  206. project name (non-RESERVED DocumentTag), a Directory object with the first project tag found is returned.
  207. Otherwise, the owner's home directory is returned.
  208. """
  209. parent_dir = self.home_dir
  210. project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
  211. if project_tags.exists():
  212. first_tag = project_tags[0]
  213. parent_dir, created = Directory.objects.get_or_create(
  214. owner=self.user,
  215. name=first_tag.tag,
  216. parent_directory=self.home_dir
  217. )
  218. return parent_dir
  219. def _sync_permissions(self, document, document2):
  220. """
  221. Syncs (creates) Document2Permissions based on the DocumentPermissions found for a given document.
  222. """
  223. doc_permissions = DocumentPermission.objects.filter(doc=document)
  224. for perm in doc_permissions:
  225. try:
  226. doc2_permission, created = Document2Permission.objects.get_or_create(doc=document2, perms=perm.perms)
  227. if perm.users:
  228. doc2_permission.users.add(*perm.users.all())
  229. if perm.groups:
  230. doc2_permission.groups.add(*perm.groups.all())
  231. except:
  232. pass
  233. def _create_doc2(self, document, doctype, name=None, description=None, data=None):
  234. try:
  235. with transaction.atomic():
  236. name = name if name else document.name
  237. name = removeInvalidChars(name)
  238. document2 = Document2.objects.create(
  239. owner=self.user,
  240. parent_directory=self._get_parent_directory(document),
  241. name=name,
  242. type=doctype,
  243. description=description,
  244. data=data
  245. )
  246. self._sync_permissions(document, document2)
  247. # Create a doc1 copy and link it for backwards compatibility
  248. Document.objects.link(
  249. document2,
  250. owner=document2.owner,
  251. name=document2.name,
  252. description=document2.description,
  253. extra=document.extra
  254. )
  255. # save() updates the last_modified to current time. Resetting it using update()
  256. Document2.objects.filter(id=document2.id).update(last_modified=document.last_modified)
  257. document.add_tag(self.imported_tag)
  258. document.save()
  259. return document2
  260. except Exception, e:
  261. raise PopupException(_("Failed to convert Document object: %s") % e)
  262. def _historify(self, notebook, user):
  263. query_type = notebook['type']
  264. name = notebook['name'] if (notebook['name'] and notebook['name'].strip() != '') else DEFAULT_HISTORY_NAME
  265. name = removeInvalidChars(name)
  266. try:
  267. history_doc = Document2.objects.create(
  268. name=name,
  269. type=query_type,
  270. owner=user,
  271. is_history=True,
  272. is_managed=notebook.get('isManaged') == True
  273. )
  274. except TypeError:
  275. history_doc = Document2.objects.create(
  276. name=name,
  277. type=query_type,
  278. owner=user,
  279. is_history=True,
  280. )
  281. # Link history of saved query
  282. if notebook['isSaved']:
  283. parent_doc = Document2.objects.get(uuid=notebook.get('parentSavedQueryUuid') or notebook['uuid']) # From previous history query or initial saved query
  284. notebook['parentSavedQueryUuid'] = parent_doc.uuid
  285. history_doc.dependencies.add(parent_doc)
  286. Document.objects.link(
  287. history_doc,
  288. name=history_doc.name,
  289. owner=history_doc.owner,
  290. description=history_doc.description,
  291. extra=query_type
  292. )
  293. notebook['uuid'] = history_doc.uuid
  294. history_doc.update_data(notebook)
  295. history_doc.search = self._get_statement(notebook)
  296. history_doc.save()
  297. return history_doc
  298. def _get_statement(self, notebook):
  299. statement = ''
  300. if notebook['snippets'] and len(notebook['snippets']) > 0:
  301. try:
  302. statement = notebook['snippets'][0]['result']['handle']['statement']
  303. if type(statement) == dict: # Old format
  304. statement = notebook['snippets'][0]['statement_raw']
  305. except KeyError: # Old format
  306. statement = notebook['snippets'][0]['statement_raw']
  307. return statement