| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365 |
- # Licensed to Cloudera, Inc. under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. Cloudera, Inc. licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- import json
- import logging
- import time
- import re
- from django.db import transaction
- from desktop.lib.exceptions_renderable import PopupException
- from django.core.exceptions import FieldError
- from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission, FilesystemException
- from notebook.models import import_saved_beeswax_query
- from doc2_utils import findMatchingQuery, removeInvalidChars
- LOG = logging.getLogger(__name__)
- class DocumentConverterHueScripts(object):
- """
- Given a user, converts any existing Document objects to Document2 objects
- """
- def __init__(self, user, allowdupes=False, startqueryname=None, startuser=None, processdocs=None):
- self.user = user
- self.allowdupes = allowdupes
- self.startqueryname = startqueryname
- self.startuser = startuser
- if (self.startqueryname or self.startuser) and not processdocs:
- self.processdocs = False
- else:
- self.processdocs = True
- # If user does not have a home directory, we need to create one and import any orphan documents to it
- try:
- self.home_dir = Document2.objects.create_user_directories(self.user)
- except FilesystemException, e:
- LOG.warn("User: %s failed: Exception: %s" % (self.user, e))
- raise
- self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
- self.imported_docs = []
- def convertfailed(self):
- # Convert SavedQuery documents
- try:
- from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
-
- docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
- for doc in docs:
- if doc.content_object:
- id_temp = doc.to_dict()
- id = id_temp['id']
- notebook = import_saved_beeswax_query(doc.content_object)
- data = notebook.get_data()
- name = data['name']
- query = data['snippets'][0]['statement_raw']
- if re.match(self.startqueryname, name) and not self.startuser:
- self.processdocs = True
- if self.processdocs:
- matchdocs = findMatchingQuery(user=self.user, id=id, name=name, query=query, include_history=False)
- if not matchdocs or self.allowdupes:
- try:
- if doc.is_historic():
- data['isSaved'] = False
- doc2 = self._create_doc2(
- document=doc,
- doctype=data['type'],
- name=data['name'],
- description=data['description'],
- data=notebook.get_json()
- )
- if doc.is_historic():
- doc2.is_history = False
- self.imported_docs.append(doc2)
-
- except:
- pass
- except ImportError:
- LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
- pass
- # Convert SQL Query history documents
- try:
- from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
- docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
- for doc in docs:
- if not doc.content_object:
- LOG.error("Content object is missing")
- elif doc.content_object:
- id_temp = doc.to_dict()
- id = id_temp['id']
- notebook = import_saved_beeswax_query(doc.content_object)
- data = notebook.get_data()
- name = data['name']
- query = data['snippets'][0]['statement_raw']
- if re.match(self.startqueryname, name) and not self.startuser:
- self.processdocs = True
- if self.processdocs:
- try:
- data['isSaved'] = False
- data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
- doc2 = self._historify(data, self.user)
- doc2.last_modified = doc.last_modified
- # save() updates the last_modified to current time. Resetting it using update()
- doc2.save()
- Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
-
- self.imported_docs.append(doc2)
-
- # Tag for not re-importing
- Document.objects.link(
- doc2,
- owner=doc2.owner,
- name=doc2.name,
- description=doc2.description,
- extra=doc.extra
- )
-
- try:
- doc.add_tag(self.imported_tag)
- except IntegrityError, e:
- LOG.exception("Failed to add imported_tag to doc %s with error %s" % (doc2.name, e))
- pass
- doc.save()
- except:
- LOG.exception("Doc name: %s" % (doc.name))
- pass
-
- except ImportError, e:
- LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
- pass
- # Convert Job Designer documents
- try:
- from oozie.models import Workflow
- # TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
- docs = self._get_unconverted_docs(Workflow)
- for doc in docs:
- try:
- if doc.content_object:
- data = doc.content_object.data_dict
- data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
- doc2 = self._create_doc2(
- document=doc,
- doctype='link-workflow',
- description=doc.description,
- data=json.dumps(data)
- )
- self.imported_docs.append(doc2)
- except Exception, e:
- self.failed_docs.append(doc)
- LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
- except ImportError, e:
- LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
- # Convert PigScript documents
- try:
- from pig.models import PigScript
- # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
- docs = self._get_unconverted_docs(PigScript)
- for doc in docs:
- try:
- if doc.content_object:
- data = doc.content_object.dict
- data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
- doc2 = self._create_doc2(
- document=doc,
- doctype='link-pigscript',
- description=doc.description,
- data=json.dumps(data)
- )
- self.imported_docs.append(doc2)
- except Exception, e:
- self.failed_docs.append(doc)
- LOG.exception('Failed to import Pig document id: %d' % doc.id)
- except ImportError, e:
- LOG.warn('Cannot convert Pig documents: pig app is not installed')
- # Add converted docs to root directory
- if self.imported_docs:
- LOG.info('Successfully imported %d documents' % len(self.imported_docs))
- # Set is_trashed field for old documents with is_trashed=None
- try:
- docs = Document2.objects.filter(owner=self.user, is_trashed=None)
- for doc in docs:
- try:
- if doc.path and doc.path != '/.Trash':
- doc_last_modified = doc.last_modified
- doc.is_trashed = doc.path.startswith('/.Trash')
- doc.save()
- # save() updates the last_modified to current time. Resetting it using update()
- Document2.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
- except Exception, e:
- LOG.exception("Failed to set is_trashed field with exception: %s" % e)
- except FieldError, e:
- LOG.info("Skipping is_trashed as does not exist in this version")
- return self.processdocs
- def _get_unconverted_docs(self, content_type, with_history=False):
- docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
- tags = [
- DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
- DocumentTag.objects.get_example_tag(user=self.user), # No examples
- # self.imported_tag # No already imported docs
- ]
- if not with_history:
- tags.append(DocumentTag.objects.get_history_tag(user=self.user)) # No history yet
- return docs.exclude(tags__in=tags)
- def _get_parent_directory(self, document):
- """
- Returns the parent directory object that should be used for a given document. If the document is tagged with a
- project name (non-RESERVED DocumentTag), a Directory object with the first project tag found is returned.
- Otherwise, the owner's home directory is returned.
- """
- parent_dir = self.home_dir
- project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
- if project_tags.exists():
- first_tag = project_tags[0]
- parent_dir, created = Directory.objects.get_or_create(
- owner=self.user,
- name=first_tag.tag,
- parent_directory=self.home_dir
- )
- return parent_dir
- def _sync_permissions(self, document, document2):
- """
- Syncs (creates) Document2Permissions based on the DocumentPermissions found for a given document.
- """
- doc_permissions = DocumentPermission.objects.filter(doc=document)
- for perm in doc_permissions:
- try:
- doc2_permission, created = Document2Permission.objects.get_or_create(doc=document2, perms=perm.perms)
- if perm.users:
- doc2_permission.users.add(*perm.users.all())
- if perm.groups:
- doc2_permission.groups.add(*perm.groups.all())
- except:
- pass
- def _create_doc2(self, document, doctype, name=None, description=None, data=None):
- try:
- with transaction.atomic():
- name = name if name else document.name
- name = removeInvalidChars(name)
- document2 = Document2.objects.create(
- owner=self.user,
- parent_directory=self._get_parent_directory(document),
- name=name,
- type=doctype,
- description=description,
- data=data
- )
- self._sync_permissions(document, document2)
- # Create a doc1 copy and link it for backwards compatibility
- Document.objects.link(
- document2,
- owner=document2.owner,
- name=document2.name,
- description=document2.description,
- extra=document.extra
- )
- # save() updates the last_modified to current time. Resetting it using update()
- Document2.objects.filter(id=document2.id).update(last_modified=document.last_modified)
- document.add_tag(self.imported_tag)
- document.save()
- return document2
- except Exception, e:
- raise PopupException(_("Failed to convert Document object: %s") % e)
- def _historify(self, notebook, user):
- query_type = notebook['type']
- name = notebook['name'] if (notebook['name'] and notebook['name'].strip() != '') else DEFAULT_HISTORY_NAME
- name = removeInvalidChars(name)
- try:
- history_doc = Document2.objects.create(
- name=name,
- type=query_type,
- owner=user,
- is_history=True,
- is_managed=notebook.get('isManaged') == True
- )
- except TypeError:
- history_doc = Document2.objects.create(
- name=name,
- type=query_type,
- owner=user,
- is_history=True,
- )
- # Link history of saved query
- if notebook['isSaved']:
- parent_doc = Document2.objects.get(uuid=notebook.get('parentSavedQueryUuid') or notebook['uuid']) # From previous history query or initial saved query
- notebook['parentSavedQueryUuid'] = parent_doc.uuid
- history_doc.dependencies.add(parent_doc)
- Document.objects.link(
- history_doc,
- name=history_doc.name,
- owner=history_doc.owner,
- description=history_doc.description,
- extra=query_type
- )
- notebook['uuid'] = history_doc.uuid
- history_doc.update_data(notebook)
- history_doc.search = self._get_statement(notebook)
- history_doc.save()
- return history_doc
- def _get_statement(self, notebook):
- statement = ''
- if notebook['snippets'] and len(notebook['snippets']) > 0:
- try:
- statement = notebook['snippets'][0]['result']['handle']['statement']
- if type(statement) == dict: # Old format
- statement = notebook['snippets'][0]['statement_raw']
- except KeyError: # Old format
- statement = notebook['snippets'][0]['statement_raw']
- return statement
|