Przeglądaj źródła

[doc2] Create separate module for converting doc1 to doc2

Jenny Kim 9 lat temu
rodzic
commit
3742cf1

+ 2 - 113
desktop/core/src/desktop/api2.py

@@ -23,18 +23,17 @@ import zipfile
 
 from django.contrib.auth.models import Group, User
 from django.core import management
-from django.db import transaction
+
 from django.http import HttpResponse
 from django.shortcuts import redirect
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
 
-from beeswax.models import SavedQuery
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.export_csvxls import make_response
 from desktop.lib.i18n import smart_str, force_unicode
-from desktop.models import Document2, Document, Directory, DocumentTag, FilesystemException, import_saved_beeswax_query
+from desktop.models import Document2, Document, Directory, DocumentTag, FilesystemException
 
 
 LOG = logging.getLogger(__name__)
@@ -404,113 +403,3 @@ def _paginate(request, queryset):
     'page': page,
     'limit': limit
   }
-
-
-def _convert_documents(user):
-  """
-  Given a user, converts any existing Document objects to Document2 objects
-  """
-  from beeswax.models import HQL, IMPALA, RDBMS
-
-  with transaction.atomic():
-    # If user does not have a home directory, we need to create one and import any orphan documents to it
-    Document2.objects.create_user_directories(user)
-
-    docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
-
-    imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
-
-    docs = docs.exclude(tags__in=[
-        DocumentTag.objects.get_trash_tag(user=user),  # No trashed docs
-        DocumentTag.objects.get_history_tag(user=user),  # No history yet
-        DocumentTag.objects.get_example_tag(user=user),  # No examples
-        imported_tag  # No already imported docs
-    ])
-
-    root_doc, created = Directory.objects.get_or_create(name='', owner=user)
-    imported_docs = []
-
-    for doc in docs:
-      if doc.content_object:
-        try:
-          notebook = import_saved_beeswax_query(doc.content_object)
-          data = notebook.get_data()
-          notebook_doc = Document2.objects.create(name=data['name'], type=data['type'], owner=user, data=notebook.get_json())
-
-          doc.add_tag(imported_tag)
-          doc.save()
-          imported_docs.append(notebook_doc)
-        except Exception, e:
-          raise e
-
-    if imported_docs:
-      root_doc.children.add(*imported_docs)
-
-
-def _convert_documents(user):
-  """
-  Given a user, converts any existing Document objects to Document2 objects
-  """
-  from beeswax.models import HQL, IMPALA, RDBMS
-  from oozie.models import Workflow
-  from pig.models import PigScript
-
-  # If user does not have a home directory, we need to create one and import any orphan documents to it
-  home_dir = Document2.objects.create_user_directories(user)
-  imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
-  imported_docs = []
-
-  def get_unconverted_docs(content_type):
-    docs = Document.objects.get_docs(user, content_type).filter(owner=user)
-    docs = docs.exclude(tags__in=[
-      DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
-      DocumentTag.objects.get_history_tag(user=user), # No history yet
-      DocumentTag.objects.get_example_tag(user=user), # No examples
-      imported_tag # No already imported docs
-    ])
-    return docs
-
-  def create_doc2(doc, name, doctype, description=None, data=None):
-    with transaction.atomic():
-      doc2 = Document2.objects.create(
-        owner=user,
-        name=name,
-        type=doctype,
-        description=description,
-        data=data,
-        parent_directory=home_dir
-      )
-      doc.add_tag(imported_tag)
-      doc.save()
-      return doc2
-
-  # Convert SavedQuery documents
-  docs = get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
-  for doc in docs:
-    if doc.content_object:
-      notebook = import_saved_beeswax_query(doc.content_object)
-      data = notebook.get_data()
-      doc2 = create_doc2(doc, name=data['name'], doctype=data['type'], description=data['description'], data=notebook.get_json())
-      imported_docs.append(doc2)
-
-  # Convert Workflow documents
-  docs = get_unconverted_docs(Workflow)
-  for doc in docs:
-    if doc.content_object:
-      data = doc.content_object.data_dict
-      data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
-      doc2 = create_doc2(doc, name=doc.name, doctype='link-workflow', description=doc.description, data=json.dumps(data))
-      imported_docs.append(doc2)
-
-  # Convert PigScript documents
-  docs = get_unconverted_docs(PigScript)
-  for doc in docs:
-    if doc.content_object:
-      data = doc.content_object.dict
-      data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
-      doc2 = create_doc2(doc, name=doc.name, doctype='link-pigscript', description=doc.description, data=json.dumps(data))
-      imported_docs.append(doc2)
-
-  # Add converted docs to root directory
-  if imported_docs:
-    LOG.info('Successfully imported %d documents' % len(imported_docs))

+ 131 - 0
desktop/core/src/desktop/converters.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.db import transaction
+
+from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+from desktop.models import Document2, Document, Directory, DocumentTag, FilesystemException, import_saved_beeswax_query
+from oozie.models import Workflow
+from pig.models import PigScript
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DocumentConverter(object):
+  """
+  Given a user, converts any existing Document objects to Document2 objects
+  """
+
+  def __init__(self, user):
+    self.user = user
+    # If user does not have a home directory, we need to create one and import any orphan documents to it
+    self.home_dir = Document2.objects.create_user_directories(self.user)
+    self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
+    self.imported_docs = []
+
+
+  def convert(self):
+    # Convert SavedQuery documents
+    docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
+    for doc in docs:
+      if doc.content_object:
+        notebook = import_saved_beeswax_query(doc.content_object)
+        data = notebook.get_data()
+        doc2 = self._create_doc2(
+            document=doc,
+            parent=self._get_parent_directory(doc),
+            name=data['name'],
+            doctype=data['type'],
+            description=data['description'],
+            data=notebook.get_json()
+        )
+        self.imported_docs.append(doc2)
+
+    # Convert Workflow documents
+    docs = self._get_unconverted_docs(Workflow)
+    for doc in docs:
+      if doc.content_object:
+        data = doc.content_object.data_dict
+        data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+        doc2 = self._create_doc2(
+            document=doc,
+            parent=self._get_parent_directory(doc),
+            name=doc.name,
+            doctype='link-workflow',
+            description=doc.description,
+            data=json.dumps(data)
+        )
+        self.imported_docs.append(doc2)
+
+    # Convert PigScript documents
+    docs = self._get_unconverted_docs(PigScript)
+    for doc in docs:
+      if doc.content_object:
+        data = doc.content_object.dict
+        data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+        doc2 = self._create_doc2(
+            document=doc,
+            parent=self._get_parent_directory(doc),
+            name=doc.name,
+            doctype='link-pigscript',
+            description=doc.description,
+            data=json.dumps(data)
+        )
+        self.imported_docs.append(doc2)
+
+    # Add converted docs to root directory
+    if self.imported_docs:
+      LOG.info('Successfully imported %d documents' % len(self.imported_docs))
+
+
+  def _get_unconverted_docs(self, content_type):
+    docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
+    docs = docs.exclude(tags__in=[
+      DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
+      DocumentTag.objects.get_history_tag(user=self.user), # No history yet
+      DocumentTag.objects.get_example_tag(user=self.user), # No examples
+      self.imported_tag # No already imported docs
+    ])
+    return docs
+
+
+  def _get_parent_directory(self, document):
+    parent_dir = self.home_dir
+    project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
+    if project_tags.exists():
+      first_tag = project_tags[0]
+      parent_dir = Directory.objects.get_or_create(owner=self.user, name=first_tag.tag, parent_directory=self.home_dir)
+    return parent_dir
+
+
+  def _create_doc2(self, document, parent, name, doctype, description=None, data=None):
+    with transaction.atomic():
+      doc2 = Document2.objects.create(
+        owner=self.user,
+        parent_directory=parent,
+        name=name,
+        type=doctype,
+        description=description,
+        data=data
+      )
+      document.add_tag(self.imported_tag)
+      document.save()
+      return doc2

+ 3 - 2
desktop/core/src/desktop/views.py

@@ -40,7 +40,7 @@ import desktop.conf
 import desktop.log.log_buffer
 
 from desktop.api import massaged_tags_for_json, massaged_documents_for_json, _get_docs
-from desktop.api2 import _convert_documents
+from desktop.converters import DocumentConverter
 from desktop.lib import django_mako
 from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig
 from desktop.lib.django_util import JsonResponse, login_notrequired, render_json, render
@@ -74,7 +74,8 @@ def home(request):
 
 
 def home2(request):
-  _convert_documents(request.user)
+  converter = DocumentConverter(request.user)
+  converter.convert()
 
   apps = appmanager.get_apps_dict(request.user)