Browse Source

HUE-6161 [doc2] Move document conversion upgrade into a migration

krish 8 years ago
parent
commit
f71958d

+ 87 - 66
desktop/core/src/desktop/converters.py

@@ -42,6 +42,7 @@ class DocumentConverter(object):
     self.home_dir = Document2.objects.create_user_directories(self.user)
     self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
     self.imported_docs = []
+    self.failed_docs = []
 
 
   def convert(self):
@@ -51,25 +52,29 @@ class DocumentConverter(object):
 
       docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
       for doc in docs:
-        if doc.content_object:
-          notebook = import_saved_beeswax_query(doc.content_object)
-          data = notebook.get_data()
-
-          if doc.is_historic():
-            data['isSaved'] = False
-
-          doc2 = self._create_doc2(
-              document=doc,
-              doctype=data['type'],
-              name=data['name'],
-              description=data['description'],
-              data=notebook.get_json()
-          )
+        try:
+          if doc.content_object:
+            notebook = import_saved_beeswax_query(doc.content_object)
+            data = notebook.get_data()
+
+            if doc.is_historic():
+              data['isSaved'] = False
+
+            doc2 = self._create_doc2(
+                document=doc,
+                doctype=data['type'],
+                name=data['name'],
+                description=data['description'],
+                data=notebook.get_json()
+            )
 
-          if doc.is_historic():
-            doc2.is_history = False
+            if doc.is_historic():
+              doc2.is_history = False
 
-          self.imported_docs.append(doc2)
+            self.imported_docs.append(doc2)
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import SavedQuery document id: %d' % doc.id)
     except ImportError:
       LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
 
@@ -80,36 +85,40 @@ class DocumentConverter(object):
       docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
 
       for doc in docs:
-        if doc.content_object:
-          notebook = import_saved_beeswax_query(doc.content_object)
-          data = notebook.get_data()
-
-          data['isSaved'] = False
-          data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
-
-          with transaction.atomic():
-            doc2 = _historify(data, self.user)
-            doc2.last_modified = doc.last_modified
-
-            # save() updates the last_modified to current time. Resetting it using update()
-            doc2.save()
-            Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
-
-            self.imported_docs.append(doc2)
+        try:
+          if doc.content_object:
+            notebook = import_saved_beeswax_query(doc.content_object)
+            data = notebook.get_data()
 
-            # Tag for not re-importing
-            Document.objects.link(
-              doc2,
-              owner=doc2.owner,
-              name=doc2.name,
-              description=doc2.description,
-              extra=doc.extra
-            )
-
-            doc.add_tag(self.imported_tag)
-            doc.save()
+            data['isSaved'] = False
+            data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
+
+            with transaction.atomic():
+              doc2 = _historify(data, self.user)
+              doc2.last_modified = doc.last_modified
+
+              # save() updates the last_modified to current time. Resetting it using update()
+              doc2.save()
+              Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
+
+              self.imported_docs.append(doc2)
+
+              # Tag for not re-importing
+              Document.objects.link(
+                doc2,
+                owner=doc2.owner,
+                name=doc2.name,
+                description=doc2.description,
+                extra=doc.extra
+              )
+
+              doc.add_tag(self.imported_tag)
+              doc.save()
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import history document id: %d' % doc.id)
     except ImportError, e:
-      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
+      LOG.warn('Cannot convert history documents: beeswax app is not installed')
 
 
     # Convert Job Designer documents
@@ -119,16 +128,20 @@ class DocumentConverter(object):
       # TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
       docs = self._get_unconverted_docs(Workflow)
       for doc in docs:
-        if doc.content_object:
-          data = doc.content_object.data_dict
-          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
-          doc2 = self._create_doc2(
-              document=doc,
-              doctype='link-workflow',
-              description=doc.description,
-              data=json.dumps(data)
-          )
-          self.imported_docs.append(doc2)
+        try:
+          if doc.content_object:
+            data = doc.content_object.data_dict
+            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+            doc2 = self._create_doc2(
+                document=doc,
+                doctype='link-workflow',
+                description=doc.description,
+                data=json.dumps(data)
+            )
+            self.imported_docs.append(doc2)
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
     except ImportError, e:
       LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
 
@@ -139,22 +152,30 @@ class DocumentConverter(object):
       # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
       docs = self._get_unconverted_docs(PigScript)
       for doc in docs:
-        if doc.content_object:
-          data = doc.content_object.dict
-          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
-          doc2 = self._create_doc2(
-              document=doc,
-              doctype='link-pigscript',
-              description=doc.description,
-              data=json.dumps(data)
-          )
-          self.imported_docs.append(doc2)
+        try:
+          if doc.content_object:
+            data = doc.content_object.dict
+            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+            doc2 = self._create_doc2(
+                document=doc,
+                doctype='link-pigscript',
+                description=doc.description,
+                data=json.dumps(data)
+            )
+            self.imported_docs.append(doc2)
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import Pig document id: %d' % doc.id)
     except ImportError, e:
       LOG.warn('Cannot convert Pig documents: pig app is not installed')
 
     # Add converted docs to root directory
     if self.imported_docs:
-      LOG.info('Successfully imported %d documents' % len(self.imported_docs))
+      LOG.info('Successfully imported %d documents for user: %s' % (len(self.imported_docs), self.user.username))
+
+    # Log docs that failed to import
+    if self.failed_docs:
+      LOG.error('Failed to import %d document(s) for user: %s - %s' % (len(self.failed_docs), self.user.username, ([doc.id for doc in self.failed_docs])))
 
     # Set is_trashed field for old documents with is_trashed=None
     docs = Document2.objects.filter(owner=self.user, is_trashed=None).exclude(is_history=True)

+ 45 - 0
desktop/core/src/desktop/management/commands/convert_documents.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+from django.contrib.auth.models import User
+from django.core.management.base import NoArgsCommand
+from django.db import transaction
+
+from desktop.converters import DocumentConverter
+
+class Command(NoArgsCommand):
+
+  def handle_noargs(self, **options):
+    self.stdout.write('Starting document conversions...\n')
+    try:
+      with transaction.atomic():
+        users = User.objects.all()
+        logging.info("Starting document conversions for %d users" % len(users))
+        for index, user in enumerate(users):
+          logging.info("Starting document conversion for user %d: %s" % (index, user.username))
+
+          start_time = time.time()
+          converter = DocumentConverter(user)
+          converter.convert()
+          logging.info("Document conversions for user:%s took %.3f seconds" % (user.username, time.time() - start_time))
+    except Exception, e:
+      logging.exception("Failed to execute the document conversions.")
+
+    self.stdout.write('Finished running document conversions.\n')

+ 157 - 0
desktop/core/src/desktop/migrations/0027_document_conversions.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from south.v2 import SchemaMigration
+
+import desktop.management.commands.convert_documents
+
+LOG = logging.getLogger(__name__)
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Earlier we did the document conversions from Doc1 to Doc2 upon loading
+        # the home page of a user. That approach had certain flaws like shared
+        # documents didn't show up until the owner logged in and opened his home
+        # page. Also, home page load time was affected when the conversions failed
+        # and loading the home page retried the conversions every single time.
+        # This migration handles the document conversion of all users at
+        # the same time preventing such flaws.
+
+        desktop.management.commands.convert_documents.Command().execute()
+
+    def backwards(self, orm):
+        pass
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'desktop.defaultconfiguration': {
+            'Meta': {'ordering': "['app', '-is_default', 'user']", 'object_name': 'DefaultConfiguration'},
+            'app': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'defaultconfiguration_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_default': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        u'desktop.document': {
+            'Meta': {'unique_together': "(('content_type', 'object_id'),)", 'object_name': 'Document'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc_owner'", 'to': u"orm['auth.User']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['desktop.DocumentTag']", 'db_index': 'True', 'symmetrical': 'False'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'})
+        },
+        u'desktop.document2': {
+            'Meta': {'ordering': "['-last_modified', 'name']", 'unique_together': "(('uuid', 'version', 'is_history'),)", 'object_name': 'Document2'},
+            'data': ('django.db.models.fields.TextField', [], {'default': "'{}'"}),
+            'dependencies': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'related_name': "'dependents'", 'symmetrical': 'False', 'to': u"orm['desktop.Document2']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_history': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'is_managed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'is_trashed': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc2_owner'", 'to': u"orm['auth.User']"}),
+            'parent_directory': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': u"orm['desktop.Document2']"}),
+            'search': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '32', 'db_index': 'True'}),
+            'uuid': ('django.db.models.fields.CharField', [], {'default': "'34efd754-13bb-46c1-b8da-f29627658d11'", 'max_length': '36', 'db_index': 'True'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'db_index': 'True'})
+        },
+        u'desktop.document2permission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'Document2Permission'},
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document2']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission2_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission2_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documentpermission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'DocumentPermission'},
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documenttag': {
+            'Meta': {'unique_together': "(('owner', 'tag'),)", 'object_name': 'DocumentTag'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'tag': ('django.db.models.fields.SlugField', [], {'max_length': '50'})
+        },
+        u'desktop.settings': {
+            'Meta': {'object_name': 'Settings'},
+            'collect_usage': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tours_and_tutorials': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'})
+        },
+        u'desktop.userpreferences': {
+            'Meta': {'object_name': 'UserPreferences'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'value': ('django.db.models.fields.TextField', [], {'max_length': '4096'})
+        }
+    }
+
+    complete_apps = ['desktop']

+ 1 - 7
desktop/core/src/desktop/views.py

@@ -45,8 +45,8 @@ import desktop.conf
 import desktop.log.log_buffer
 
 from desktop.api import massaged_tags_for_json, massaged_documents_for_json, _get_docs
+
 from desktop.conf import USE_NEW_EDITOR, IS_HUE_4, HUE_LOAD_BALANCER, HTTP_PORT
-from desktop.converters import DocumentConverter
 from desktop.lib import django_mako
 from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig
 from desktop.lib.django_util import JsonResponse, login_notrequired, render_json, render
@@ -113,12 +113,6 @@ def home(request):
 
 
 def home2(request, is_embeddable=False):
-  try:
-    converter = DocumentConverter(request.user)
-    converter.convert()
-  except Exception, e:
-    LOG.warning("Failed to convert and import documents: %s" % e)
-
   apps = appmanager.get_apps_dict(request.user)
 
   return render('home2.mako', request, {