소스 검색

HUE-4346 [editor] Query History disappeared after upgrade to 3.10

Import everything, not limit, but import only once
Re-import from cleared list in old version
Romain Rigaux 9 년 전
부모
커밋
96836c66a7
2개의 변경된 파일112개의 추가작업 그리고 10개의 파일을 삭제
  1. 55 6
      desktop/core/src/desktop/converters.py
  2. 57 4
      desktop/core/src/desktop/tests_converter.py

+ 55 - 6
desktop/core/src/desktop/converters.py

@@ -17,11 +17,12 @@
 
 import json
 import logging
+import time
 
 from django.db import transaction
 
-from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission, \
-    FilesystemException
+from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission
+from notebook.api import _historify
 from notebook.models import import_saved_beeswax_query
 
 
@@ -51,6 +52,10 @@ class DocumentConverter(object):
         if doc.content_object:
           notebook = import_saved_beeswax_query(doc.content_object)
           data = notebook.get_data()
+
+          if doc.is_historic():
+            data['isSaved'] = False
+
           doc2 = self._create_doc2(
               document=doc,
               doctype=data['type'],
@@ -58,10 +63,49 @@ class DocumentConverter(object):
               description=data['description'],
               data=notebook.get_json()
           )
+
+          if doc.is_historic():
+            doc2.is_history = False
+
+          self.imported_docs.append(doc2)
+    except ImportError, e:
+      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
+
+    # Convert SQL Query history documents
+    try:
+      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+
+      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
+
+      for doc in docs:
+        if doc.content_object:
+          notebook = import_saved_beeswax_query(doc.content_object)
+          data = notebook.get_data()
+
+          data['isSaved'] = False
+          data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
+
+          doc2 = _historify(data, self.user)
+          doc2.last_modified = doc.last_modified
+          doc2.save()
+
           self.imported_docs.append(doc2)
+
+          # Tag for not re-importing
+          Document.objects.link(
+            doc2,
+            owner=doc2.owner,
+            name=doc2.name,
+            description=doc2.description,
+            extra=doc.extra
+          )
+
+          doc.add_tag(self.imported_tag)
+          doc.save()
     except ImportError, e:
       LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
 
+
     # Convert Job Designer documents
     try:
       from oozie.models import Workflow
@@ -107,14 +151,19 @@ class DocumentConverter(object):
       LOG.info('Successfully imported %d documents' % len(self.imported_docs))
 
 
-  def _get_unconverted_docs(self, content_type):
+  def _get_unconverted_docs(self, content_type, with_history=False):
     docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
-    return docs.exclude(tags__in=[
+
+    tags = [
       DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
-      DocumentTag.objects.get_history_tag(user=self.user), # No history yet
       DocumentTag.objects.get_example_tag(user=self.user), # No examples
       self.imported_tag # No already imported docs
-    ])
+    ]
+
+    if not with_history:
+      tags.append(DocumentTag.objects.get_history_tag(user=self.user)) # No history yet
+
+    return docs.exclude(tags__in=tags)
 
 
   def _get_parent_directory(self, document):

+ 57 - 4
desktop/core/src/desktop/tests_converter.py

@@ -24,12 +24,12 @@ from django.contrib.auth.models import User
 from desktop.converters import DocumentConverter
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
-from desktop.models import Directory, Document, Document2, DocumentPermission, Document2Permission, DocumentTag
+from desktop.models import Directory, Document, Document2, DocumentPermission, DocumentTag
 from librdbms.design import SQLdesign
 
 from beeswax.models import SavedQuery
 from beeswax.design import hql_query
-from pig.models import create_or_update_script, PigScript
+from pig.models import create_or_update_script
 from useradmin.models import get_default_user_group
 
 
@@ -68,14 +68,30 @@ class TestDocumentConverter(object):
     )
     doc = Document.objects.link(query, owner=query.owner, extra=query.type, name=query.name, description=query.desc)
 
+    query2 = SavedQuery.objects.create(
+        type=SavedQuery.TYPES_MAPPING['hql'],
+        owner=self.user,
+        data=design.dumps(),
+        name='Hive query history',
+        desc='Test Hive query history',
+        is_auto=True
+    )
+    doch = Document.objects.link(query2, owner=query2.owner, extra=query2.type, name=query2.name, description=query2.desc)
+    doch.add_to_history()
+
     try:
       # Test that corresponding doc2 is created after convert
-      assert_false(Document2.objects.filter(owner=self.user, type='query-hive').exists())
+      assert_equal(0, Document2.objects.filter(owner=self.user, type='query-hive').count())
 
       converter = DocumentConverter(self.user)
       converter.convert()
 
-      doc2 = Document2.objects.get(owner=self.user, type='query-hive')
+      assert_equal(2, Document2.objects.filter(owner=self.user, type='query-hive').count())
+
+      #
+      # Query
+      #
+      doc2 = Document2.objects.get(owner=self.user, type='query-hive', is_history=False)
 
       # Verify Document2 attributes
       assert_equal(doc.name, doc2.data_dict['name'])
@@ -97,8 +113,45 @@ class TestDocumentConverter(object):
 
       # Verify default properties
       assert_true(doc2.data_dict['isSaved'])
+
+
+      #
+      # Query History
+      #
+      doc2 = Document2.objects.get(owner=self.user, type='query-hive', is_history=True)
+
+      # Verify Document2 attributes
+      assert_equal(doch.name, doc2.data_dict['name'])
+      assert_equal(doch.description, doc2.data_dict['description'])
+
+      # Verify session type
+      assert_equal('hive', doc2.data_dict['sessions'][0]['type'])
+
+      # Verify snippet values
+      assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
+      assert_equal(sql, doc2.data_dict['snippets'][0]['statement'])
+      assert_equal(sql, doc2.data_dict['snippets'][0]['statement_raw'])
+      assert_equal('etl', doc2.data_dict['snippets'][0]['database'])
+
+      # Verify snippet properties
+      assert_equal(settings, doc2.data_dict['snippets'][0]['properties']['settings'])
+      assert_equal(file_resources, doc2.data_dict['snippets'][0]['properties']['files'])
+      assert_equal(functions, doc2.data_dict['snippets'][0]['properties']['functions'])
+
+      # Verify default properties
+      assert_false(doc2.data_dict['isSaved'])
+
+
+      #
+      # Check that we don't re-import again
+      #
+      converter = DocumentConverter(self.user)
+      converter.convert()
+
+      assert_equal(2, Document2.objects.filter(owner=self.user, type='query-hive').count())
     finally:
       query.delete()
+      query2.delete()
 
 
   def test_convert_impala_query(self):