Bläddra i källkod

[doc2] Import Hive and Impala queries from document1 to document2

Old format queries are not deleted, similarly to Oozie v1 to v2.
The import happens by user when then go on the new home page.
When a document 1 is imported, it will be tagged so that it is not re-imported again.
Pig and Job Designers docs will be linked shortly with a doc2.
History docs are not imported for now.
Tests are coming up when it stabilizes.
Romain Rigaux 10 år sedan
förälder
incheckning
9e033f7

+ 37 - 1
desktop/core/src/desktop/api2.py

@@ -30,10 +30,11 @@ from django.utils import html
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
 
+from beeswax.models import SavedQuery
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.export_csvxls import make_response
 from desktop.lib.i18n import smart_str, force_unicode
-from desktop.models import Document2, Document, Directory
+from desktop.models import Document2, Document, Directory, DocumentTag, import_saved_beeswax_query
 from desktop.lib.exceptions_renderable import PopupException
 from hadoop.fs.hadoopfs import Hdfs
 
@@ -74,6 +75,8 @@ def get_documents(request): # TODO only here for not breaking assist for now
 def get_documents2(request):
   path = request.GET.get('path', '/') # Expects path to be a Directory for now
 
+  _import_documents1(request.user)
+
   try:
     file_doc = Directory.objects.get(owner=request.user, name=path) # TODO perms
   except Directory.DoesNotExist, e:
@@ -90,6 +93,39 @@ def get_documents2(request):
   })
 
 
+def _import_documents1(user):
+  from beeswax.models import HQL, IMPALA, RDBMS
+  docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA]) # TODO RDBMS
+
+  imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
+
+  docs = docs.exclude(tags__in=[
+      DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
+      DocumentTag.objects.get_history_tag(user=user), # No history yet
+      DocumentTag.objects.get_example_tag(user=user), # No examples
+      imported_tag # No already imported docs
+  ])
+
+  root_doc, created = Directory.objects.get_or_create(name='/', type='directory', owner=user)
+  imported_docs = []
+
+  for doc in docs:
+    if doc.content_object:
+      try:
+        notebook = import_saved_beeswax_query(doc.content_object)
+        data = notebook.get_data()
+        notebook_doc = Document2.objects.create(name=data['name'], type='query-%s' % data['type'], owner=user, data=notebook.get_json())
+
+        doc.add_tag(imported_tag)
+        doc.save()
+        imported_docs.append(notebook_doc)
+      except Exception, e:
+        raise e
+
+  if imported_docs:
+    root_doc.dependencies.add(*imported_docs)
+
+
 @api_error_handler
 def get_document(request):
   if request.GET.get('id'):

+ 6 - 2
desktop/core/src/desktop/models.py

@@ -91,6 +91,9 @@ class DocumentTagManager(models.Manager):
   def get_example_tag(self, user):
     return self._get_tag(user, DocumentTag.EXAMPLE)
 
+  def get_imported2_tag(self, user):
+    return self._get_tag(user, DocumentTag.IMPORTED2)
+
   def tag(self, owner, doc_id, tag_name='', tag_id=None):
     try:
       tag = DocumentTag.objects.get(id=tag_id, owner=owner)
@@ -151,8 +154,9 @@ class DocumentTag(models.Model):
   TRASH = 'trash' # There when the document is trashed
   HISTORY = 'history' # There when the document is a submission history
   EXAMPLE = 'example' # Hue examples
+  IMPORTED2 = 'imported2' # Was imported to document2
 
-  RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE)
+  RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)
 
   objects = DocumentTagManager()
 
@@ -996,7 +1000,7 @@ def get_data_link(meta):
   return link
 
 
-def import_beeswax_query(bquery):
+def import_saved_beeswax_query(bquery):
   design = bquery.get_design()
 
   return make_notebook(

+ 10 - 0
desktop/core/src/desktop/templates/home.mako

@@ -93,6 +93,16 @@ ${ commonheader(_('Welcome Home'), "home", user) | n,unicode }
            </li>
         </ul>
       </div>
+      <div class="nav-collapse pull-right">
+        <ul class="nav">
+          <li class="currentApp">
+            <a href="${ url('desktop.views.home2') }">
+              <img src="${ static('desktop/art/home.png') }" class="app-icon" />
+              ${ _('New Home') }
+            </a>
+           </li>
+        </ul>
+      </div>
     </div>
   </div>
 </div>

+ 11 - 1
desktop/core/src/desktop/templates/home2.mako

@@ -136,12 +136,22 @@ ${ assist.assistPanel() }
       <div class="nav-collapse">
         <ul class="nav">
           <li class="currentApp">
-            <a href="${ url('desktop.views.home') }">
+            <a href="${ url('desktop.views.home2') }">
               <img src="${ static('desktop/art/home.png') }" class="app-icon" />
               ${ _('My documents') }
             </a>
            </li>
         </ul>
+        </div>
+      <div class="nav-collapse pull-right">
+        <ul class="nav">
+          <li class="currentApp">
+            <a href="${ url('desktop.views.home') }">
+              <img src="${ static('desktop/art/home.png') }" class="app-icon" />
+              ${ _('Old home') }
+            </a>
+           </li>
+        </ul>
       </div>
     </div>
   </div>

+ 2 - 2
desktop/core/src/desktop/tests_doc2.py

@@ -24,7 +24,7 @@ from desktop.lib.test_utils import grant_access
 
 from beeswax.models import SavedQuery
 from beeswax.design import hql_query
-from desktop.models import import_beeswax_query
+from desktop.models import import_saved_beeswax_query
 
 
 class TestDocument2(object):
@@ -51,7 +51,7 @@ class TestDocument2(object):
         desc='Example of old format'
     )
 
-    new_query = import_beeswax_query(old_query)
+    new_query = import_saved_beeswax_query(old_query)
     new_query_data = new_query.get_data()
 
     assert_equal('query-hive', new_query_data['type'])