Bläddra i källkod

HUE-1660 [core] API for export/import all stored scripts

Export or import Hue documents2. Implemented genericly and for Spark Notebooks.

If a document uuid already exists it will be replaced. A non admin cannot export documents he does not own (later will extend to document he can read),
and cannot import documents of someone else.

Exports are done by using the id of the documents, this keeps the data simple.
Imports are done by using the uuid of the documents, this guarantees that each document is unique, independently of its DB id.

Export
------
/desktop/api2/doc/export?documents=[50093,50091]

Parameters:
Json list of pk of documents2
Returns:
json file (default)
?format=json: json response
?format=zip: zip file with json data and text representation of compatible documents, e.g. oozie (later), SQL (later), Notebooks...

Import
------
/desktop/api2/doc/import
POST:
With a json document named 'documents', same as the one coming from export in json.

or

FILE:
With a file named 'documents' that containts the content of some exported documents in json.
With a 'redirect' input.

Examples
-------
Examples in notebooks.mako
Romain Rigaux 10 år sedan
förälder
incheckning
b5cd9bf4fb

+ 3 - 0
apps/spark/src/spark/models.py

@@ -79,6 +79,9 @@ class Notebook():
 
     return _data
 
+  def get_str(self):
+    return '\n\n'.join([snippet['statement_raw'] for snippet in self.get_data()['snippets']])
+
 
 def get_api(user, snippet):
   if snippet['type'] in ('hive', 'impala', 'spark-sql'):

+ 32 - 3
apps/spark/src/spark/templates/notebooks.mako

@@ -49,15 +49,21 @@ ${ commonheader(_("Notebooks"), "spark", user, "60px") | n,unicode }
           <i class="fa fa-times"></i> ${ _('Delete') }
         </a>
 
+        <a data-bind="click: function() { atLeastOneSelected() ? exportDocuments() : void(0) }, css: {'btn': true, 'disabled': ! atLeastOneSelected() }">
+          <i class="fa fa-upload"></i> ${ _('Export') }
+        </a>
       </div>
     </%def>
 
     <%def name="creation()">
       <a href="${ url('spark:new') }" class="btn"><i class="fa fa-plus-circle"></i> ${ _('Create') }</a>
+      <a data-bind="click: function() { $('#import-documents').modal('show'); }" class="btn">
+        <i class="fa fa-download"></i> ${ _('Import') }
+      </a>
     </%def>
   </%actionbar:render>
 
-       
+
   <table id="notebookTable" class="table datatables">
     <thead>
       <tr>
@@ -71,7 +77,7 @@ ${ commonheader(_("Notebooks"), "spark", user, "60px") | n,unicode }
     <tbody data-bind="foreach: { data: jobs }">
       <tr>
         <td data-bind="click: $root.handleSelect" class="center" style="cursor: default" data-row-selector-exclude="true">
-          <div data-bind="css: { 'hueCheckbox': true, 'fa': true, 'fa-check': isSelected }" data-row-selector-exclude="true"></div>          
+          <div data-bind="css: { 'hueCheckbox': true, 'fa': true, 'fa-check': isSelected }" data-row-selector-exclude="true"></div>
           <a data-bind="attr: { 'href': '${ url('spark:editor') }?notebook=' + id() }" data-row-selector="true"></a>
         </td>
         <td data-bind="text: name"></td>
@@ -111,6 +117,24 @@ ${ commonheader(_("Notebooks"), "spark", user, "60px") | n,unicode }
   </form>
 </div>
 
+<div id="export-documents" class="modal hide">
+  <form method="POST" action="/desktop/api2/doc/export" style="display: inline">
+    ${ csrf_token(request) | n,unicode }
+    <input type="hidden" name="documents"/>
+  </form>
+</div>
+
+<div id="import-documents" class="modal hide fade">
+  <form method="POST" action="/desktop/api2/doc/import" style="display: inline" enctype="multipart/form-data">
+    ${ csrf_token(request) | n,unicode }
+    <input type="file" name="documents" accept="application/json"/>
+    <input type="hidden" name="redirect" value="${ request.get_full_path() }"/>
+    </br>
+    <a href="#" class="btn" data-dismiss="modal">${ _('Cancel') }</a>
+    <input type="submit" class="btn btn-danger" value="${ _('Import') }"/>
+  </form>
+</div>
+
 
 </div>
 
@@ -176,8 +200,13 @@ ${ commonshare() | n,unicode }
       });
     };
 
+    self.exportDocuments = function() {
+      $('#export-documents').find('input[name=\'documents\']').val(ko.mapping.toJSON($.map(self.selectedJobs(), function(doc) { return doc.id(); })));
+      $('#export-documents').find('form').submit();
+    };
+
     self.prepareShareModal = function() {
-     shareViewModel.setDocId(self.selectedJobs()[0].doc1_id());
+      shareViewModel.setDocId(self.selectedJobs()[0].doc1_id());
       openShareModal();
     };
   }

+ 95 - 13
desktop/core/src/desktop/api2.py

@@ -15,21 +15,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import itertools
 import logging
 import json
+import tempfile
 import time
+import StringIO
+import zipfile
 
-from collections import defaultdict
-
-from django.core.urlresolvers import reverse
-
+from django.core import management
+from django.shortcuts import redirect
 from django.utils import html
-from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import JsonResponse
-from desktop.lib.i18n import force_unicode
-from desktop.models import Document2, DocumentTag
+from desktop.lib.export_csvxls import make_response
+from desktop.lib.i18n import smart_str
+from desktop.models import Document2, Document
+from django.http import HttpResponse
 
 
 LOG = logging.getLogger(__name__)
@@ -51,23 +52,104 @@ def _massage_doc_for_json(document, user, with_data=False):
   massaged_doc = {
     'id': document.id,
     'uuid': document.uuid,
-    
+
     'owner': document.owner.username,
-    'type': html.conditional_escape(document.type),    
+    'type': html.conditional_escape(document.type),
     'name': html.conditional_escape(document.name),
-    'description': html.conditional_escape(document.description),    
+    'description': html.conditional_escape(document.description),
 
     'isMine': document.owner == user,
     'lastModified': document.last_modified.strftime("%x %X"),
     'lastModifiedInMillis': time.mktime(document.last_modified.timetuple()),
     'version': document.version,
     'is_history': document.is_history,
-    
+
     # tags
     # dependencies
   }
-  
+
   if with_data:
     massaged_doc['data'] = document.data_dict
 
   return massaged_doc
+
+
+def export_documents(request):
+  if request.GET.get('documents'):
+    selection = json.loads(request.GET.get('documents'))
+  else:
+    selection = json.loads(request.POST.get('documents'))
+
+  # If non admin, only export documents the user owns
+  docs = Document2.objects
+  if request.user.is_superuser:
+    docs = docs.filter(owner=request.user)
+  docs = docs.filter(id__in=selection).order_by('-id')
+  doc_ids = docs.values_list('id', flat=True)
+
+  f = StringIO.StringIO()
+
+  if doc_ids:
+    doc_ids = ','.join(map(str, doc_ids))
+    management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_keys=True, verbosity=2, stdout=f)
+
+  if request.GET.get('format') == 'json':
+    return JsonResponse(f.getvalue(), safe=False)
+  elif request.GET.get('format') == 'zip':
+    zfile = zipfile.ZipFile(f, 'w')
+    zfile.writestr("hue.json", f.getvalue())
+    for doc in docs:
+      if doc.type == 'notebook':
+        try:
+          from spark.models import Notebook
+          zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
+        except Exception, e:
+          print e
+          LOG.exception(e)
+    zfile.close()
+    response = HttpResponse(content_type="application/zip")
+    response["Content-Length"] = len(f.getvalue())
+    response['Content-Disposition'] = 'attachment; filename="hue.zip"'
+    response.write(f.getvalue())
+    return response
+  else:
+    return make_response(f.getvalue(), 'json', 'hue')
+
+
+
+def import_documents(request):
+  if request.FILES.get('documents'):
+    documents = request.FILES['documents'].read()
+  else:
+    documents = json.loads(request.POST.get('documents'))
+
+  documents = json.loads(documents)
+  docs = []
+
+  for doc in documents:
+    if not request.user.is_superuser:
+      doc['fields']['owner'] = [request.user.username]
+    owner = doc['fields']['owner'][0]
+
+    doc['fields']['tags'] = []
+
+    if Document2.objects.filter(uuid=doc['fields']['uuid'], owner__username=owner).exists():
+      doc['pk'] = Document2.objects.get(uuid=doc['fields']['uuid'], owner__username=owner).pk
+    else:
+      doc['pk'] = None
+
+    docs.append(doc)
+
+  f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
+  f.write(json.dumps(docs))
+  f.flush()
+
+  stdout = StringIO.StringIO()
+  management.call_command('loaddata', f.name, stdout=stdout)
+
+  Document.objects.sync()
+
+  if request.POST.get('redirect'):
+    return redirect(request.POST.get('redirect'))
+  else:
+    return JsonResponse({'message': stdout.getvalue()})

+ 2 - 0
desktop/core/src/desktop/lib/export_csvxls.py

@@ -98,6 +98,8 @@ def make_response(generator, format, name, encoding=None):
     content_type = 'application/csv'
   elif format == 'xls':
     content_type = 'application/xls'
+  elif format == 'json':
+    content_type = 'application/json'
   else:
     raise Exception("Unknown format: %s" % format)
 

+ 1 - 0
desktop/core/src/desktop/templates/common_header.mako

@@ -522,6 +522,7 @@ from django.utils.translation import ugettext as _
                  <li class="divider"></li>
                  % if 'search' in apps:
                  <li><a href="${ url('search:new_search') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-plus" style="vertical-align: middle"></i> ${ _('Dashboard') }</a></li>
+                 <li><a href="${ url('search:admin_collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-tags" style="vertical-align: middle"></i> ${ _('Dashboards') }</a></li>
                  % endif
                  % if 'indexer' in apps:
                  <li><a href="${ url('indexer:collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-database" style="vertical-align: middle"></i> ${ _('Indexes') }</a></li>

+ 2 - 0
desktop/core/src/desktop/urls.py

@@ -96,6 +96,8 @@ dynamic_patterns += patterns('desktop.api',
 
 dynamic_patterns += patterns('desktop.api2',
   (r'^desktop/api2/doc/get$', 'get_document'),
+  (r'^desktop/api2/doc/export$', 'export_documents'),
+  (r'^desktop/api2/doc/import$', 'import_documents'),
 )
 
 dynamic_patterns += patterns('useradmin.views',