Просмотр исходного кода

HUE-3314 [editor] Save result as csv HDFS file

Romain Rigaux 9 лет назад
Родитель
Сommit
943fa8c

+ 2 - 4
desktop/core/src/desktop/static/desktop/js/assist/assistHelper.js

@@ -634,8 +634,7 @@
             pollRefresh(url);
           }, 1000);
         }
-      })
-        .fail(self.assistErrorCallback(options));
+      }).fail(self.assistErrorCallback(options));
     };
 
     $.post("/" + (options.sourceType == "hive" ? "beeswax" : options.sourceType) + "/api/analyze/" + options.databaseName + "/" + options.tableName + "/"  + (options.columnName || ""), function (data) {
@@ -644,8 +643,7 @@
       } else {
         self.assistErrorCallback(options)(data);
       }
-    })
-      .fail(self.assistErrorCallback(options));
+    }).fail(self.assistErrorCallback(options));
   };
 
   /**

+ 93 - 1
desktop/core/src/desktop/templates/ko_components.mako

@@ -522,8 +522,59 @@ from desktop.views import _ko
             <i class="fa fa-fw fa-file-excel-o"></i> ${ _('Excel') }
           </a>
         </li>
+        <liv>
+          <a class="inactive-action download" href="javascript:void(0)" data-bind="click: function() { $('#saveResultsModal').modal('show'); }" title="${ _('Save the results to HDFS or a new table') }">
+            <i class="fa fa-save"></i> ${ _('Export') }
+          </a>
+        </li>
       </ul>
     </div>
+
+    <div id="saveResultsModal" class="modal hide fade">
+      <div class="loader hide">
+        <div class="overlay"></div>
+        <!--[if !IE]><!--><i class="fa fa-spinner fa-spin"></i><!--<![endif]-->
+        <!--[if IE]><img class="spinner" src="${ static('desktop/art/spinner-big-inverted.gif') }"/><![endif]-->
+      </div>
+
+      <div class="modal-header">
+        <a href="#" class="close" data-dismiss="modal">&times;</a>
+        <h3>${_('Save Query Results')}</h3>
+      </div>
+      <div class="modal-body" style="padding: 4px">
+        <form id="saveResultsForm" method="POST" class="form form-inline">
+          ${ csrf_token(request) | n,unicode }
+          <fieldset>
+            <div class="control-group">
+              <div class="controls">
+                <label class="radio">
+                  <input data-bind="checked: saveTarget() == 'hdfs-file'" type="radio" name="save-results-type">
+                  &nbsp;${ _('In an HDFS file') }
+                </label>
+                <span data-bind="visible: saveTarget() == 'hdfs-file'">
+                  <input data-bind="value: savePath" type="text" name="target_file" placeholder="${_('Path to CSV file')}" class="pathChooser">
+                </span>
+                <label class="radio" data-bind="visible: saveTarget() == 'hdfs-file'">
+                  <input data-bind="checked: saveOverwrite" type="checkbox" name="overwrite">
+                  ${ _('Overwrite') }
+                </label>
+              </div>
+            </div>
+          </fieldset>
+        </form>
+        <div id="hdfs-directory-help-content" class="hide">
+          <p>${ _("Use this option if you have a large result. It will rerun the entire query and save the results to the chosen HDFS directory.") }</p>
+        </div>
+      </div>
+      <div class="modal-footer">
+        % if app_name != 'impala':
+        <a id="save-results-advanced" href="javascript:void(0)" class="pull-left">${ _('Show advanced fields') }</a>
+        % endif
+        <a id="save-results-simple" href="javascript:void(0)" class="pull-left hide">${ _('Hide advanced fields') }</a>
+        <button class="btn" data-dismiss="modal">${_('Cancel')}</button>
+        <button data-bind="click: trySaveResults" class="btn btn-primary disable-feedback">${_('Save')}</button>
+      </div>
+    </div>
   </script>
 
   <script type="text/javascript" charset="utf-8">
@@ -539,7 +590,48 @@ from desktop.views import _ko
         self.$downloadForm = $(element).find(".download-form");
         self.snippet = params.snippet;
         self.notebook = params.notebook;
-      }
+
+        self.saveTarget = ko.observable('hdfs-file');
+        self.savePath = ko.observable('');
+        self.saveOverwrite = ko.observable(true);
+
+        self.trySaveResults = function() {
+          self.saveResults();
+
+          $("#saveResultsModal button.btn-primary").button('loading');
+          $("#saveResultsModal .loader").show();
+        };
+
+        self.saveResults = function() {
+          var self = this;
+
+           $.post("${ url('notebook:export_result') }", {
+            notebook: ko.mapping.toJSON(self.notebook.getContext()),
+            snippet: ko.mapping.toJSON(self.snippet.getContext()),
+            format: ko.mapping.toJSON(self.saveTarget()),
+            destination: ko.mapping.toJSON(self.savePath()),
+            overwrite: ko.mapping.toJSON(self.saveOverwrite())
+          },
+          function(data) {
+            if (data.status == 0) {
+              if (data.id) {
+                // Not used yet
+                self.design.watch.url(data.watch_url);
+                self.watchQueryLoop(redirect_fn);
+              } else {
+                window.location.href = data.success_url;
+              }
+            } else {
+              $(document).trigger('error', data.message);
+            }
+          }).fail(function (xhr, textStatus, errorThrown) {
+            $(document).trigger("error", xhr.responseText);
+          }).done(function() {
+            $("#saveResultsModal button.btn-primary").button('reset');
+            $("#saveResultsModal .loader").hide();
+          });
+        };
+      };
 
       DownloadResultsViewModel.prototype.download = function (format) {
         var self = this;

+ 29 - 6
desktop/libs/notebook/src/notebook/api.py

@@ -18,6 +18,7 @@
 import json
 import logging
 
+from django.forms import ValidationError
 from django.http import HttpResponseBadRequest, HttpResponseRedirect
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_GET, require_POST
@@ -383,12 +384,8 @@ def get_sample_data(request, server=None, database=None, table=None):
   notebook = json.loads(request.POST.get('notebook', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
 
-  try:
-    sample_data = get_api(request, snippet).get_sample_data(snippet, database, table)
-    response.update(sample_data)
-  except QueryExpired, e:
-    LOG.debug('get_sample_data query expired: %s' % e)
-    pass
+  sample_data = get_api(request, snippet).get_sample_data(snippet, database, table)
+  response.update(sample_data)
 
   response['status'] = 0
 
@@ -451,3 +448,29 @@ def github_callback(request):
     return HttpResponseRedirect(redirect_base + "0&github_fetch=" + request.session['github_callback_fetch'])
   else:
     return HttpResponseRedirect(redirect_base + "-1&github_fetch=" + request.session['github_callback_fetch'])
+
+
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def export_result(request):
+  response = {'status': -1, 'message': _('Exporting result failed.')}
+
+  # Passed by check_document_access_permission but unused by APIs
+  notebook = json.loads(request.POST.get('notebook', '{}'))
+  snippet = json.loads(request.POST.get('snippet', '{}'))
+  format = json.loads(request.POST.get('format', 'hdfs-file'))
+  destination = json.loads(request.POST.get('destination', ''))
+  overwrite = json.loads(request.POST.get('overwrite', False))
+
+  if format == 'hdfs-file':
+    if overwrite and request.fs.exists(destination):
+      if request.fs.isfile(destination):
+        request.fs.do_as_user(request.user.username, request.fs.rmtree, destination)
+      else:
+        raise ValidationError(_("The target path is a directory"))
+
+    response['success_url'] = get_api(request, snippet).export_data_as_csv_file(snippet, destination, overwrite)
+    response['status'] = 0
+
+  return JsonResponse(response)

+ 3 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -94,7 +94,7 @@ def get_api(request, snippet):
   interface = interpreter['interface']
 
   if interface == 'hiveserver2':
-    return HS2Api(user=request.user)
+    return HS2Api(user=request.user, request=request)
   elif interface == 'livy':
     return SparkApi(request.user)
   elif interface == 'livy-batch':
@@ -155,3 +155,5 @@ class Api(object):
 
   def get_jobs(self, notebook, snippet, logs):
     return []
+
+  def export_data_as_hdfs_file(self, snippet, target_file, overwrite): raise NotImplementedError()

+ 12 - 0
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -33,6 +33,7 @@ LOG = logging.getLogger(__name__)
 try:
   from beeswax import data_export
   from beeswax.api import _autocomplete, _get_sample_data
+  from beeswax.data_export import upload
   from beeswax.design import hql_query, strip_trailing_semicolon, split_statements
   from beeswax import conf as beeswax_conf
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
@@ -331,3 +332,14 @@ class HS2Api(Api):
       name = 'spark-sql'
 
     return dbms.get(self.user, query_server=get_query_server_config(name=name))
+
+
+  @query_error_handler
+  def export_data_as_hdfs_file(self, snippet, target_file, overwrite):
+    db = self._get_db(snippet)
+
+    handle = self._get_handle(snippet)
+
+    upload(target_file, handle, self.request.user, db, self.request.fs)
+
+    return '/filebrowser/view=%s' % target_file

+ 1 - 0
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1252,6 +1252,7 @@ ${ require.config() }
   </div>
 </div>
 
+
 <div id="clearHistoryModal" class="modal hide fade">
   <div class="modal-header">
     <a href="#" class="close" data-dismiss="modal">&times;</a>

+ 2 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -63,6 +63,8 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/notebook/save/?$', 'save_notebook', name='save_notebook'),
   url(r'^api/notebook/open/?$', 'open_notebook', name='open_notebook'),
   url(r'^api/notebook/close/?$', 'close_notebook', name='close_notebook'),
+
+  url(r'^api/notebook/export_result/?$', 'export_result', name='export_result'),
 )
 
 # Github