Эх сурвалжийг харах

HUE-2142 [editor] Offer exporting query result in a scalable way

TSV currently
Romain Rigaux 9 жил өмнө
parent
commit
28411f474f

+ 1 - 1
apps/beeswax/src/beeswax/conf.py

@@ -99,7 +99,7 @@ LIST_PARTITIONS_LIMIT = Config(
 # Deprecated
 DOWNLOAD_CELL_LIMIT = Config(
   key='download_cell_limit',
-  default=10000000,
+  default=5000000,
   type=int,
   help=_t('A limit to the number of cells (rows * columns) that can be downloaded from a query '
           '(e.g. - 10K rows * 1K columns = 10M cells.) '

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -255,7 +255,7 @@ from django.utils.translation import ugettext as _
     <div id="changePermissionModal" class="modal hide fade">
       <div class="modal-header">
         <a href="#" class="close" data-dismiss="modal">&times;</a>
-        <h3>${_('Change Permissions:')} </h3>
+        <h3>${_('Change Permissions')} </h3>
       </div>
       <div class="modal-body table-margin">
         <table class="table table-striped">

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -289,4 +289,4 @@ class Api(object):
 
 
 def _get_snippet_name(notebook):
-  return ('%(name)s' if notebook.get('name') else '%(type)s-%(id)s') % notebook
+  return (('%(name)s' if notebook.get('name') else '%(type)s-%(id)s') % notebook).replace('-', '_')

+ 17 - 19
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -475,7 +475,7 @@ class HS2Api(Api):
     return hql, success_url
 
 
-  def export_large_data_to_hdfs1(self, notebook, snippet, destination):
+  def export_large_data_to_hdfs(self, notebook, snippet, destination):
     db = self._get_db(snippet)
 
     response = self._get_current_statement(db, snippet)
@@ -485,27 +485,25 @@ class HS2Api(Api):
     if 'select' not in query.hql_query.strip().lower():
       raise PopupException(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
 
-    db.use(query.database)
-
-    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
-    success_url = '/filebrowser/view=%s' % destination
-
-    return hql, success_url
-
-
-  def export_large_data_to_hdfs(self, notebook, snippet, destination):
-    db = self._get_db(snippet)
-
-    response = self._get_current_statement(db, snippet)
-    session = self._get_session(notebook, snippet['type'])
-    query = self._prepare_hql_query(snippet, response.pop('statement'), session)
+    hql = '''
+DROP TABLE IF EXISTS `%(table)s`;
 
-    if 'select' not in query.hql_query.strip().lower():
-      raise PopupException(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
+CREATE TABLE `%(table)s` ROW FORMAT DELIMITED
+     FIELDS TERMINATED BY '\\t'
+     ESCAPED BY '\\\\'
+     LINES TERMINATED BY '\\n'
+     STORED AS TEXTFILE LOCATION '%(location)s'
+     AS
+%(hql)s;
 
-    db.use(query.database)
+ALTER TABLE `%(table)s` SET TBLPROPERTIES('EXTERNAL'='TRUE');
 
-    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
+DROP TABLE IF EXISTS `%(table)s`;
+    ''' % {
+      'table': _get_snippet_name(notebook),
+      'location': destination,
+      'hql': query.hql_query
+    }
     success_url = '/filebrowser/view=%s' % destination
 
     return hql, success_url

+ 1 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1283,7 +1283,7 @@ var EditorViewModel = (function() {
                   }
                 }
               }
-              if (vm.successUrl()) {
+              if (! self.result.handle().has_more_statements && vm.successUrl()) {
                 window.location.href = vm.successUrl();
               }
             }

+ 18 - 7
desktop/libs/notebook/src/notebook/templates/notebook_ko_components.mako

@@ -220,22 +220,33 @@ except ImportError, e:
         <form id="saveResultsForm" method="POST" class="form form-inline">
           ${ csrf_token(request) | n,unicode }
           <fieldset>
+            <div class="control-group">
+              <div class="controls">
+                 <label class="radio">
+                  <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="hdfs-file">
+                  &nbsp;${ _('File (first %s rows)') % DOWNLOAD_ROW_LIMIT.get() }
+                </label>
+                <div data-bind="visible: saveTarget() == 'hdfs-file'" class="inline">
+                  <input data-bind="value: savePath, valueUpdate:'afterkeydown', filechooser: { value: savePath, isNestedModal: true }, filechooserOptions: { uploadFile: false, skipInitialPathIfEmpty: true, linkMarkup: true }, hdfsAutocomplete: savePath" type="text" name="target_file" placeholder="${_('Path to CSV file')}" class="pathChooser margin-left-10">
+                </div>
+                <label class="radio" data-bind="visible: saveTarget() == 'hdfs-file'">
+                  <input data-bind="checked: saveOverwrite" type="checkbox" name="overwrite">
+                  ${ _('Overwrite') }
+                </label>
+              </div>
+            </div>
             <div class="control-group">
               <div class="controls">
                 <label class="radio">
                   <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="hdfs-directory">
-                  &nbsp;${ _('File') }
+                  &nbsp;${ _('Directory') }
                 </label>
                 <div data-bind="visible: saveTarget() == 'hdfs-directory'" class="inline">
                   <input data-bind="value: savePath, valueUpdate:'afterkeydown', filechooser: { value: savePath, isNestedModal: true }, filechooserOptions: { uploadFile: false, skipInitialPathIfEmpty: true, displayOnlyFolders: true, linkMarkup: true }, hdfsAutocomplete: savePath" type="text" name="target_dir" placeholder="${_('Path to empty directory')}" class="pathChooser margin-left-10 input-xlarge">
                 </div>
-                <div class="inline-block" data-bind="visible: saveTarget() == 'hdfs-directory', tooltip: { title: '${ _ko("Save a large result as CSV") }', placement: 'top' }" style="padding: 8px">
+                <div class="inline-block" data-bind="visible: saveTarget() == 'hdfs-directory', tooltip: { title: '${ _ko("Save a large result as TSV") }', placement: 'top' }" style="padding: 8px">
                   <i class="fa fa-fw fa-question-circle muted"></i>
                 </div>
-                ##<label class="radio" data-bind="visible: saveTarget() == 'hdfs-directory'">
-                ##  <input data-bind="checked: saveOverwrite" type="checkbox" name="overwrite">
-                ##  ${ _('Download') }
-                ##</label>
               </div>
             </div>
             <div class="control-group">
@@ -310,7 +321,7 @@ except ImportError, e:
         self.snippet = params.snippet;
         self.notebook = params.notebook;
 
-        self.saveTarget = ko.observable('hdfs-directory');
+        self.saveTarget = ko.observable('hdfs-file');
         self.savePath = ko.observable('');
         self.saveOverwrite = ko.observable(true);
 

+ 9 - 4
desktop/libs/notebook/src/notebook/views.py

@@ -23,7 +23,7 @@ from django.db.models import Q
 from django.shortcuts import redirect
 from django.utils.translation import ugettext as _
 
-from desktop.conf import USE_NEW_EDITOR
+from desktop.conf import USE_NEW_EDITOR, IS_HUE_4
 from desktop.lib.django_util import render, JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.json_utils import JSONEncoderForHTML
@@ -175,11 +175,16 @@ def execute_and_watch(request):
     sql, success_url = api.export_data_as_table(notebook, snippet, destination)
     editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute', database=snippet['database'])
   elif action == 'insert_as_query':
-    sql, success_url = api.export_large_data_to_hdfs(notebook, snippet, destination)
-    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute', database=snippet['database'])
+    if IS_HUE_4.get():
+      # TODO: checks/workarounds in case of non impersonation or Sentry
+      # TODO: keep older simpler way in case of known not many rows?
+      sql, success_url = api.export_large_data_to_hdfs(notebook, snippet, destination)
+    else:
+      sql, success_url = api.export_large_data_to_hdfs1(notebook, snippet, destination)
+    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute', database=snippet['database'], on_success_url=success_url)
   elif action == 'index_query':
     if destination == '__hue__':
-      destination = _get_snippet_name(notebook).replace('-', '_')
+      destination = _get_snippet_name(notebook)
       live_indexing = True
     else:
       live_indexing = False