Эх сурвалжийг харах

HUE-2142 [editor] Change result download truncation limit to be a number of rows

Romain Rigaux 9 жил өмнө
parent
commit
18a5ca0

+ 12 - 0
apps/beeswax/src/beeswax/conf.py

@@ -96,6 +96,7 @@ LIST_PARTITIONS_LIMIT = Config(
   type=int,
   help=_t('Limit the number of partitions that can be listed. A positive value will be set as the LIMIT.'))
 
+# Deprecated
 DOWNLOAD_CELL_LIMIT = Config(
   key='download_cell_limit',
   default=10000000,
@@ -104,6 +105,17 @@ DOWNLOAD_CELL_LIMIT = Config(
           '(e.g. - 10K rows * 1K columns = 10M cells.) '
           'A value of -1 means there will be no limit.'))
 
+def get_deprecated_download_cell_limit():
+  """Get the old default"""
+  return DOWNLOAD_CELL_LIMIT.get() / 100 if DOWNLOAD_CELL_LIMIT.get() > 0 else DOWNLOAD_CELL_LIMIT.get()
+
+DOWNLOAD_ROW_LIMIT = Config(
+  key='download_row_limit',
+  dynamic_default=get_deprecated_download_cell_limit,
+  type=int,
+  help=_t('A limit to the number of rows that can be downloaded from a query before it is truncated. '
+          'A value of -1 means there will be no limit.'))
+
 APPLY_NATURAL_SORT_MAX = Config(
   key="apply_natural_sort_max",
   help=_t("The max number of records in the result set permitted to apply a natural sort to the database or tables list."),

+ 10 - 10
apps/beeswax/src/beeswax/data_export.py

@@ -41,9 +41,9 @@ def download(handle, format, db, id=None, file_name='query_result'):
     LOG.error('Unknown download format "%s"' % (format,))
     return
 
-  max_cells = conf.DOWNLOAD_CELL_LIMIT.get()
+  max_rows = conf.DOWNLOAD_ROW_LIMIT.get()
 
-  content_generator = HS2DataAdapter(handle, db, max_cells=max_cells, start_over=True)
+  content_generator = HS2DataAdapter(handle, db, max_rows=max_rows, start_over=True)
   generator = export_csvxls.create_generator(content_generator, format)
 
   resp = export_csvxls.make_response(generator, format, file_name)
@@ -61,7 +61,7 @@ def download(handle, format, db, id=None, file_name='query_result'):
   return resp
 
 
-def upload(path, handle, user, db, fs, max_cells=-1):
+def upload(path, handle, user, db, fs, max_rows=-1):
   """
   upload(query_model, path, user, db, fs) -> None
 
@@ -72,7 +72,7 @@ def upload(path, handle, user, db, fs, max_cells=-1):
   else:
     fs.do_as_user(user.username, fs.create, path)
 
-  content_generator = HS2DataAdapter(handle, db, max_cells=max_cells, start_over=True)
+  content_generator = HS2DataAdapter(handle, db, max_rows=max_rows, start_over=True)
   for header, data in content_generator:
     dataset = export_csvxls.dataset(None, data)
     fs.do_as_user(user.username, fs.append, path, dataset.csv)
@@ -80,13 +80,13 @@ def upload(path, handle, user, db, fs, max_cells=-1):
 
 class HS2DataAdapter:
 
-  def __init__(self, handle, db, max_cells=-1, start_over=True):
+  def __init__(self, handle, db, max_rows=-1, start_over=True):
     self.handle = handle
     self.db = db
-    self.max_cells = max_cells
+    self.max_rows = max_rows
     self.start_over = start_over
     self.fetch_size = FETCH_SIZE
-    self.limit_cells = max_cells > -1
+    self.limit_rows = max_rows > -1
 
     self.first_fetched = True
     self.headers = None
@@ -106,7 +106,7 @@ class HS2DataAdapter:
 
       # For result sets with high num of columns, fetch in smaller batches to avoid serialization cost
       if self.num_cols > 100:
-        LOG.warn('The query results contain %d columns and may take an extremely long time to download, will reduce fetch size to 100.' % self.num_cols)
+        LOG.warn('The query results contain %d columns and may take long time to download, reducing fetch size to 100.' % self.num_cols)
         self.fetch_size = 100
 
     if not self.is_truncated and (self.first_fetched or results.has_more):
@@ -116,8 +116,8 @@ class HS2DataAdapter:
 
       for row in results.rows():
         self.row_counter += 1
-        if self.limit_cells and (self.row_counter * self.num_cols) > self.max_cells:
-          LOG.warn('The query results exceeded the maximum cell limit of %d. Data has been truncated to first %d rows.' % (self.max_cells, self.row_counter))
+        if self.limit_rows and self.row_counter > self.max_rows:
+          LOG.warn('The query results exceeded the maximum row limit of %d and has been truncated to first %d rows.' % (self.max_rows, self.row_counter))
           self.is_truncated = True
           break
         data.append(row)

+ 1 - 1
apps/beeswax/src/beeswax/tests.py

@@ -833,7 +833,7 @@ for x in sys.stdin:
     assert_equal(sheet_data, csv_data)
 
     # Test max cell limit truncation
-    finish = conf.DOWNLOAD_CELL_LIMIT.set_for_testing(num_cols * 5)
+    finish = conf.DOWNLOAD_ROW_LIMIT.set_for_testing(5)
     try:
       hql = 'SELECT * FROM `%(db)s`.`test`' % {'db': self.db_name}
       query = hql_query(hql)

+ 5 - 3
desktop/conf.dist/hue.ini

@@ -898,15 +898,17 @@
   # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
   ## query_partitions_limit=10
 
-  # A limit to the number of cells (rows * columns) that can be downloaded from a query
-  # (e.g. - 10K rows * 1K columns = 10M cells.)
+  # A limit to the number of rows that can be downloaded from a query before it is truncated.
   # A value of -1 means there will be no limit.
-  ## download_cell_limit=10000000
+  ## download_row_limit=100000
 
   # Hue will try to close the Hive query when the user leaves the editor page.
   # This will free all the query resources in HiveServer2, but also make its results inaccessible.
   ## close_queries=false
 
+  # Hue will use at most this many HiveServer2 sessions per user at a time.
+  ## max_number_of_sessions=1
+
   # Thrift version to use when communicating with HiveServer2.
   # New column format is from version 7.
   ## thrift_version=7

+ 4 - 5
desktop/conf/pseudo-distributed.ini.tmpl

@@ -902,16 +902,15 @@
   # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.
   ## query_partitions_limit=10
 
-  # A limit to the number of cells (rows * columns) that can be downloaded from a query
-  # (e.g. - 10K rows * 1K columns = 10M cells.)
+  # A limit to the number of rows that can be downloaded from a query before it is truncated.
   # A value of -1 means there will be no limit.
-  ## download_cell_limit=10000000
+  ## download_row_limit=100000
 
   # Hue will try to close the Hive query when the user leaves the editor page.
   # This will free all the query resources in HiveServer2, but also make its results inaccessible.
   ## close_queries=false
 
-  # Hue will use at most this many HiveServer2 sessions per user at a time
+  # Hue will use at most this many HiveServer2 sessions per user at a time.
   ## max_number_of_sessions=1
 
   # Thrift version to use when communicating with HiveServer2.
@@ -919,7 +918,7 @@
   ## thrift_version=7
 
   # A comma-separated list of white-listed Hive configuration properties that users are authorized to set.
-  # config_whitelist=hive.map.aggr,hive.exec.compress.output,hive.exec.parallel,hive.execution.engine,mapreduce.job.queuename
+  ## config_whitelist=hive.map.aggr,hive.exec.compress.output,hive.exec.parallel,hive.execution.engine,mapreduce.job.queuename
 
   # Override the default desktop username and password of the hue user used for authentications with other services.
   # e.g. Used for LDAP/PAM pass-through authentication.

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -883,7 +883,7 @@ ${ assist.assistPanel() }
       self.inputFormats = ko.observableArray([
           {'value': 'file', 'name': 'File'},
           {'value': 'table', 'name': 'Table'},
-          {'value': 'text', 'name': 'Copy paste text'},
+          {'value': 'text', 'name': 'Paste Text'},
           {'value': 'query', 'name': 'SQL Query'},
           {'value': 'dbms', 'name': 'DBMS'},
           {'value': 'manual', 'name': 'Manually'},

+ 21 - 3
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -45,7 +45,7 @@ LOG = logging.getLogger(__name__)
 try:
   from beeswax import conf as beeswax_conf, data_export
   from beeswax.api import _autocomplete, _get_sample_data
-  from beeswax.conf import CONFIG_WHITELIST as hive_settings, DOWNLOAD_CELL_LIMIT
+  from beeswax.conf import CONFIG_WHITELIST as hive_settings, DOWNLOAD_ROW_LIMIT
   from beeswax.data_export import upload
   from beeswax.design import hql_query, strip_trailing_semicolon, split_statements
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
@@ -444,9 +444,9 @@ class HS2Api(Api):
     db = self._get_db(snippet)
 
     handle = self._get_handle(snippet)
-    max_cells = DOWNLOAD_CELL_LIMIT.get()
+    max_rows = DOWNLOAD_ROW_LIMIT.get()
 
-    upload(target_file, handle, self.request.user, db, self.request.fs, max_cells=max_cells)
+    upload(target_file, handle, self.request.user, db, self.request.fs, max_rows=max_rows)
 
     return '/filebrowser/view=%s' % target_file
 
@@ -475,6 +475,24 @@ class HS2Api(Api):
     return hql, success_url
 
 
+  def export_large_data_to_hdfs1(self, notebook, snippet, destination):
+    db = self._get_db(snippet)
+
+    response = self._get_current_statement(db, snippet)
+    session = self._get_session(notebook, snippet['type'])
+    query = self._prepare_hql_query(snippet, response.pop('statement'), session)
+
+    if 'select' not in query.hql_query.strip().lower():
+      raise PopupException(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
+
+    db.use(query.database)
+
+    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
+    success_url = '/filebrowser/view=%s' % destination
+
+    return hql, success_url
+
+
   def export_large_data_to_hdfs(self, notebook, snippet, destination):
     db = self._get_db(snippet)
 

+ 14 - 25
desktop/libs/notebook/src/notebook/templates/notebook_ko_components.mako

@@ -22,10 +22,10 @@ from desktop.lib.i18n import smart_unicode
 from desktop.views import _ko
 
 try:
-  from beeswax.conf import DOWNLOAD_CELL_LIMIT
+  from beeswax.conf import DOWNLOAD_ROW_LIMIT
 except ImportError, e:
   LOG.warn("Hive app is not enabled")
-  DOWNLOAD_CELL_LIMIT = None
+  DOWNLOAD_ROW_LIMIT = None
 
 try:
   from indexer.conf import ENABLE_NEW_INDEXER
@@ -181,12 +181,12 @@ except ImportError, e:
       </a>
       <ul class="dropdown-menu less-padding">
         <li>
-          <a class="inactive-action download" href="javascript:void(0)" data-bind="click: downloadCsv, event: { mouseover: function(){ window.onbeforeunload = null; }, mouseout: function() { window.onbeforeunload = $(window).data('beforeunload'); } }" title="${ _('Download first %s cells as CSV') % DOWNLOAD_CELL_LIMIT.get() }">
+          <a class="inactive-action download" href="javascript:void(0)" data-bind="click: downloadCsv, event: { mouseover: function(){ window.onbeforeunload = null; }, mouseout: function() { window.onbeforeunload = $(window).data('beforeunload'); } }" title="${ _('Download first %s rows as CSV') % DOWNLOAD_ROW_LIMIT.get() }">
             <i class="fa fa-fw fa-file-o"></i> ${ _('CSV') }
           </a>
         </li>
         <li>
-          <a class="inactive-action download" href="javascript:void(0)" data-bind="click: downloadXls, event: { mouseover: function(){ window.onbeforeunload = null; }, mouseout: function() { window.onbeforeunload = $(window).data('beforeunload'); } }" title="${ _('Download first %s cells as XLS') % DOWNLOAD_CELL_LIMIT.get() }">
+          <a class="inactive-action download" href="javascript:void(0)" data-bind="click: downloadXls, event: { mouseover: function(){ window.onbeforeunload = null; }, mouseout: function() { window.onbeforeunload = $(window).data('beforeunload'); } }" title="${ _('Download first %s rows as XLS') % DOWNLOAD_ROW_LIMIT.get() }">
             <i class="fa fa-fw fa-file-excel-o"></i> ${ _('Excel') }
           </a>
         </li>
@@ -222,31 +222,20 @@ except ImportError, e:
           <fieldset>
             <div class="control-group">
               <div class="controls">
-                <label class="radio">
-                  <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="hdfs-file">
-                  &nbsp;${ _('File (max %s cells)') % DOWNLOAD_CELL_LIMIT.get() }
-                </label>
-                <div data-bind="visible: saveTarget() == 'hdfs-file'" class="inline">
-                  <input data-bind="value: savePath, valueUpdate:'afterkeydown', filechooser: { value: savePath, isNestedModal: true }, filechooserOptions: { uploadFile: false, skipInitialPathIfEmpty: true, linkMarkup: true }, hdfsAutocomplete: savePath" type="text" name="target_file" placeholder="${_('Path to CSV file')}" class="pathChooser margin-left-10">
-                </div>
-                <label class="radio" data-bind="visible: saveTarget() == 'hdfs-file'">
-                  <input data-bind="checked: saveOverwrite" type="checkbox" name="overwrite">
-                  ${ _('Overwrite') }
-                </label>
-              </div>
-            </div>
-            <div class="control-group">
-              <div class="controls" data-bind="visible: snippet.type() == 'hive'">
                 <label class="radio">
                   <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="hdfs-directory">
-                  &nbsp;${ _('File (large result)') }
+                  &nbsp;${ _('File') }
                 </label>
                 <div data-bind="visible: saveTarget() == 'hdfs-directory'" class="inline">
-                  <input data-bind="value: savePath, valueUpdate:'afterkeydown', filechooser: { value: savePath, isNestedModal: true }, filechooserOptions: { uploadFile: false, skipInitialPathIfEmpty: true, displayOnlyFolders: true, linkMarkup: true }, hdfsAutocomplete: savePath" type="text" name="target_dir" placeholder="${_('Path to empty directory')}" class="pathChooser margin-left-10">
-                  <div class="inline-block" data-bind="tooltip: { title: '${ _ko("Use this option if you have a large result. It will rerun the entire query and save the results to the chosen HDFS directory.") }', placement: 'top' }" style="padding: 8px">
-                    <i class="fa fa-fw fa-question-circle muted"></i>
-                  </div>
+                  <input data-bind="value: savePath, valueUpdate:'afterkeydown', filechooser: { value: savePath, isNestedModal: true }, filechooserOptions: { uploadFile: false, skipInitialPathIfEmpty: true, displayOnlyFolders: true, linkMarkup: true }, hdfsAutocomplete: savePath" type="text" name="target_dir" placeholder="${_('Path to empty directory')}" class="pathChooser margin-left-10 input-xlarge">
+                </div>
+                <div class="inline-block" data-bind="visible: saveTarget() == 'hdfs-directory', tooltip: { title: '${ _ko("Save a large result as CSV") }', placement: 'top' }" style="padding: 8px">
+                  <i class="fa fa-fw fa-question-circle muted"></i>
                 </div>
+                ##<label class="radio" data-bind="visible: saveTarget() == 'hdfs-directory'">
+                ##  <input data-bind="checked: saveOverwrite" type="checkbox" name="overwrite">
+                ##  ${ _('Download') }
+                ##</label>
               </div>
             </div>
             <div class="control-group">
@@ -321,7 +310,7 @@ except ImportError, e:
         self.snippet = params.snippet;
         self.notebook = params.notebook;
 
-        self.saveTarget = ko.observable('hdfs-file');
+        self.saveTarget = ko.observable('hdfs-directory');
         self.savePath = ko.observable('');
         self.saveOverwrite = ko.observable(true);