Quellcode durchsuchen

HUE-3096 [core] Log warning when downloading to CSV or XLS and file is truncated

Jenny Kim vor 9 Jahren
Ursprung
Commit
032e896978

+ 1 - 1
apps/beeswax/src/beeswax/conf.py

@@ -100,7 +100,7 @@ DOWNLOAD_ROW_LIMIT = Config(
   key='download_row_limit',
   key='download_row_limit',
   default=1000000,
   default=1000000,
   type=int,
   type=int,
-  help=_t('A limit to the number of rows that can be downloaded from a query. A value of -1 means there will be no limit. A maximum of 65,000 is applied to XLS downloads.'))
+  help=_t('A limit to the number of rows that can be downloaded from a query. A value of -1 means there will be no limit. A maximum of 30,000 is applied to XLS downloads.'))
 
 
 APPLY_NATURAL_SORT_MAX = Config(
 APPLY_NATURAL_SORT_MAX = Config(
   key="apply_natural_sort_max",
   key="apply_natural_sort_max",

+ 1 - 0
apps/beeswax/src/beeswax/data_export.py

@@ -84,6 +84,7 @@ def HS2DataAdapter(handle, db, max_rows=0, start_over=True):
     for row in results.rows():
     for row in results.rows():
       num_rows_seen += 1
       num_rows_seen += 1
       if limit_rows and num_rows_seen > max_rows:
       if limit_rows and num_rows_seen > max_rows:
+        LOG.warn('The query results exceeded the maximum row limit of %d. Data has been truncated.' % max_rows)
         break
         break
       data.append(row)
       data.append(row)
 
 

+ 1 - 1
desktop/conf.dist/hue.ini

@@ -793,7 +793,7 @@
 
 
   # A limit to the number of rows that can be downloaded from a query.
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.
   # A value of -1 means there will be no limit.
-  # A maximum of 65,000 is applied to XLS downloads.
+  # A maximum of 30,000 is applied to XLS downloads.
   ## download_row_limit=1000000
   ## download_row_limit=1000000
 
 
   # Hue will try to close the Hive query when the user leaves the editor page.
   # Hue will try to close the Hive query when the user leaves the editor page.

+ 1 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -795,7 +795,7 @@
 
 
   # A limit to the number of rows that can be downloaded from a query.
   # A limit to the number of rows that can be downloaded from a query.
   # A value of -1 means there will be no limit.
   # A value of -1 means there will be no limit.
-  # A maximum of 65,000 is applied to XLS downloads.
+  # A maximum of 30,000 is applied to XLS downloads.
   ## download_row_limit=1000000
   ## download_row_limit=1000000
 
 
   # Hue will try to close the Hive query when the user leaves the editor page.
   # Hue will try to close the Hive query when the user leaves the editor page.

+ 3 - 0
desktop/core/src/desktop/lib/export_csvxls.py

@@ -93,9 +93,11 @@ def create_generator(content_generator, format, encoding=None):
     for _headers, _data in content_generator:
     for _headers, _data in content_generator:
       # Forced limit on size from tablib
       # Forced limit on size from tablib
       if len(data) > MAX_XLS_ROWS:
       if len(data) > MAX_XLS_ROWS:
+        LOG.warn('The query results exceeded the maximum row limit of %d. Data has been truncated.' % MAX_XLS_ROWS)
         break
         break
 
 
       if _headers and len(_headers) > MAX_XLS_COLS:
       if _headers and len(_headers) > MAX_XLS_COLS:
+        LOG.warn('The query results exceeded the maximum column limit of %d. Data has been truncated.' % MAX_XLS_COLS)
         _headers = _headers[:MAX_XLS_COLS]
         _headers = _headers[:MAX_XLS_COLS]
 
 
       headers = _headers
       headers = _headers
@@ -107,6 +109,7 @@ def create_generator(content_generator, format, encoding=None):
         data.append(row)
         data.append(row)
 
 
     if len(data) > MAX_XLS_ROWS:
     if len(data) > MAX_XLS_ROWS:
+      LOG.warn('The query results exceeded the maximum row limit of %d. Data has been truncated.' % MAX_XLS_ROWS)
       data = data[:MAX_XLS_ROWS]
       data = data[:MAX_XLS_ROWS]
 
 
     yield xls_dataset(headers, data, encoding).xls
     yield xls_dataset(headers, data, encoding).xls