Эх сурвалжийг харах

HUE-1826 [impala] Download or export query result

Number of initial rows of a result set to ask Impala to cache in order
to support re-fetching them for downloading them.
Set to -1 for disabling the option and backward compatibility.
Romain Rigaux 12 жил өмнө
parent
commit
6bee6a9

+ 0 - 1
apps/beeswax/src/beeswax/conf.py

@@ -25,7 +25,6 @@ from desktop.lib.conf import ConfigSection, Config, coerce_bool
 from beeswax.settings import NICE_NAME
 
 
-
 HIVE_SERVER_HOST = Config(
   key="hive_server_host",
   help=_t("Host where HiveServer2 server is running. If Kerberos security is enabled, "

+ 1 - 0
apps/beeswax/src/beeswax/data_export.py

@@ -30,6 +30,7 @@ LOG = logging.getLogger(__name__)
 _DATA_WAIT_SLEEP = 0.1                  # Sleep 0.1 sec before checking for data availability
 FETCH_ROWS = 100000
 
+
 def download(handle, format, db):
   """
   download(query_model, format) -> HttpResponse

+ 6 - 4
apps/beeswax/src/beeswax/server/dbms.py

@@ -50,14 +50,15 @@ def get(user, query_server=None):
 def get_query_server_config(name='beeswax', server=None):
   if name == 'impala':
     from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
-        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED
+        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS
 
     query_server = {
         'server_name': 'impala',
         'server_host': IMPALA_SERVER_HOST.get(),
         'server_port': IMPALA_SERVER_PORT.get(),
         'principal': IMPALA_PRINCIPAL.get(),
-        'impersonation_enabled': IMPERSONATION_ENABLED.get()
+        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
+        'querycache_rows': QUERYCACHE_ROWS.get()
     }
   else:
     kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
@@ -159,12 +160,14 @@ class HiveServer2Dbms(object):
         self.close(handle)
         return result
 
+
   def analyze_table_table(self, database, table):
     hql = 'analyze table `%(database)s.%(table_name)` compute statistics' % {'database': database, 'table_name': table.name}
     query = hql_query(hql, database)
 
     return self.execute_query(query)
 
+
   def analyze_table_column(self):
     # analyze table <table_name> partition <part_name> compute statistics for columns <col_name1>, <col_name2>...
     pass
@@ -369,8 +372,7 @@ class HiveServer2Dbms(object):
     try:
       handle = self.client.query(query, query_history.statement_number)
       if not handle.is_valid():
-        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % \
-              {'id': query_history.id, 'query': query[:40]}
+        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {'id': query_history.id, 'query': query[:40]}
         raise QueryServerException(msg)
     except QueryServerException, ex:
       LOG.exception(ex)

+ 3 - 4
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -484,6 +484,9 @@ class HiveServerClient:
     configuration = self._get_query_configuration(query)
     query_statement =  query.get_query_statement(statement)
 
+    if self.query_server['server_name'] == 'impala' and self.query_server['querycache_rows'] > 0:
+      configuration['impala.resultset.cache.size'] = str(self.query_server['querycache_rows'])
+
     return self.execute_async_statement(statement=query_statement, confOverlay=configuration)
 
 
@@ -680,10 +683,6 @@ class HiveServerClientCompatible(object):
     if max_rows is None:
       max_rows = 1000
 
-    # Impala does not support FETCH_FIRST
-    if self.query_server['server_name'] == 'impala':
-      start_over = False
-
     if start_over:
       orientation = TFetchOrientation.FETCH_FIRST
     else:

+ 19 - 12
apps/beeswax/src/beeswax/templates/execute.mako

@@ -265,14 +265,16 @@ ${layout.menubar(section='query')}
 
       % if app_name != 'impala':
       <a id="save-results" data-bind="click: saveResultsModal" href="javascript:void(0)" title="${_('Save the results to HDFS or a new Hive table')}" rel="tooltip"
-        class="view-query-results hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-save"></i></h4></a>
+        class="view-query-results hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-save"></i></h4>
+      </a>
+      % endif
 
       <a id="download-csv" data-bind="attr: {'href': '/beeswax/download/' + $root.design.history.id() + '/csv'}" href="javascript:void(0)" title="${_('Download the results in CSV format')}" rel="tooltip"
-        class="view-query-results hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-arrow-circle-o-down"></i></h4></a>
+        class="view-query-results download hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-arrow-circle-o-down"></i></h4>
+      </a>
 
       <a id="download-excel" data-bind="attr: {'href': '/beeswax/download/' + $root.design.history.id() + '/xls'}" href="javascript:void(0)" title="${_('Download the results for excel')}" rel="tooltip"
-        class="view-query-results hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-arrow-circle-o-down"></i></h4></a>
-      % endif
+        class="view-query-results download hide pull-right"><h4 style="margin-right: 20px"><i class="fa fa-arrow-circle-o-down"></i></h4></a>
    </div>
 
     <div class="card-body">
@@ -1762,6 +1764,7 @@ $(document).ready(function () {
 
 % if app_name == 'impala':
 $(document).ready(function () {
+
   $("#downloadQuery").click(function () {
     $("<input>").attr("type", "hidden").attr("name", "button-submit").attr("value", "Execute").appendTo($("#advancedSettingsForm"));
     $("<input>").attr("type", "hidden").attr("name", "download").attr("value", "true").appendTo($("#advancedSettingsForm"));
@@ -1775,13 +1778,6 @@ $(document).ready(function () {
     'html': true
   });
 
-  $("#refresh-tip").popover({
-    'title': "${_('Missing some tables? In order to update the list of tables/metadata seen by Impala, execute one of these queries:')}",
-    'content': $("#refresh-content").html(),
-    'trigger': 'hover',
-    'html': true
-  });
-
   $(document).on('explain.query', function() {
     viewModel.closeQuery();
   });
@@ -1789,11 +1785,22 @@ $(document).ready(function () {
   $(document).on('execute.query', function() {
     viewModel.closeQuery();
   });
+
+  // Tricks for not triggering the closing of the query on download
+  $("a.download").hover(function(){
+      window.onbeforeunload = null;
+    },function() {
+      window.onbeforeunload = $(window).data('beforeunload');
+    }
+  );
 });
 
+// Close the query when leaving the page, backup for later when disabling the close before downloading results.
 window.onbeforeunload = function(e) {
   viewModel.closeQuery();
 };
+$(window).data('beforeunload', window.onbeforeunload);
+
 % endif
 
 $(".pathChooser:not(:has(~ button))").after(getFileAndFolderBrowseButton($(".pathChooser:not(:has(~ button))"), true));
@@ -1996,7 +2003,7 @@ function cacheQueryTextEvents() {
 
 function databaseCacheWriter() {
   $(".chosen-select").chosen().change(function () {
-    $.totalStorage("${app_name}_last_database", viewModel.database()); 
+    $.totalStorage("${app_name}_last_database", viewModel.database());
   });
 }
 

+ 0 - 4
apps/beeswax/src/beeswax/views.py

@@ -412,9 +412,6 @@ def view_results(request, id, first_row=0):
   context_param = request.GET.get('context', '')
   query_context = parse_query_context(context_param)
 
-  # To remove when Impala has start_over support
-  download = request.GET.get('download', '') or app_name != 'impala'
-
   # Update the status as expired should not be accessible
   # Impala does not support startover for now
   expired = state == models.QueryHistory.STATE.expired
@@ -456,7 +453,6 @@ def view_results(request, id, first_row=0):
     'context_param': context_param,
     'expired': expired,
     'app_name': app_name,
-    'download': download,
     'next_json_set': None,
     'is_finished': query_history.is_finished()
   }

+ 8 - 0
apps/impala/src/impala/conf.py

@@ -47,6 +47,14 @@ IMPERSONATION_ENABLED=Config(
   type=coerce_bool,
   default=False)
 
+QUERYCACHE_ROWS=Config(
+  key='querycache_rows',
+  help=_t("Number of initial rows of a resultset to ask Impala to cache in order to"
+          " support re-fetching them for downloading them."
+          " Set to 0 for disabling the option and backward compatibility."),
+  type=int,
+  default=50000)
+
 
 def config_validator(user):
   # dbms is dependent on beeswax.conf (this file)

+ 5 - 0
desktop/conf.dist/hue.ini

@@ -665,6 +665,11 @@
   # Turn on/off impersonation mechanism when talking to Impala
   ## impersonation_enabled=False
 
+  # Number of initial rows of a result set to ask Impala to cache in order
+  # to support re-fetching them for downloading them.
+  # Set to 0 for disabling the option and backward compatibility.
+  ## querycache_rows=50000
+
 
 ###########################################################################
 # Settings to configure Hbase

+ 5 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -707,6 +707,11 @@
   # Turn on/off impersonation mechanism when talking to Impala
   ## impersonation_enabled=False
 
+  # Number of initial rows of a result set to ask Impala to cache in order
+  # to support re-fetching them for downloading them.
+  # Set to 0 for disabling the option and backward compatibility.
+  ## querycache_rows=50000
+
 
 ###########################################################################
 # Settings to configure Hbase