Browse Source

HUE-1843 [beeswax] Multi query support

Fix HUE-1824 [impala] Cancel operation only for Impala
Fix UDF styling bug in editor
Fix bug when previous query is closed
Fix some tests
Romain Rigaux 12 năm trước cách đây
mục cha
commit
c6c2426

+ 16 - 12
apps/beeswax/src/beeswax/api.py

@@ -33,11 +33,10 @@ import beeswax.models
 from beeswax.forms import QueryForm
 from beeswax.design import HQLdesign
 from beeswax.server import dbms
-from beeswax.server.dbms import expand_exception, get_query_server_config,\
-  QueryServerException
+from beeswax.server.dbms import expand_exception, get_query_server_config
 from beeswax.views import authorized_get_design, authorized_get_history, make_parameterization_form,\
                           safe_get_design, save_design, massage_columns_for_json, _get_query_handle_and_state,\
-  _parse_out_hadoop_jobs
+                          _parse_out_hadoop_jobs
 from desktop.lib.i18n import force_unicode
 
 
@@ -132,7 +131,8 @@ def execute_directly(request, query, design, query_server, tablename=None, **kwa
   response = {
     'status': 0,
     'id': history_obj.id,
-    'watch_url': watch_url
+    'watch_url': watch_url,
+    'statement': history_obj.get_current_statement()
   }
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
@@ -142,13 +142,15 @@ def execute_directly(request, query, design, query_server, tablename=None, **kwa
 def watch_query_refresh_json(request, id):
   query_history = authorized_get_history(request, id, must_exist=True)
   db = dbms.get(request.user, query_history.get_query_server_config())
-  handle, state = _get_query_handle_and_state(query_history)
-  query_history.save_state(state)
 
-  # Multi query if more statements
+  if not request.POST.get('next'): # We need this as multi query would fail as current query is closed
+    handle, state = _get_query_handle_and_state(query_history)
+    query_history.save_state(state)
+
+  # Go to next statement if asked to continue or when a statement with no dataset finished.
   try:
-    if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
-      db.execute_next_statement(query_history)
+    if request.POST.get('next') or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
+      query_history = db.execute_next_statement(query_history)
       handle, state = _get_query_handle_and_state(query_history)
   except Exception, ex:
     LOG.exception(ex)
@@ -166,9 +168,11 @@ def watch_query_refresh_json(request, id):
     'log': log,
     'jobs': jobs,
     'jobUrls': job_urls,
-    'isSuccess': query_history.is_finished() or (query_history.is_success() and query_history.has_results and query_history.is_failure()),
+    'isSuccess': query_history.is_success(),
     'isFailure': query_history.is_failure(),
-    'id': id
+    'id': id,
+    'statement': query_history.get_current_statement(),
+    'watch_url': reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
   }
 
   # Show popup message if error, should be better in error tab instead and merged into the result response below
@@ -178,7 +182,7 @@ def watch_query_refresh_json(request, id):
       result['message'] = res.errorMessage
     else:
       result['message'] = ''
-    
+
     result['error'] = 'Bad status for request %s:\n%s' % (id, res)
     result['status'] = 1
 

+ 12 - 9
apps/beeswax/src/beeswax/templates/execute.mako

@@ -96,10 +96,11 @@ ${layout.menubar(section='query')}
             <a data-bind="click: function() { $root.addFileResources('','') }" class="btn btn-mini paramAdd">${_('Add')}</a>
           </div>
         </li>
-        <li class="nav-header 
+        <li title="${ _('User-Defined Functions') }" class="nav-header
           % if app_name == 'impala':
             hide
-          % endif" title="${_("User-Defined Functions")}">
+          % endif
+          ">
           ${_('UDFs')}
         </li>
         <li class="white paramContainer
@@ -233,8 +234,14 @@ ${layout.menubar(section='query')}
           <textarea class="hide" tabindex="1" name="query" id="queryField"></textarea>
 
           <div class="actions">
-            <button data-bind="click: tryExecuteQuery, visible: !$root.query.isRunning()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
+            % if app_name == 'impala':
+            <button data-bind="click: tryExecuteQuery, visible: !$root.query.isRunning() && $root.query.isFinished()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
             <button data-bind="click: tryCancelQuery, visible: $root.query.isRunning()" class="btn btn-danger" data-loading-text="${ _('Canceling...') }" rel="tooltip" data-original-title="${ _('Cancel the query') }">${ _('Cancel') }</button>
+            % else:
+            <button data-bind="click: tryExecuteQuery, enable: !$root.query.isRunning(), visible: $root.query.isFinished()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
+            % endif
+            <button data-bind="click: executeNextStatement, visible: !$root.query.isFinished()" type="button" class="btn btn-primary disable-feedback" tabindex="2">${_('Next')}</button>
+
             <button data-bind="click: trySaveQuery, css: {'hide': !$root.query.id() || $root.query.id() == -1}" type="button" class="btn hide">${_('Save')}</button>
             <button data-bind="click: saveAsModal" type="button" class="btn">${_('Save as...')}</button>
             <button data-bind="click: tryExplainQuery" type="button" id="explainQuery" class="btn">${_('Explain')}</button>
@@ -263,10 +270,10 @@ ${layout.menubar(section='query')}
 
     <div class="card-body">
       <ul class="nav nav-tabs">
+        <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
         <!-- ko if: !query.explain() -->
         <li><a href="#log" data-toggle="tab">${_('Log')}</a></li>
         <!-- /ko -->
-        <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
         <!-- ko if: !query.explain() -->
         <li><a href="#columns" data-toggle="tab">${_('Columns')}</a></li>
         <li><a href="#results" data-toggle="tab">${_('Results')}</a></li>
@@ -279,7 +286,7 @@ ${layout.menubar(section='query')}
 
       <div class="tab-content">
         <div class="tab-pane" id="query">
-          <pre data-bind="text: viewModel.query.query()"></pre>
+          <pre data-bind="text: viewModel.query.statement()"></pre>
         </div>
         <!-- ko if: query.explain() -->
         <div class="tab-pane" id="explanation">
@@ -1809,12 +1816,8 @@ $(document).ready(function () {
   });
 });
 
-// @TODO: Make file resources list automatically generated.
 // @TODO: Improve resize logs to be more relative. See FF versus Chrome.
-// @TODO: Close
 // @TODO: Stop operation
-// @TODO: Re-add filebrowser to file resources.
-// @TODO: Re-add tooltips for execute query button
 // @TODO: Re-add download query for impala
 // @TODO: Re-enable type ahead for settings
 </script>

+ 2 - 2
apps/beeswax/src/beeswax/templates/list_designs.mako

@@ -16,7 +16,7 @@
 <%!
     import time
     from django.template.defaultfilters import timesince
-    from django.utils.encoding import force_unicode    
+    from django.utils.encoding import force_unicode
     from django.utils.translation import ugettext as _
 
     from desktop.views import commonheader, commonfooter
@@ -69,7 +69,7 @@ ${ layout.menubar(section='saved queries') }
       </%def>
 
       <%def name="creation()">
-        <div class="btn-toolbar" style="display: inline; vertical-align: middle">          
+        <div class="btn-toolbar" style="display: inline; vertical-align: middle">
           <a class="btn" href="${ url(app_name + ':execute_query') }" title="${_('Create new %s' % noun)}"><i class="fa fa-plus-circle"></i> ${_('New %s' % noun)}</a>
           <a class="btn" href="${ url(app_name + ':list_trashed_designs') }" title="${_('Go to the trash')}"><i class="fa fa-trash-o"></i> ${_('View trash')}</a>
         </div>

+ 1 - 1
apps/beeswax/src/beeswax/templates/list_trashed_designs.mako

@@ -52,7 +52,7 @@ ${layout.menubar(section='saved queries')}
         <div class="btn-toolbar" style="display: inline; vertical-align: middle">
           <button id="emptyTrashBtn" class="btn" title="${_('Empty trash')}" data-bind="enabled: availableSavedQueries().length > 0">
             <i class="fa fa-fire"></i> ${_('Empty trash')}
-          </button>        
+          </button>
           <button id="viewQueriesBtn" class="btn" title="${_('View queries')}">
             <i class="fa fa-home"></i> ${_('Back')}
           </button>

+ 22 - 12
apps/beeswax/src/beeswax/tests.py

@@ -458,11 +458,12 @@ for x in sys.stdin:
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
 
     content = json.loads(resp.content)
-    design_id = content['id']
+    history_id = content['id']
+    query_history = QueryHistory.get(id=history_id)
 
-    resp = self.client.get("/beeswax/results/%s/0?format=json" % design_id)
+    resp = self.client.get("/beeswax/results/%s/0?format=json" % history_id)
     content = json.loads(resp.content)
-    assert_true('DROP TABLE test_multiple_statements_2' in content, content) # HUE-1843
+    assert_equal('DROP TABLE test_multiple_statements_2', query_history.get_current_statement(), content)
 
   def test_multiple_statements_with_result_set(self):
     hql = """
@@ -471,16 +472,20 @@ for x in sys.stdin:
     """
 
     resp = _make_query(self.client, hql)
-    query = hql_query(hql)
 
-    handle = self.db.execute_and_wait(query)
+    content = json.loads(resp.content)
+    assert_true('watch_url' in content, content)
+    watch_url = content['watch_url']
+    assert_equal('SELECT foo FROM test', content.get('statement'), content)
+
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    content = fetch_query_result_data(self.client, resp)
 
-    assert_true('multiStatementsQuery' in resp.content, resp.content) # HUE-1843
+    assert_false(content.get('is_finished'), content)
 
-    resp = self.client.post(reverse('beeswax:watch_query', args=[resp.context['query'].id]))
-    assert_true('Waiting for query' in resp.content, resp.content)
-    assert_true('SELECT count(*) FROM test' in resp.content, resp.content)
+    resp = self.client.post(watch_url, {'next': True})
+    content = json.loads(resp.content)
+    assert_equal('SELECT count(*) FROM test', content.get('statement'), content)
 
   def test_multiple_statements_various_queries(self):
     hql = """
@@ -490,12 +495,16 @@ for x in sys.stdin:
     """
 
     resp = _make_query(self.client, hql)
-    query = hql_query(hql)
 
-    handle = self.db.execute_and_wait(query)
+    content = json.loads(resp.content)
+    assert_equal('CREATE TABLE test_multiple_statements_2 (a int)', content.get('statement'), content)
+
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    content = json.loads(resp.content)
+    assert_equal('SELECT foo FROM test', content.get('statement'), content)
 
-    assert_true('SELECT foo FROM test' in resp.content, resp.content) # HUE-1843
+    content = fetch_query_result_data(self.client, resp)
+    assert_true(content.get('is_finished'), content)
 
   def test_parallel_queries(self):
     """
@@ -751,6 +760,7 @@ for x in sys.stdin:
         'path': target_dir
       }
       resp = self.client.post('/beeswax/api/query/%s/results/save' % qid, save_data, follow=True)
+
       resp = wait_for_query_to_finish(self.client, resp, max=60)
 
       # Check that data is right

+ 2 - 1
apps/beeswax/src/beeswax/views.py

@@ -583,7 +583,8 @@ def view_results(request, id, first_row=0):
     'expired': expired,
     'app_name': app_name,
     'download': download,
-    'next_json_set': None
+    'next_json_set': None,
+    'is_finished': query_history.is_finished()
   }
 
   if not error:

+ 40 - 5
apps/beeswax/static/js/beeswax.vm.js

@@ -48,7 +48,9 @@ function BeeswaxViewModel(server, query_id) {
       'url': null,
       'errors': []
     },
-    'isRunning': false
+    'isRunning': false,
+    'statement': '',
+    'isFinished': true
   };
 
   self.server = ko.observable(server);
@@ -104,7 +106,7 @@ function BeeswaxViewModel(server, query_id) {
     self.database(design.database);
     self.query.isParameterized(design.is_parameterized);
     self.query.email(design.email_notify);
-    
+
     self.query.settings.removeAll();
     self.query.fileResources.removeAll();
     self.query.functions.removeAll();
@@ -347,6 +349,7 @@ function BeeswaxViewModel(server, query_id) {
     $(document).trigger('execute.query', data);
     self.query.explain(false);
     self.query.isRunning(true);
+    self.query.isFinished(true);
     self.query.errors.removeAll();
 
     var data = {
@@ -368,6 +371,7 @@ function BeeswaxViewModel(server, query_id) {
           self.query.id(data.id);
           self.query.results.url('/' + self.server() + '/results/' + self.query.id() + '/0?format=json');
           self.query.watch.url(data.watch_url);
+          self.query.statement(data.statement);
           self.watchQueryLoop();
         } else {
           self.query.errors.push(data.message);
@@ -382,6 +386,34 @@ function BeeswaxViewModel(server, query_id) {
     $.ajax(request);
   };
 
+  self.executeNextStatement = function() {
+    $(document).trigger('execute.query', data);
+    self.query.explain(false);
+    self.query.isRunning(true);
+    self.query.isFinished(true);
+    self.query.errors.removeAll();
+
+    var data = {
+      'next': true
+    };
+    var request = {
+      url: self.query.watch.url(),
+      dataType: 'json',
+      type: 'POST',
+      success: function(data) {
+        self.query.errors.removeAll();
+        self.query.statement(data.statement);
+        self.query.watch.url(data.watch_url);
+        self.query.results.url('/' + self.server() + '/results/' + self.query.id() + '/0?format=json');
+        self.watchQueryLoop();
+        $(document).trigger('executed.query', data);
+      },
+      error: error_fn,
+      data: data
+    };
+    $.ajax(request);
+  };
+
   self.watchQuery = function() {
     var data = {
       'query-query': self.query.query(),
@@ -442,11 +474,12 @@ function BeeswaxViewModel(server, query_id) {
             }
           }
         } else {
+          self.query.statement(data.statement); // In case new no result statement executed
           if (data.log) {
             self.query.watch.logs.push(data.log);
             // scroll logs
           }
-          
+
           timer = setTimeout(_fn, TIMEOUT);
         }
       });
@@ -464,6 +497,7 @@ function BeeswaxViewModel(server, query_id) {
       type: 'GET',
       success: function(data) {
         self.query.isRunning(false);
+        self.query.isFinished(data.is_finished);
         if (self.query.results.columns().length == 0){
           self.query.results.columns(data.columns);
         }
@@ -545,7 +579,7 @@ function BeeswaxViewModel(server, query_id) {
         success: function(data) {
           if (data.status == 0) {
             $(document).trigger('closed.query', data);
-            self.resetQuery();
+            //self.resetQuery(); // Would fail multiqueries
           } else {
             $(document).trigger('error_close.query');
           }
@@ -553,7 +587,8 @@ function BeeswaxViewModel(server, query_id) {
         error: function(data) {
           $(document).trigger('error_close.results');
         },
-        data: data
+        data: data,
+        async: false // for multi query
       };
       $.ajax(request);
     }