Browse Source

HUE-1843 [beeswax] Multi query support

Fix HUE-1824 [impala] Cancel operation only for Impala
Fix UDF styling bug in editor
Fix bug when previous query is closed
Fix some tests
Romain Rigaux 12 years ago
parent
commit
c6c2426a1c

+ 16 - 12
apps/beeswax/src/beeswax/api.py

@@ -33,11 +33,10 @@ import beeswax.models
 from beeswax.forms import QueryForm
 from beeswax.forms import QueryForm
 from beeswax.design import HQLdesign
 from beeswax.design import HQLdesign
 from beeswax.server import dbms
 from beeswax.server import dbms
-from beeswax.server.dbms import expand_exception, get_query_server_config,\
-  QueryServerException
+from beeswax.server.dbms import expand_exception, get_query_server_config
 from beeswax.views import authorized_get_design, authorized_get_history, make_parameterization_form,\
 from beeswax.views import authorized_get_design, authorized_get_history, make_parameterization_form,\
                           safe_get_design, save_design, massage_columns_for_json, _get_query_handle_and_state,\
                           safe_get_design, save_design, massage_columns_for_json, _get_query_handle_and_state,\
-  _parse_out_hadoop_jobs
+                          _parse_out_hadoop_jobs
 from desktop.lib.i18n import force_unicode
 from desktop.lib.i18n import force_unicode
 
 
 
 
@@ -132,7 +131,8 @@ def execute_directly(request, query, design, query_server, tablename=None, **kwa
   response = {
   response = {
     'status': 0,
     'status': 0,
     'id': history_obj.id,
     'id': history_obj.id,
-    'watch_url': watch_url
+    'watch_url': watch_url,
+    'statement': history_obj.get_current_statement()
   }
   }
 
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
   return HttpResponse(json.dumps(response), mimetype="application/json")
@@ -142,13 +142,15 @@ def execute_directly(request, query, design, query_server, tablename=None, **kwa
 def watch_query_refresh_json(request, id):
 def watch_query_refresh_json(request, id):
   query_history = authorized_get_history(request, id, must_exist=True)
   query_history = authorized_get_history(request, id, must_exist=True)
   db = dbms.get(request.user, query_history.get_query_server_config())
   db = dbms.get(request.user, query_history.get_query_server_config())
-  handle, state = _get_query_handle_and_state(query_history)
-  query_history.save_state(state)
 
 
-  # Multi query if more statements
+  if not request.POST.get('next'): # We need this as multi query would fail as current query is closed
+    handle, state = _get_query_handle_and_state(query_history)
+    query_history.save_state(state)
+
+  # Go to next statement if asked to continue or when a statement with no dataset finished.
   try:
   try:
-    if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
-      db.execute_next_statement(query_history)
+    if request.POST.get('next') or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
+      query_history = db.execute_next_statement(query_history)
       handle, state = _get_query_handle_and_state(query_history)
       handle, state = _get_query_handle_and_state(query_history)
   except Exception, ex:
   except Exception, ex:
     LOG.exception(ex)
     LOG.exception(ex)
@@ -166,9 +168,11 @@ def watch_query_refresh_json(request, id):
     'log': log,
     'log': log,
     'jobs': jobs,
     'jobs': jobs,
     'jobUrls': job_urls,
     'jobUrls': job_urls,
-    'isSuccess': query_history.is_finished() or (query_history.is_success() and query_history.has_results and query_history.is_failure()),
+    'isSuccess': query_history.is_success(),
     'isFailure': query_history.is_failure(),
     'isFailure': query_history.is_failure(),
-    'id': id
+    'id': id,
+    'statement': query_history.get_current_statement(),
+    'watch_url': reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
   }
   }
 
 
   # Show popup message if error, should be better in error tab instead and merged into the result response below
   # Show popup message if error, should be better in error tab instead and merged into the result response below
@@ -178,7 +182,7 @@ def watch_query_refresh_json(request, id):
       result['message'] = res.errorMessage
       result['message'] = res.errorMessage
     else:
     else:
       result['message'] = ''
       result['message'] = ''
-    
+
     result['error'] = 'Bad status for request %s:\n%s' % (id, res)
     result['error'] = 'Bad status for request %s:\n%s' % (id, res)
     result['status'] = 1
     result['status'] = 1
 
 

+ 12 - 9
apps/beeswax/src/beeswax/templates/execute.mako

@@ -96,10 +96,11 @@ ${layout.menubar(section='query')}
             <a data-bind="click: function() { $root.addFileResources('','') }" class="btn btn-mini paramAdd">${_('Add')}</a>
             <a data-bind="click: function() { $root.addFileResources('','') }" class="btn btn-mini paramAdd">${_('Add')}</a>
           </div>
           </div>
         </li>
         </li>
-        <li class="nav-header 
+        <li title="${ _('User-Defined Functions') }" class="nav-header
           % if app_name == 'impala':
           % if app_name == 'impala':
             hide
             hide
-          % endif" title="${_("User-Defined Functions")}">
+          % endif
+          ">
           ${_('UDFs')}
           ${_('UDFs')}
         </li>
         </li>
         <li class="white paramContainer
         <li class="white paramContainer
@@ -233,8 +234,14 @@ ${layout.menubar(section='query')}
           <textarea class="hide" tabindex="1" name="query" id="queryField"></textarea>
           <textarea class="hide" tabindex="1" name="query" id="queryField"></textarea>
 
 
           <div class="actions">
           <div class="actions">
-            <button data-bind="click: tryExecuteQuery, visible: !$root.query.isRunning()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
+            % if app_name == 'impala':
+            <button data-bind="click: tryExecuteQuery, visible: !$root.query.isRunning() && $root.query.isFinished()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
             <button data-bind="click: tryCancelQuery, visible: $root.query.isRunning()" class="btn btn-danger" data-loading-text="${ _('Canceling...') }" rel="tooltip" data-original-title="${ _('Cancel the query') }">${ _('Cancel') }</button>
             <button data-bind="click: tryCancelQuery, visible: $root.query.isRunning()" class="btn btn-danger" data-loading-text="${ _('Canceling...') }" rel="tooltip" data-original-title="${ _('Cancel the query') }">${ _('Cancel') }</button>
+            % else:
+            <button data-bind="click: tryExecuteQuery, enable: !$root.query.isRunning(), visible: $root.query.isFinished()" type="button" id="executeQuery" class="btn btn-primary disable-feedback" tabindex="2">${_('Execute')}</button>
+            % endif
+            <button data-bind="click: executeNextStatement, visible: !$root.query.isFinished()" type="button" class="btn btn-primary disable-feedback" tabindex="2">${_('Next')}</button>
+
             <button data-bind="click: trySaveQuery, css: {'hide': !$root.query.id() || $root.query.id() == -1}" type="button" class="btn hide">${_('Save')}</button>
             <button data-bind="click: trySaveQuery, css: {'hide': !$root.query.id() || $root.query.id() == -1}" type="button" class="btn hide">${_('Save')}</button>
             <button data-bind="click: saveAsModal" type="button" class="btn">${_('Save as...')}</button>
             <button data-bind="click: saveAsModal" type="button" class="btn">${_('Save as...')}</button>
             <button data-bind="click: tryExplainQuery" type="button" id="explainQuery" class="btn">${_('Explain')}</button>
             <button data-bind="click: tryExplainQuery" type="button" id="explainQuery" class="btn">${_('Explain')}</button>
@@ -263,10 +270,10 @@ ${layout.menubar(section='query')}
 
 
     <div class="card-body">
     <div class="card-body">
       <ul class="nav nav-tabs">
       <ul class="nav nav-tabs">
+        <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
         <!-- ko if: !query.explain() -->
         <!-- ko if: !query.explain() -->
         <li><a href="#log" data-toggle="tab">${_('Log')}</a></li>
         <li><a href="#log" data-toggle="tab">${_('Log')}</a></li>
         <!-- /ko -->
         <!-- /ko -->
-        <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
         <!-- ko if: !query.explain() -->
         <!-- ko if: !query.explain() -->
         <li><a href="#columns" data-toggle="tab">${_('Columns')}</a></li>
         <li><a href="#columns" data-toggle="tab">${_('Columns')}</a></li>
         <li><a href="#results" data-toggle="tab">${_('Results')}</a></li>
         <li><a href="#results" data-toggle="tab">${_('Results')}</a></li>
@@ -279,7 +286,7 @@ ${layout.menubar(section='query')}
 
 
       <div class="tab-content">
       <div class="tab-content">
         <div class="tab-pane" id="query">
         <div class="tab-pane" id="query">
-          <pre data-bind="text: viewModel.query.query()"></pre>
+          <pre data-bind="text: viewModel.query.statement()"></pre>
         </div>
         </div>
         <!-- ko if: query.explain() -->
         <!-- ko if: query.explain() -->
         <div class="tab-pane" id="explanation">
         <div class="tab-pane" id="explanation">
@@ -1809,12 +1816,8 @@ $(document).ready(function () {
   });
   });
 });
 });
 
 
-// @TODO: Make file resources list automatically generated.
 // @TODO: Improve resize logs to be more relative. See FF versus Chrome.
 // @TODO: Improve resize logs to be more relative. See FF versus Chrome.
-// @TODO: Close
 // @TODO: Stop operation
 // @TODO: Stop operation
-// @TODO: Re-add filebrowser to file resources.
-// @TODO: Re-add tooltips for execute query button
 // @TODO: Re-add download query for impala
 // @TODO: Re-add download query for impala
 // @TODO: Re-enable type ahead for settings
 // @TODO: Re-enable type ahead for settings
 </script>
 </script>

+ 2 - 2
apps/beeswax/src/beeswax/templates/list_designs.mako

@@ -16,7 +16,7 @@
 <%!
 <%!
     import time
     import time
     from django.template.defaultfilters import timesince
     from django.template.defaultfilters import timesince
-    from django.utils.encoding import force_unicode    
+    from django.utils.encoding import force_unicode
     from django.utils.translation import ugettext as _
     from django.utils.translation import ugettext as _
 
 
     from desktop.views import commonheader, commonfooter
     from desktop.views import commonheader, commonfooter
@@ -69,7 +69,7 @@ ${ layout.menubar(section='saved queries') }
       </%def>
       </%def>
 
 
       <%def name="creation()">
       <%def name="creation()">
-        <div class="btn-toolbar" style="display: inline; vertical-align: middle">          
+        <div class="btn-toolbar" style="display: inline; vertical-align: middle">
           <a class="btn" href="${ url(app_name + ':execute_query') }" title="${_('Create new %s' % noun)}"><i class="fa fa-plus-circle"></i> ${_('New %s' % noun)}</a>
           <a class="btn" href="${ url(app_name + ':execute_query') }" title="${_('Create new %s' % noun)}"><i class="fa fa-plus-circle"></i> ${_('New %s' % noun)}</a>
           <a class="btn" href="${ url(app_name + ':list_trashed_designs') }" title="${_('Go to the trash')}"><i class="fa fa-trash-o"></i> ${_('View trash')}</a>
           <a class="btn" href="${ url(app_name + ':list_trashed_designs') }" title="${_('Go to the trash')}"><i class="fa fa-trash-o"></i> ${_('View trash')}</a>
         </div>
         </div>

+ 1 - 1
apps/beeswax/src/beeswax/templates/list_trashed_designs.mako

@@ -52,7 +52,7 @@ ${layout.menubar(section='saved queries')}
         <div class="btn-toolbar" style="display: inline; vertical-align: middle">
         <div class="btn-toolbar" style="display: inline; vertical-align: middle">
           <button id="emptyTrashBtn" class="btn" title="${_('Empty trash')}" data-bind="enabled: availableSavedQueries().length > 0">
           <button id="emptyTrashBtn" class="btn" title="${_('Empty trash')}" data-bind="enabled: availableSavedQueries().length > 0">
             <i class="fa fa-fire"></i> ${_('Empty trash')}
             <i class="fa fa-fire"></i> ${_('Empty trash')}
-          </button>        
+          </button>
           <button id="viewQueriesBtn" class="btn" title="${_('View queries')}">
           <button id="viewQueriesBtn" class="btn" title="${_('View queries')}">
             <i class="fa fa-home"></i> ${_('Back')}
             <i class="fa fa-home"></i> ${_('Back')}
           </button>
           </button>

+ 22 - 12
apps/beeswax/src/beeswax/tests.py

@@ -458,11 +458,12 @@ for x in sys.stdin:
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
 
 
     content = json.loads(resp.content)
     content = json.loads(resp.content)
-    design_id = content['id']
+    history_id = content['id']
+    query_history = QueryHistory.get(id=history_id)
 
 
-    resp = self.client.get("/beeswax/results/%s/0?format=json" % design_id)
+    resp = self.client.get("/beeswax/results/%s/0?format=json" % history_id)
     content = json.loads(resp.content)
     content = json.loads(resp.content)
-    assert_true('DROP TABLE test_multiple_statements_2' in content, content) # HUE-1843
+    assert_equal('DROP TABLE test_multiple_statements_2', query_history.get_current_statement(), content)
 
 
   def test_multiple_statements_with_result_set(self):
   def test_multiple_statements_with_result_set(self):
     hql = """
     hql = """
@@ -471,16 +472,20 @@ for x in sys.stdin:
     """
     """
 
 
     resp = _make_query(self.client, hql)
     resp = _make_query(self.client, hql)
-    query = hql_query(hql)
 
 
-    handle = self.db.execute_and_wait(query)
+    content = json.loads(resp.content)
+    assert_true('watch_url' in content, content)
+    watch_url = content['watch_url']
+    assert_equal('SELECT foo FROM test', content.get('statement'), content)
+
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    content = fetch_query_result_data(self.client, resp)
 
 
-    assert_true('multiStatementsQuery' in resp.content, resp.content) # HUE-1843
+    assert_false(content.get('is_finished'), content)
 
 
-    resp = self.client.post(reverse('beeswax:watch_query', args=[resp.context['query'].id]))
-    assert_true('Waiting for query' in resp.content, resp.content)
-    assert_true('SELECT count(*) FROM test' in resp.content, resp.content)
+    resp = self.client.post(watch_url, {'next': True})
+    content = json.loads(resp.content)
+    assert_equal('SELECT count(*) FROM test', content.get('statement'), content)
 
 
   def test_multiple_statements_various_queries(self):
   def test_multiple_statements_various_queries(self):
     hql = """
     hql = """
@@ -490,12 +495,16 @@ for x in sys.stdin:
     """
     """
 
 
     resp = _make_query(self.client, hql)
     resp = _make_query(self.client, hql)
-    query = hql_query(hql)
 
 
-    handle = self.db.execute_and_wait(query)
+    content = json.loads(resp.content)
+    assert_equal('CREATE TABLE test_multiple_statements_2 (a int)', content.get('statement'), content)
+
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    content = json.loads(resp.content)
+    assert_equal('SELECT foo FROM test', content.get('statement'), content)
 
 
-    assert_true('SELECT foo FROM test' in resp.content, resp.content) # HUE-1843
+    content = fetch_query_result_data(self.client, resp)
+    assert_true(content.get('is_finished'), content)
 
 
   def test_parallel_queries(self):
   def test_parallel_queries(self):
     """
     """
@@ -751,6 +760,7 @@ for x in sys.stdin:
         'path': target_dir
         'path': target_dir
       }
       }
       resp = self.client.post('/beeswax/api/query/%s/results/save' % qid, save_data, follow=True)
       resp = self.client.post('/beeswax/api/query/%s/results/save' % qid, save_data, follow=True)
+
       resp = wait_for_query_to_finish(self.client, resp, max=60)
       resp = wait_for_query_to_finish(self.client, resp, max=60)
 
 
       # Check that data is right
       # Check that data is right

+ 2 - 1
apps/beeswax/src/beeswax/views.py

@@ -583,7 +583,8 @@ def view_results(request, id, first_row=0):
     'expired': expired,
     'expired': expired,
     'app_name': app_name,
     'app_name': app_name,
     'download': download,
     'download': download,
-    'next_json_set': None
+    'next_json_set': None,
+    'is_finished': query_history.is_finished()
   }
   }
 
 
   if not error:
   if not error:

+ 40 - 5
apps/beeswax/static/js/beeswax.vm.js

@@ -48,7 +48,9 @@ function BeeswaxViewModel(server, query_id) {
       'url': null,
       'url': null,
       'errors': []
       'errors': []
     },
     },
-    'isRunning': false
+    'isRunning': false,
+    'statement': '',
+    'isFinished': true
   };
   };
 
 
   self.server = ko.observable(server);
   self.server = ko.observable(server);
@@ -104,7 +106,7 @@ function BeeswaxViewModel(server, query_id) {
     self.database(design.database);
     self.database(design.database);
     self.query.isParameterized(design.is_parameterized);
     self.query.isParameterized(design.is_parameterized);
     self.query.email(design.email_notify);
     self.query.email(design.email_notify);
-    
+
     self.query.settings.removeAll();
     self.query.settings.removeAll();
     self.query.fileResources.removeAll();
     self.query.fileResources.removeAll();
     self.query.functions.removeAll();
     self.query.functions.removeAll();
@@ -347,6 +349,7 @@ function BeeswaxViewModel(server, query_id) {
     $(document).trigger('execute.query', data);
     $(document).trigger('execute.query', data);
     self.query.explain(false);
     self.query.explain(false);
     self.query.isRunning(true);
     self.query.isRunning(true);
+    self.query.isFinished(true);
     self.query.errors.removeAll();
     self.query.errors.removeAll();
 
 
     var data = {
     var data = {
@@ -368,6 +371,7 @@ function BeeswaxViewModel(server, query_id) {
           self.query.id(data.id);
           self.query.id(data.id);
           self.query.results.url('/' + self.server() + '/results/' + self.query.id() + '/0?format=json');
           self.query.results.url('/' + self.server() + '/results/' + self.query.id() + '/0?format=json');
           self.query.watch.url(data.watch_url);
           self.query.watch.url(data.watch_url);
+          self.query.statement(data.statement);
           self.watchQueryLoop();
           self.watchQueryLoop();
         } else {
         } else {
           self.query.errors.push(data.message);
           self.query.errors.push(data.message);
@@ -382,6 +386,34 @@ function BeeswaxViewModel(server, query_id) {
     $.ajax(request);
     $.ajax(request);
   };
   };
 
 
+  self.executeNextStatement = function() {
+    $(document).trigger('execute.query', data);
+    self.query.explain(false);
+    self.query.isRunning(true);
+    self.query.isFinished(true);
+    self.query.errors.removeAll();
+
+    var data = {
+      'next': true
+    };
+    var request = {
+      url: self.query.watch.url(),
+      dataType: 'json',
+      type: 'POST',
+      success: function(data) {
+        self.query.errors.removeAll();
+        self.query.statement(data.statement);
+        self.query.watch.url(data.watch_url);
+        self.query.results.url('/' + self.server() + '/results/' + self.query.id() + '/0?format=json');
+        self.watchQueryLoop();
+        $(document).trigger('executed.query', data);
+      },
+      error: error_fn,
+      data: data
+    };
+    $.ajax(request);
+  };
+
   self.watchQuery = function() {
   self.watchQuery = function() {
     var data = {
     var data = {
       'query-query': self.query.query(),
       'query-query': self.query.query(),
@@ -442,11 +474,12 @@ function BeeswaxViewModel(server, query_id) {
             }
             }
           }
           }
         } else {
         } else {
+          self.query.statement(data.statement); // In case new no result statement executed
           if (data.log) {
           if (data.log) {
             self.query.watch.logs.push(data.log);
             self.query.watch.logs.push(data.log);
             // scroll logs
             // scroll logs
           }
           }
-          
+
           timer = setTimeout(_fn, TIMEOUT);
           timer = setTimeout(_fn, TIMEOUT);
         }
         }
       });
       });
@@ -464,6 +497,7 @@ function BeeswaxViewModel(server, query_id) {
       type: 'GET',
       type: 'GET',
       success: function(data) {
       success: function(data) {
         self.query.isRunning(false);
         self.query.isRunning(false);
+        self.query.isFinished(data.is_finished);
         if (self.query.results.columns().length == 0){
         if (self.query.results.columns().length == 0){
           self.query.results.columns(data.columns);
           self.query.results.columns(data.columns);
         }
         }
@@ -545,7 +579,7 @@ function BeeswaxViewModel(server, query_id) {
         success: function(data) {
         success: function(data) {
           if (data.status == 0) {
           if (data.status == 0) {
             $(document).trigger('closed.query', data);
             $(document).trigger('closed.query', data);
-            self.resetQuery();
+            //self.resetQuery(); // Would fail multiqueries
           } else {
           } else {
             $(document).trigger('error_close.query');
             $(document).trigger('error_close.query');
           }
           }
@@ -553,7 +587,8 @@ function BeeswaxViewModel(server, query_id) {
         error: function(data) {
         error: function(data) {
           $(document).trigger('error_close.results');
           $(document).trigger('error_close.results');
         },
         },
-        data: data
+        data: data,
+        async: false // for multi query
       };
       };
       $.ajax(request);
       $.ajax(request);
     }
     }