Browse Source

HUE-1848 [metastore] Watch query page with the new API

Remove deprecated views.
remove watch_query.
Update execute_directly view to not use contexts.
Should re-add contexts later.
No changes to metastore required.
Fix tests as well.
Abraham Elmahrek 12 years ago
parent
commit
6b4ccbf

+ 10 - 3
apps/beeswax/src/beeswax/api.py

@@ -403,9 +403,12 @@ def save_results(request, query_history_id):
           response['path'] = target_dir
           response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
         elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
-          db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
+          query_history = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
+          response['id'] = query_history.id
+          response['query'] = query_history.query
           response['type'] = 'hive-table'
           response['path'] = form.cleaned_data['target_table']
+          response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
       except Exception, ex:
         error_msg, log = expand_exception(ex, db)
         response['message'] = _('The result could not be saved: %s.') % error_msg
@@ -434,17 +437,21 @@ def design_to_dict(design):
 
 
 def query_history_to_dict(request, query_history):
-  return {
+  query_history_dict = {
     'id': query_history.id,
     'state': query_history.last_state,
     'query': query_history.query,
     'has_results': query_history.has_results,
     'statement_number': query_history.statement_number,
-    'design': design_to_dict(query_history.design),
     'watch_url': reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}),
     'results_url': reverse(get_app_name(request) + ':view_results', kwargs={'id': query_history.id, 'first_row': 0})
   }
 
+  if query_history.design:
+    query_history_dict['design'] = design_to_dict(query_history.design)
+
+  return query_history_dict
+
 
 def get_query_form(request):
   # Get database choices

+ 1 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -249,7 +249,6 @@ class HiveServer2Dbms(object):
 
       hql = 'CREATE TABLE `%s.%s` AS %s' % (database, target_table, design.query['query'])
       query_history = self.execute_statement(hql)
-      url = redirect(reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:describe_table', args=[database, target_table]))
     else:
       # Case 2: The results are in some temporary location
       # Beeswax backward compatibility and optimization
@@ -298,7 +297,7 @@ class HiveServer2Dbms(object):
         raise ex
       url = format_preserving_redirect(request, reverse('metastore:index'))
 
-    return url
+    return query_history
 
 
   def use(self, database):

+ 172 - 51
apps/beeswax/src/beeswax/templates/execute.mako

@@ -27,7 +27,7 @@ ${layout.menubar(section='query')}
 
 <div id="query-editor" class="container-fluid hide section">
 <div class="row-fluid">
-<div class="span2">
+<div class="span2" id="advanced-settings">
   <form id="advancedSettingsForm" action="" method="POST" class="form form-horizontal">
     <div class="sidebar-nav">
       <ul class="nav nav-list">
@@ -188,7 +188,7 @@ ${layout.menubar(section='query')}
 </div>
 
 <div id="querySide" class="span8">
-  <div class="card card-small">
+  <div id="queryContainer" class="card card-small">
     <div style="margin-bottom: 30px">
         % if can_edit_name:
         <h1 class="card-heading simple">
@@ -1619,41 +1619,6 @@ function getLastDatabase(server) {
 }
 
 
-// Knockout
-function clickHard(el) {
-  var timer = setInterval(function () {
-    if ($(el).length > 0) {
-      $(el).click();
-      clearInterval(timer);
-    }
-  }, 100);
-}
-
-viewModel = new BeeswaxViewModel("${app_name}");
-% if query:
-  viewModel.design.history.id(${query.id});
-  viewModel.fetchQueryHistory();
-  $(document).on('fetched.query', function(e) {
-    viewModel.watchQueryLoop();
-  });
-% elif design.id:
-  viewModel.design.id(${design.id});
-  viewModel.fetchDesign();
-% endif
-if (viewModel.design.id() > 0 || viewModel.design.history.id() > 0) {
-  // Code mirror and ko.
-  var codeMirrorSubscription = viewModel.design.query.subscribe(function(value) {
-    codeMirror.setValue(value);
-    codeMirrorSubscription.dispose();
-  });
-}
-viewModel.design.fileResources.subscribe(function() {
-  // File chooser button for file resources.
-  $(".pathChooser:not(:has(~ button))").after(getFileBrowseButton($(".pathChooser:not(:has(~ button))")));
-});
-viewModel.fetchDatabases();
-ko.applyBindings(viewModel);
-
 // Server error handling.
 $(document).on('server.error', function (e, data) {
   $(document).trigger('error', "${_('Server error occured: ')}" + data.message ? data.message : data.error);
@@ -1777,57 +1742,146 @@ $(document).ready(function () {
 
 // Routie
 $(document).ready(function () {
+  function queryPageComponents() {
+    $('#advanced-settings').show();
+    $('#navigator').show();
+    $('#queryContainer').show();
+    $('a[href="#query"]').parent().show();
+    if (!$('#querySide').hasClass('span8')) {
+      $('#querySide').addClass('span8');
+    }
+  }
+
+  function watchPageComponents() {
+    $('#advanced-settings').hide();
+    $('#navigator').hide();
+    $('#queryContainer').hide();
+    $('a[href="#query"]').parent().hide();
+    if ($('#querySide').hasClass('span8')) {
+      $('#querySide').removeClass('span8');
+    }
+  }
+
+  function queryPage() {
+    queryPageComponents();
+    $('.resultsContainer').hide();
+    $('.resultsContainer .watch-query').hide();
+    $('.resultsContainer .view-query-results').hide();
+  }
+
+  function queryLogPage() {
+    queryPageComponents();
+    $('.resultsContainer').show();
+    $('.resultsContainer .watch-query').show();
+    $('.resultsContainer .view-query-results').hide();
+  }
+
+  function queryResultsPage() {
+    queryPageComponents();
+    $('.resultsContainer').show();
+    $('.resultsContainer .watch-query').hide();
+    $('.resultsContainer .view-query-results').show();
+  }
+
+  function parametersPage() {
+    queryPageComponents();
+    $('.resultsContainer').hide();
+    $('.resultsContainer .watch-query').hide();
+    $('.resultsContainer .view-query-results').hide();
+  }
+
+  function watchLogsPage() {
+    watchPageComponents();
+    $('.resultsContainer').show();
+    $('.resultsContainer .watch-query').show();
+    $('.resultsContainer .view-query-results').hide();
+  }
+
+  function watchResultsPage() {
+    watchPageComponents();
+    $('.resultsContainer').show();
+    $('.resultsContainer .watch-query').hide();
+    $('.resultsContainer .view-query-results').show();
+  }
+
   routie({
     'query': function () {
       showSection('query-editor');
-      $('.resultsContainer').hide();
+      queryPage();
+
       codeMirror.setSize("99%", $(window).height() - 270 - $("#queryPane .alert-error").outerHeight() - $(".nav-tabs").outerHeight());
     },
     'query/execute/params': function () {
       if (viewModel.design.parameters().length == 0) {
         routie('query');
       }
+
       showSection('execute-parameter-selection');
+      parametersPage();
     },
     'query/explain/params': function () {
       if (viewModel.design.parameters().length == 0) {
         routie('query');
       }
+
       showSection('explain-parameter-selection');
+      parametersPage();
     },
     'query/logs': function () {
       if (viewModel.design.watch.logs().length == 0 && viewModel.design.watch.errors().length == 0) {
         routie('query');
       }
-      codeMirror.setSize("99%", 100);
-      $('.resultsContainer').show();
-      $('.resultsContainer .watch-query').show();
-      $('.resultsContainer .view-query-results').hide();
+
+      showSection('query-editor');
+      queryLogPage();
+
       clickHard('.resultsContainer .nav-tabs a[href="#log"]');
+
+      codeMirror.setSize("99%", 100);
     },
     'query/results': function () {
+      showSection('query-editor');
+      queryResultsPage();
+
+      $("html, body").animate({ scrollTop: ($(".resultsContainer").position().top - 80) + "px" });
+
       codeMirror.setSize("99%", 100);
-      $('.resultsContainer').show();
-      $('.resultsContainer .watch-query').hide();
-      $('.resultsContainer .view-query-results').show();
+
       clickHard('.resultsContainer .nav-tabs a[href="#results"]');
-      $("html, body").animate({ scrollTop: ($(".resultsContainer").position().top - 80) + "px" });
     },
     'query/explanation': function () {
       if (! viewModel.design.results.explanation()) {
         routie('query');
       }
-      codeMirror.setSize("99%", 100);
-      $('.resultsContainer').show();
-      $('.resultsContainer .watch-query').hide();
-      $('.resultsContainer .view-query-results').show();
+
+      showSection('query-editor');
+      queryResultsPage();
+
       clickHard('.resultsContainer .nav-tabs a[href="#explanation"]');
+
+      codeMirror.setSize("99%", 100);
+    },
+    'watch/logs': function() {
+      showSection('query-editor');
+      watchLogsPage();
+
+      clickHard('.resultsContainer .nav-tabs a[href="#log"]');
+    },
+    'watch/results': function() {
+      showSection('query-editor');
+      watchResultsPage();
+
+      clickHard('.resultsContainer .nav-tabs a[href="#results"]');
     },
-    '': function () {
+    '*': function () {
       routie('query');
     }
   });
+});
 
+
+// Event setup
+function queryEvents() {
   $(document).on('fetched.parameters', function () {
     if (viewModel.design.parameters().length > 0) {
       routie('query/execute/params');
@@ -1853,7 +1907,74 @@ $(document).ready(function () {
   $(document).on('execute.query', function() {
     routie('query');
   });
+  $(document).ready(function() {
+    routie('query');
+  });
+}
+
+function watchEvents() {
+  $(document).ready(function() {
+    routie('watch/logs');
+  });
+  $(document).on('error_watch.query', function () {
+    routie('watch/logs');
+  });
+  $(document).on('fetched.results', function () {
+    routie('watch/results');
+  });
+}
+
+
+// Knockout
+viewModel = new BeeswaxViewModel("${app_name}");
+% if query:
+  viewModel.design.history.id(${query.id});
+  viewModel.fetchQueryHistory();
+  $(document).on('fetched.query', function(e) {
+    viewModel.watchQueryLoop();
+  });
+% elif design.id:
+  viewModel.design.id(${design.id});
+  viewModel.fetchDesign();
+% endif
+if (viewModel.design.id() > 0 || viewModel.design.history.id() > 0) {
+  // Code mirror and ko.
+  var codeMirrorSubscription = viewModel.design.query.subscribe(function(value) {
+    codeMirror.setValue(value);
+    codeMirrorSubscription.dispose();
+  });
+}
+viewModel.design.fileResources.subscribe(function() {
+  // File chooser button for file resources.
+  $(".pathChooser:not(:has(~ button))").after(getFileBrowseButton($(".pathChooser:not(:has(~ button))")));
 });
+viewModel.fetchDatabases();
+ko.applyBindings(viewModel);
+
+
+% if action == 'watch-results':
+  watchEvents();
+  $(document).one('fetched.query', function(e) {
+    viewModel.watchQueryLoop();
+  });
+% elif action == 'watch-redirect':
+  watchEvents();
+  $(document).one('fetched.query', function(e) {
+    viewModel.watchQueryLoop();
+  });
+  $(document).on('stop_watch.query', function(e) {
+    if (viewModel.design.results.errors().length == 0) {
+      window.location.href = "${request.GET['on_success_url']}";
+    }
+  });
+% elif action == 'editor-results':
+  queryEvents();
+  $(document).one('fetched.query', function(e) {
+    viewModel.watchQueryLoop();
+  });
+% else:
+  queryEvents();
+% endif
 
 // @TODO: Stop operation
 // @TODO: Re-add download query for impala

+ 1 - 1
apps/beeswax/src/beeswax/templates/my_queries.mako

@@ -136,7 +136,7 @@ ${layout.menubar(section='my queries')}
               <div class="hueCheckbox runCheck fa"
                 data-edit-url="${ url(app_name + ':execute_design', design_id=query.design.id) }"
                 % if qcontext and query.last_state != models.QueryHistory.STATE.expired.index:
-                  data-view-url="${ url(app_name + ':watch_query', id=query.id) }?context=${qcontext|u}"
+                  data-view-url="${ url(app_name + ':watch_query_history', query_history_id=query.id) }?context=${qcontext|u}"
                 % endif
                 data-row-selector-exclude="true"></div>
             </td>

+ 1 - 1
apps/beeswax/src/beeswax/templates/watch_results.mako

@@ -112,7 +112,7 @@ ${layout.menubar(section='query')}
               <strong>${_('Multi-statement query')}</strong></br>
               ${_('Hue stopped as one of your query contains some results.') }
               ${_('Click on') }
-              <form action="${ url(app_name + ':watch_query', query.id) }?context=${ query.design.get_query_context() }" method="POST">
+              <form action="${ url(app_name + ':watch_query_history', query.id) }?context=${ query.design.get_query_context() }" method="POST">
                 <input type="submit" value="${ _("next") }"/ class="btn btn-primary">
               </form>
               ${_('to continue execution of the remaining statements.') }

+ 0 - 170
apps/beeswax/src/beeswax/templates/watch_wait.mako

@@ -1,170 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-<%!
-from desktop.views import commonheader, commonfooter
-from django.utils.encoding import force_unicode
-from django.utils.translation import ugettext as _
-%>
-
-<%namespace name="layout" file="layout.mako" />
-<%namespace name="util" file="util.mako" />
-
-${ commonheader(_('Waiting for query...'), app_name, user) | n,unicode }
-${layout.menubar(section='query')}
-
-## Required for unit tests
-<!-- <meta http-equiv="refresh" content="3;${url(app_name + ':watch_query', query.id)}?${fwd_params}" /> -->
-
-<div class="container-fluid">
-    <div class="row-fluid">
-        <div class="span3">
-            % if app_name == 'impala':
-              <li class="nav">
-                <div class="control-group">
-                  <button id="cancel-btn" class="btn btn-small" data-loading-text="${ _('Canceling...') }" rel="tooltip" data-placement="right" data-original-title="${ _('Cancel the query') }">
-                    ${ _('Cancel') }
-                  </button>
-                </div>
-              </li>
-            % endif
-            <div class="sidebar-nav
-                % if app_name == 'impala':
-                  hide
-                % endif
-            ">
-                <ul class="nav nav-list">
-                    <%
-                      n_jobs = hadoop_jobs and len(hadoop_jobs) or 0
-                      mr_jobs = (n_jobs == 1) and _('MR Job') or _('MR Jobs')
-                    %>
-                     % if n_jobs > 0:
-                        <li id="jobsHeader" class="nav-header">${mr_jobs} (${n_jobs})</li>
-                        % for jobid in hadoop_jobs:
-                            <li><a class="jobLink" href="${url("jobbrowser.views.single_job", job=jobid)}">${jobid.replace("job_", "")}</a></li>
-                        % endfor
-                    % else:
-                        <li id="jobsHeader" class="nav-header">${mr_jobs}</li>
-                        <li class="white jobLink">${_('No Hadoop jobs were launched in running this query.')}</li>
-                    % endif
-                </ul>
-            </div>
-        </div>
-        <div class="span9">
-          <div class="card card-small">
-            <h1 class="card-heading simple">${_('Waiting for query...')} ${util.render_query_context(query_context)}</h1>
-            <div class="card-body">
-              <p>
-                <ul class="nav nav-tabs">
-                    <li class="active"><a href="#log" data-toggle="tab">${_('Log')}</a></li>
-                    <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
-                </ul>
-
-                <div class="tab-content">
-                    <div class="active tab-pane" id="log">
-                        <pre>${ force_unicode(log) }</pre>
-                    </div>
-                    <div class="tab-pane" id="query">
-                        <pre>${ query.get_current_statement() }</pre>
-                    </div>
-                </div>
-              </p>
-            </div>
-          </div>
-        </div>
-    </div>
-</div>
-
-<script>
-  $(document).ready(function(){
-    var fwdUrl = "${url(app_name + ':watch_query', query.id)}?${fwd_params}";
-    var labels = {
-      MRJOB: "${_('MR Job')}",
-      MRJOBS: "${_('MR Jobs')}"
-    }
-
-    resizeLogs();
-    refreshView();
-    var logsAtEnd = true;
-
-    function refreshView() {
-      $.ajaxSetup({ cache: false });
-      $.getJSON("${url(app_name + ':api_watch_query_refresh_json', query.id)}", function (data) {
-        if (data.isSuccess || data.isFailure) {
-          location.href = fwdUrl;
-        }
-        if (data.jobs && data.jobs.length > 0) {
-          $(".jobLink").remove();
-          $("#jobsHeader").text((data.jobs.length > 1 ? labels.MRJOBS : labels.MRJOB) + " (" + data.jobs.length + ")");
-          for (var i = 0; i < data.jobs.length; i++) {
-            $("#jobsHeader").after($("<li>").addClass("jobLink").html("<a href=\"" + data.jobUrls[data.jobs[i]] + "\">" + data.jobs[i].replace("job_", "") + "</a>"));
-          }
-        }
-        var _logsEl = $("#log pre");
-        % if app_name == 'impala':
-          _logsEl.text($.trim(data.log));
-        % else:
-          var newLines = data.log.split("\n").slice(_logsEl.text().split("\n").length);
-          _logsEl.text(_logsEl.text() + newLines.join("\n"));
-        % endif
-
-        if (logsAtEnd) {
-          _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
-        }
-        window.setTimeout(refreshView, 1000);
-      });
-    }
-
-    $(window).resize(function () {
-      resizeLogs();
-    });
-
-    $("a[href='#log']").on("shown", function () {
-      resizeLogs();
-    });
-
-    $("#log pre").scroll(function () {
-      if ($(this).scrollTop() + $(this).height() + 20 >= $(this)[0].scrollHeight) {
-        logsAtEnd = true;
-      }
-      else {
-        logsAtEnd = false;
-      }
-    });
-
-    function resizeLogs() {
-      $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
-    }
-
-    % if app_name == 'impala':
-      $("#cancel-btn").click(function() {
-        var _this = this;
-        $(_this).button('loading');
-        $.post("${ url(app_name + ':cancel_operation', query.id) }",
-          function(response) {
-            if (response['status'] != 0) {
-              $(document).trigger("error", "${ _('Problem: ') }" + response['message']);
-            } else {
-              $(document).trigger("info", "${ _('Query canceled!') }")
-            }
-          }
-        );
-        return false;
-      });
-    % endif
-  });
-</script>
-
-${ commonfooter(messages) | n,unicode }

+ 3 - 1
apps/beeswax/src/beeswax/test_base.py

@@ -175,6 +175,7 @@ def get_shared_beeswax_server():
 
 def wait_for_query_to_finish(client, response, max=30.0):
   # Take a async API execute_query() response in input
+
   start = time.time()
   sleep_time = 0.05
 
@@ -191,11 +192,12 @@ def wait_for_query_to_finish(client, response, max=30.0):
     time.sleep(sleep_time)
     sleep_time = min(1.0, sleep_time * 2) # Capped exponential
     if (time.time() - start) > max:
-      message = "Query took too long! %d seconds" % (time.time() - start,)
+      message = "Query took too long! %d seconds" % (time.time() - start)
       LOG.warning(message)
       raise Exception(message)
 
     response = client.get(watch_url, follow=True)
+
   return response
 
 

+ 28 - 42
apps/beeswax/src/beeswax/tests.py

@@ -114,7 +114,6 @@ class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
     response = _make_query(self.client, "CREATE TABLE test (foo INT)", wait=True)
     content = json.loads(response.content)
     assert_true("AlreadyExistsException" in content.get('message'), content)
-    assert_true("Table test already exists" in content.get('message'), content)
 
   def test_configuration(self):
     # No HS2 API
@@ -262,13 +261,11 @@ for x in sys.stdin:
 
   def test_query_with_simple_errors(self):
     hql = "SELECT KITTENS ARE TASTY"
-    resp = _make_query(self.client, hql, name='tasty kittens', wait=True, local=False)
-    assert_true("ParseException line" in json.loads(resp.content)["error"])
+    resp = _make_query(self.client, hql, name='tasty kittens', wait=False, local=False)
     assert_true("ParseException line" in json.loads(resp.content)["message"])
 
     # Watch page will fail as operationHandle=None
     query_id = self._verify_query_state(beeswax.models.QueryHistory.STATE.failed)
-    assert_raises(TypeError, self.client.get, '/beeswax/watch/%s' % (query_id,), follow=True)
 
   def test_sync_query_exec(self):
     # Execute Query Synchronously, set fetch size and fetch results
@@ -420,7 +417,7 @@ for x in sys.stdin:
     # Selecting from latin1 table should not blow up
     query = u"SELECT * FROM test_latin1 WHERE bar='%s'" % (unichr(200),)
     response = _make_query(self.client, query, wait=True)
-    assert_true(response.context.has_key("results"), "selecting from latin1 table should not blow up")
+    assert_true('results' in response.context, "selecting from latin1 table should not blow up")
 
     # Describe table should be fine with non-ascii comment
     response = self.client.get('/beeswax/table/default/test_utf8')
@@ -586,9 +583,19 @@ for x in sys.stdin:
 
     # Retrieve that design. It's the first one since it's most recent
     design = beeswax.models.SavedQuery.objects.all()[0]
-    resp = cli.get('/beeswax/execute/%s' % (design.id,))
+    resp = cli.get('/beeswax/execute/design/%s' % design.id)
+    assert_true('query' in resp.context, resp.context)
+    assert_true(resp.context['query'] is None, resp.context)
     assert_equal(design, resp.context['design'], resp.context)
 
+    # Retrieve that query history. It's the first one since it's most recent
+    query_history = beeswax.models.QueryHistory.objects.all()[0]
+    resp = cli.get('/beeswax/execute/query/%s' % query_history.id)
+    assert_true('query' in resp.context, resp.context)
+    assert_true(resp.context['query'] is not None, resp.context)
+    assert_true('design' in resp.context, resp.context)
+    assert_true(resp.context['design'] is not None, resp.context)
+
     resp = cli.get(reverse('beeswax:api_fetch_saved_design', kwargs={'design_id': design.id}))
     content = json.loads(resp.content)
     assert_true('SELECT bogus FROM test' in content['design']['query'], content)
@@ -762,7 +769,8 @@ for x in sys.stdin:
         'path': target_dir
       }
       resp = self.client.post('/beeswax/api/query/%s/results/save' % qid, save_data, follow=True)
-
+      context = json.loads(resp.content)
+      resp = self.client.get(context['watch_url'], follow=True)
       resp = wait_for_query_to_finish(self.client, resp, max=60)
 
       # Check that data is right
@@ -820,13 +828,16 @@ for x in sys.stdin:
         'type': 'hive-table',
         'path': target_tbl
       }
-      resp = self.client.post('/beeswax/api/query/%s/results/save' % (qid,), save_data, follow=True)
+      resp = self.client.post('/beeswax/api/query/%s/results/save' % qid, save_data, follow=True)
+      context = json.loads(resp.content)
+      resp = self.client.get(context['watch_url'], follow=True)
       wait_for_query_to_finish(self.client, resp, max=120)
 
       # Check that data is right. The SELECT may not give us the whole table.
-      resp = _make_query(self.client, 'SELECT * FROM %s' % (target_tbl,), wait=True, local=False)
+      resp = _make_query(self.client, 'SELECT * FROM %s' % target_tbl, wait=True, local=False)
+      resp = fetch_query_result_data(self.client, resp)
       for i in xrange(90):
-        assert_equal([i, '0x%x' % (i,)], resp.context['results'][i])
+        assert_equal([i, '0x%x' % (i,)], context['results'][i])
 
     TARGET_TBL_ROOT = 'test_copy'
 
@@ -895,27 +906,9 @@ for x in sys.stdin:
       'create': 'Create table',
     }, follow=True)
 
-    templates = [_template.filename for _template in resp.template]
-
-    if any(['watch_wait.mako' in template for template in templates]):
-      assert_equal_mod_whitespace("""
-          CREATE EXTERNAL TABLE `default.my_table`
-          (
-           `my_col` string
-          )
-          COMMENT "Yo>>>>dude"
-          ROW FORMAT DELIMITED
-            FIELDS TERMINATED BY ','
-            COLLECTION ITEMS TERMINATED BY '\\002'
-            MAP KEYS TERMINATED BY '\\003'
-            STORED AS TextFile LOCATION "/tmp/foo"
-      """, resp.context['query'].query)
-      assert_true('on_success_url=%2Fmetastore%2Ftable%2Fdefault%2Fmy_table' in resp.context['fwd_params'], resp.context['fwd_params'])
-    else:
-      # Create was fast
-      templates = [_template.filename for _template in resp.template]
-      assert_true(any(['describe_table.mako' in template for template in templates]), templates)
-      assert_true('Table : my_table' in resp.content, resp.content)
+    # Ensure we can see table.
+    response = self.client.get("/metastore/table/default/my_table")
+    assert_true("my_col" in response.content)
 
 
   def test_create_table_timestamp(self):
@@ -1140,7 +1133,7 @@ for x in sys.stdin:
       'cols-2-column_type': 'string',
       'cols-next_form_id': '3',
     }, follow=True)
-
+    resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
     resp = wait_for_query_to_finish(self.client, resp, max=180.0)
 
     # Check data is in the table (by describing it)
@@ -1172,7 +1165,7 @@ for x in sys.stdin:
       'cols-next_form_id': '3',
       'removeHeader': 'on'
     }, follow=True)
-
+    resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
     resp = wait_for_query_to_finish(self.client, resp, max=180.0)
 
     # Check data is in the table (by describing it)
@@ -1194,16 +1187,9 @@ for x in sys.stdin:
       'create': 'Create database',
       'use_default_location': True,
     }, follow=True)
-
-    templates = [_template.filename for _template in resp.template]
-
-    if [template for template in templates if "watch_wait.mako" in template]:
-      assert_equal_mod_whitespace("CREATE DATABASE my_db COMMENT \"foo\"", resp.context['query'].query, resp.content)
-    else:
-      # Create was fast
-      assert_true([template for template in templates if 'databases.mako' in template], templates)
-
+    resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
     resp = wait_for_query_to_finish(self.client, resp, max=180.0)
+    resp = self.client.get("/metastore/databases/")
     assert_true('my_db' in resp.context['databases'], resp)
 
 

+ 0 - 2
apps/beeswax/src/beeswax/urls.py

@@ -24,10 +24,8 @@ urlpatterns = patterns('beeswax.views',
   url(r'^execute/?$', 'execute_query', name='execute_query'),
   url(r'^execute/design/(?P<design_id>\d+)$', 'execute_query', name='execute_design'),
   url(r'^execute/query/(?P<query_history_id>\d+)$', 'execute_query', name='watch_query_history'),
-  url(r'^watch/(?P<id>\d+)$', 'watch_query', name='watch_query'),
   url(r'^results/(?P<id>\d+)/(?P<first_row>\d+)$', 'view_results', name='view_results'),
   url(r'^download/(?P<id>\d+)/(?P<format>\w+)$', 'download', name='download'),
-  url(r'^save_results/(?P<id>\d+)$', 'save_results', name='save_results'),
 
   url(r'^my_queries$', 'my_queries', name='my_queries'),
   url(r'^list_designs$', 'list_designs', name='list_designs'),

+ 25 - 225
apps/beeswax/src/beeswax/views.py

@@ -35,18 +35,16 @@ from desktop.lib.django_util import login_notrequired, get_desktop_uri_prefix
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document
 
-from jobsub.parameterization import find_variables, substitute_variables
+from jobsub.parameterization import find_variables
 
 import beeswax.forms
 import beeswax.design
 import beeswax.management.commands.beeswax_install_examples
 
 from beeswax import common, data_export, models
-from beeswax.forms import QueryForm
-from beeswax.design import HQLdesign
 from beeswax.models import SavedQuery, make_query_context, QueryHistory
 from beeswax.server import dbms
-from beeswax.server.dbms import expand_exception, get_query_server_config, QueryServerException
+from beeswax.server.dbms import expand_exception, get_query_server_config
 
 
 LOG = logging.getLogger(__name__)
@@ -342,9 +340,19 @@ def execute_query(request, design_id=None, query_history_id=None):
   """
   View function for executing an arbitrary query.
   """
+  action = 'query'
+
   if query_history_id:
     query = authorized_get_query_history(request, query_history_id, must_exist=True)
     design = query.design
+
+    if 'on_success_url' in request.GET:
+      if request.GET.get('on_success_url'):
+        action = 'watch-redirect'
+      else:
+        action = 'watch-results'
+    else:
+      action = 'editor-results'
   else:
     # Check perms.
     authorized_get_design(request, design_id)
@@ -358,93 +366,26 @@ def execute_query(request, design_id=None, query_history_id=None):
     'design': design,
     'query': query,
     'autocomplete_base_url': reverse(get_app_name(request) + ':api_autocomplete_databases', kwargs={}),
-    'can_edit_name': design.id and not design.is_auto
+    'can_edit_name': design and design.id and not design.is_auto,
+    'action': action
   }
 
   return render('execute.mako', request, context)
 
 
-def watch_query(request, id):
-  """
-  DEPRECATED!!!
-
-  Wait for the query to finish and (by default) displays the results of query id.
-  It understands the optional GET params:
-
-    on_success_url
-      If given, it will be displayed when the query is successfully finished.
-      Otherwise, it will display the view query results page by default.
-
-    context
-      A string of "name:data" that describes the context
-      that generated this query result. It may be:
-        - "table":"<table_name>"
-        - "design":<design_id>
-
-  All other GET params will be passed to on_success_url (if present).
-  """
-  # Coerce types: manage arguments
-  query_history = authorized_get_query_history(request, id, must_exist=True)
-  db = dbms.get(request.user, query_history.get_query_server_config())
-
-  # GET param: context.
-  context_param = request.GET.get('context', '')
-
-  # GET param: on_success_url. Default to view_results
-  results_url = reverse(get_app_name(request) + ':view_results', kwargs={'id': id, 'first_row': 0})
-  if request.GET.get('download', ''):
-    results_url += '?download=true'
-  on_success_url = request.GET.get('on_success_url')
-  if not on_success_url:
-    on_success_url = results_url
-
-  # Go to next statement if asked to continue or when a statement with no dataset finished.
-  if request.method == 'POST' or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
-    try:
-      query_history = db.execute_next_statement(query_history)
-    except Exception, ex:
-      pass
-
-  # Check query state
-  handle, state = _get_query_handle_and_state(query_history)
-  query_history.save_state(state)
-
-  if query_history.is_failure():
-    # When we fetch, Beeswax server will throw us a Exception, which has the
-    # log we want to display.
-    return format_preserving_redirect(request, results_url, request.GET)
-  elif query_history.is_finished() or (query_history.is_success() and query_history.has_results):
-    return format_preserving_redirect(request, on_success_url, request.GET)
-
-  # Still running
-  log = db.get_log(handle)
-
-  # Keep waiting
-  # - Translate context into something more meaningful (type, data)
-  query_context = parse_query_context(context_param)
-
-  return render('watch_wait.mako', request, {
-                'query': query_history,
-                'fwd_params': request.GET.urlencode(),
-                'log': log,
-                'hadoop_jobs': _parse_out_hadoop_jobs(log),
-                'query_context': query_context,
-              })
-
-
 def view_results(request, id, first_row=0):
   """
   Returns the view for the results of the QueryHistory with the given id.
 
   The query results MUST be ready.
-  To display query results, one should always go through the watch_query view.
+  To display query results, one should always go through the execute_query view.
   If the result set has has_result_set=False, display an empty result.
 
   If ``first_row`` is 0, restarts (if necessary) the query read.  Otherwise, just
   spits out a warning if first_row doesn't match the servers conception.
   Multiple readers will produce a confusing interaction here, and that's known.
 
-  It understands the ``context`` GET parameter. (See watch_query().)
+  It understands the ``context`` GET parameter. (See execute_query().)
   """
   first_row = long(first_row)
   start_over = (first_row == 0)
@@ -558,64 +499,6 @@ def view_results(request, id, first_row=0):
     return render('watch_results.mako', request, context)
 
 
-def save_results(request, id):
-  """
-  DEPRECATED. Need to get rid of watch_wait dependency first.
-
-  Save the results of a query to an HDFS directory or Hive table.
-  """
-  query_history = authorized_get_query_history(request, id, must_exist=True)
-
-  app_name = get_app_name(request)
-  server_id, state = _get_query_handle_and_state(query_history)
-  query_history.save_state(state)
-  error_msg, log = None, None
-
-  if request.method == 'POST':
-    if not query_history.is_success():
-      msg = _('This query is %(state)s. Results unavailable.') % {'state': state}
-      raise PopupException(msg)
-
-    db = dbms.get(request.user, query_history.get_query_server_config())
-    form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs)
-
-    if request.POST.get('cancel'):
-      return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id))
-
-    if form.is_valid():
-      try:
-        handle, state = _get_query_handle_and_state(query_history)
-        result_meta = db.get_results_metadata(handle)
-      except Exception, ex:
-        raise PopupException(_('Cannot find query: %s') % ex)
-
-      try:
-        if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
-          target_dir = form.cleaned_data['target_dir']
-          query_history = db.insert_query_into_directory(query_history, target_dir)
-          redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \
-                                + '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir}))
-        elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
-          redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
-      except Exception, ex:
-        error_msg, log = expand_exception(ex, db)
-        raise PopupException(_('The result could not be saved: %s.') % log, detail=ex)
-
-      return redirected
-  else:
-    form = beeswax.forms.SaveResultsForm()
-
-  if error_msg:
-    error_msg = _('Failed to save results from query: %(error)s.') % {'error': error_msg}
-
-  return render('save_results.mako', request, {
-    'action': reverse(get_app_name(request) + ':save_results', kwargs={'id': str(id)}),
-    'form': form,
-    'error_msg': error_msg,
-    'log': log,
-  })
-
-
 def configuration(request):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)
@@ -677,7 +560,7 @@ def query_done_cb(request, server_id):
 
     link = "%s%s" % \
               (get_desktop_uri_prefix(),
-               reverse(get_app_name(request) + ':watch_query', kwargs={'id': query_history.id}))
+               reverse(get_app_name(request) + ':watch_query_history', kwargs={'query_history_id': query_history.id}))
     body = _("%(subject)s. See the results here: %(link)s\n\nQuery:\n%(query)s") % {
                'subject': subject, 'link': link, 'query': query_history.query
              }
@@ -762,21 +645,6 @@ def safe_get_design(request, design_type, design_id=None):
 
   return design
 
-def get_parameterization(request, query_str, form, design, is_explain):
-  """
-  DEPRECATED!!!
-  Figures out whether a design is parameterizable, and, if so,
-  returns a form to fill out.  Returns None if there's no parameterization
-  to do.
-  """
-  if form.query.cleaned_data["is_parameterized"]:
-    parameters_form = make_parameterization_form(query_str)
-    if parameters_form:
-      return render("parameterization.mako", request, dict(
-        form=parameters_form(prefix="parameterization"),
-        design=design,
-        explain=is_explain))
-  return None
 
 def make_parameterization_form(query_str):
   """
@@ -793,67 +661,9 @@ def make_parameterization_form(query_str):
     return None
 
 
-def _run_parameterized_query(request, design_id, explain):
-  """
-  DEPRECATED!!!
-  Given a design and arguments to parameterize that design, runs the query.
-  - explain is a boolean to determine whether to run as an explain or as an
-  execute.
-
-  This is an extra "step" in the flow from execute_query.
-  """
-  design = authorized_get_design(request, design_id, must_exist=True)
-
-  # Reconstitute the form
-  design_obj = beeswax.design.HQLdesign.loads(design.data)
-  query_form = QueryForm()
-  params = design_obj.get_query_dict()
-  params.update(request.POST)
-
-  databases = get_db_choices(request)
-  query_form.bind(params)
-  query_form.query.fields['database'].choices = databases # Could not do it in the form
-
-  if not query_form.is_valid():
-    raise PopupException(_("Query form is invalid: %s") % query_form.errors)
-
-  query_str = query_form.query.cleaned_data["query"]
-  app_name = get_app_name(request)
-  query_server = get_query_server_config(app_name)
-  query_type = SavedQuery.TYPES_MAPPING[app_name]
-
-  parameterization_form_cls = make_parameterization_form(query_str)
-  if not parameterization_form_cls:
-    raise PopupException(_("Query is not parameterizable."))
-
-  parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
-
-  if parameterization_form.is_valid():
-    real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
-    query = HQLdesign(query_form, query_type=query_type)
-    query._data_dict['query']['query'] = real_query
-    try:
-      if explain:
-        return explain_directly(request, query, design, query_server)
-      else:
-        return execute_directly(request, query, query_server, design)
-    except Exception, ex:
-      db = dbms.get(request.user, query_server)
-      error_message, log = expand_exception(ex, db)
-      return render('execute.mako', request, {
-        'action': reverse(get_app_name(request) + ':execute_design'),
-        'design': design,
-        'error_message': error_message,
-        'form': query_form,
-        'log': log,
-        'autocomplete_base_url': reverse(get_app_name(request) + ':api_autocomplete', kwargs={}),
-      })
-  else:
-    return render("parameterization.mako", request, dict(form=parameterization_form, design=design, explain=explain))
-
-
-def execute_directly(request, query, query_server=None, design=None, tablename=None,
-                           on_success_url=None, on_success_params=None, **kwargs):
+def execute_directly(request, query, query_server=None,
+                     design=None, on_success_url=None, on_success_params=None,
+                     **kwargs):
   """
   execute_directly(request, query_msg, tablename, design) -> HTTP response for execution
 
@@ -870,9 +680,6 @@ def execute_directly(request, query, query_server=None, design=None, tablename=N
     design
       The design associated with the query.
 
-    tablename
-      The associated table name for the context.
-
     on_success_url
       Where to go after the query is done. The URL handler may expect an option "context" GET
       param. (See ``watch_query``.) For advanced usage, on_success_url can be a function, in
@@ -892,27 +699,20 @@ def execute_directly(request, query, query_server=None, design=None, tablename=N
   database = query.query.get('database', 'default')
   db.use(database)
 
-  history_obj = db.execute_query(query, design)
+  query_history = db.execute_query(query, design)
 
-  watch_url = reverse(get_app_name(request) + ':watch_query', kwargs={'id': history_obj.id})
-  if 'download' in kwargs and kwargs['download']:
-    watch_url += '?download=true'
+  watch_url = reverse(get_app_name(request) + ':watch_query_history', kwargs={'query_history_id': query_history.id})
 
   # Prepare the GET params for the watch_url
   get_dict = QueryDict(None, mutable=True)
-  # (1) context
-  if design:
-    get_dict['context'] = make_query_context('design', design.id)
-  elif tablename:
-    get_dict['context'] = make_query_context('table', '%s:%s' % (tablename, database))
 
-  # (2) on_success_url
+  # (1) on_success_url
   if on_success_url:
     if callable(on_success_url):
-      on_success_url = on_success_url(history_obj)
+      on_success_url = on_success_url(query_history)
     get_dict['on_success_url'] = on_success_url
 
-  # (3) misc
+  # (2) misc
   if on_success_params:
     get_dict.update(on_success_params)
 

+ 14 - 1
apps/beeswax/static/js/beeswax.vm.js

@@ -141,7 +141,9 @@ function BeeswaxViewModel(server) {
     self.design.history.id(history.id);
     viewModel.design.results.url('/' + viewModel.server() + '/results/' + history.id + '/0?format=json');
     viewModel.design.watch.url('/' + viewModel.server() + '/api/watch/json/' + history.id);
-    self.updateDesign(history.design);
+    if (history.design) {
+      self.updateDesign(history.design);
+    }
   };
 
   self.updateParameters = function(parameters) {
@@ -715,3 +717,14 @@ function getFileBrowseButton(inputElement) {
     $("#chooseFile").modal("show");
   });
 }
+
+
+// utils
+function clickHard(el) {
+  var timer = setInterval(function () {
+    if ($(el).length > 0) {
+      $(el).click();
+      clearInterval(timer);
+    }
+  }, 100);
+}

+ 8 - 10
apps/metastore/src/metastore/tests.py

@@ -17,6 +17,7 @@
 # limitations under the License.
 
 import logging
+import json
 
 from nose.tools import assert_true, assert_equal, assert_false
 
@@ -32,7 +33,7 @@ from useradmin.models import HuePermission, GroupPermission,\
 
 from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
 from beeswax.views import collapse_whitespace
-from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config
+from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config, fetch_query_result_data
 from beeswax.models import QueryHistory
 from beeswax.server import dbms
 from beeswax.test_base import BeeswaxSampleProvider
@@ -57,7 +58,6 @@ def _make_query(client, query, submission_type="Execute",
 
   return res
 
-
 class TestMetastoreWithHadoop(BeeswaxSampleProvider):
   requires_hadoop = True
 
@@ -91,16 +91,13 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
     # Show table data.
     response = self.client.get("/metastore/table/default/test/read", follow=True)
+    response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context['query'].id}), follow=True)
     response = wait_for_query_to_finish(self.client, response, max=30.0)
     # Note that it may not return all rows at once. But we expect at least 10.
-    assert_true(len(response.context['results']) > 10)
-    # Column names
-    assert_true("foo" in response.content)
-    assert_true("bar" in response.content)
+    results = fetch_query_result_data(self.client, response)
+    assert_true(len(results['results']) > 0)
     # This should NOT go into the query history.
     assert_equal(verify_history(self.client, fragment='test'), history_cnt, 'Implicit queries should not be saved in the history')
-    assert_equal(str(response.context['query_context'][0]), 'table')
-    assert_equal(str(response.context['query_context'][1]), 'test:default')
 
   def test_describe_view(self):
     resp = self.client.get('/metastore/table/default/myview')
@@ -140,9 +137,10 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
   def test_browse_partitions(self):
     response = self.client.get("/metastore/table/default/test_partitions/partitions/0", follow=True)
+    response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context['query'].id}), follow=True)
     response = wait_for_query_to_finish(self.client, response, max=30.0)
-    page_context = [context for context in response.context if 'results' in context][0]
-    assert_true(len(page_context['results']) > 10)
+    results = fetch_query_result_data(self.client, response)
+    assert_true(len(results['results']) > 0, results)
 
   def test_drop_multi_tables(self):
     hql = """

+ 7 - 7
apps/metastore/src/metastore/views.py

@@ -81,7 +81,7 @@ def drop_database(request):
       # Can't be simpler without an important refactoring
       design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
       query_history = db.drop_databases(databases, design)
-      url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:databases')
+      url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
       return redirect(url)
     except Exception, ex:
       error_message, log = dbms.expand_exception(ex, db)
@@ -181,7 +181,7 @@ def drop_table(request, database):
       # Can't be simpler without an important refactoring
       design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
       query_history = db.drop_tables(database, tables_objects, design)
-      url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:show_tables')
+      url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables')
       return redirect(url)
     except Exception, ex:
       error_message, log = dbms.expand_exception(ex, db)
@@ -198,8 +198,8 @@ def read_table(request, database, table):
   table = db.get_table(database, table)
 
   try:
-    history = db.select_star_from(database, table)
-    url = reverse('beeswax:watch_query', args=[history.id]) + '?context=table:%s:%s' % (table.name, database)
+    query_history = db.select_star_from(database, table)
+    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=&context=table:%s:%s' % (table.name, database)
     return redirect(url)
   except Exception, e:
     raise PopupException(_('Cannot read table'), detail=e)
@@ -209,7 +209,7 @@ def read_partition(request, database, table, partition_id):
   db = dbms.get(request.user)
   try:
     partition = db.get_partition(database, table, int(partition_id))
-    url = reverse('beeswax:watch_query', args=[partition.id]) + '?context=table:%s:%s' % (table, database)
+    url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': partition.id}) + '?on_success_url=&context=table:%s:%s' % (table, database)
     return redirect(url)
   except Exception, e:
     raise PopupException(_('Cannot read table'), detail=e)
@@ -229,7 +229,7 @@ def load_table(request, database, table):
       try:
         design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
         query_history = db.load_data(database, table, load_form, design)
-        url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + on_success_url
+        url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + on_success_url
         response['status'] = 0
         response['data'] = url
       except Exception, e:
@@ -287,7 +287,7 @@ def analyze_table(request, database, table, column=None):
   if request.POST:
     if column is None:
       query_history = db.analyze_table(database, table)
-      response['redirect'] = reverse('beeswax:watch_query', args=[query_history.id]) + \
+      response['redirect'] = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + \
                                      '?on_success_url=' + reverse('metastore:describe_table',
                                                                   kwargs={'database': database, 'table': table.name})
       response['status'] = 0