Browse Source

HUE-871 [beeswax] Expired queries should have a better user experience

Set Impala queries as expired right away
Support Impala settings
Always set DB used with Impala as 'default' and do not require Hive metastore for sending queries
Improve error handling of expired result: explain the error and propose to rerun the query
If a query looks good but is actually expired update its status when looking at it
Fix save result into a table of HDFS file when form error
Romain Rigaux 13 năm trước cách đây
mục cha
commit
42aa60d

+ 18 - 12
apps/beeswax/src/beeswax/forms.py

@@ -96,10 +96,11 @@ class SaveForm(forms.Form):
 class SaveResultsForm(DependencyAwareForm):
   """Used for saving the query result data"""
 
-  SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = (_('to a new table'), _('to HDFS directory'))
+  SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = ('to a new table', 'to HDFS directory')
   save_target = forms.ChoiceField(required=True,
                                   choices=common.to_choices(SAVE_TYPES),
-                                  widget=forms.RadioSelect)
+                                  widget=forms.RadioSelect,
+                                  initial=SAVE_TYPE_TBL)
   target_table = common.HiveIdentifierField(
                                   label=_t("Table Name"),
                                   required=False,
@@ -116,16 +117,21 @@ class SaveResultsForm(DependencyAwareForm):
     self.db = kwargs.pop('db', None)
     super(SaveResultsForm, self).__init__(*args, **kwargs)
 
-  def clean_target_table(self):
-    tbl = self.cleaned_data.get('target_table')
-    if tbl:
-      try:
-        if self.db is not None:
-          self.db.get_table("default", tbl)
-        raise forms.ValidationError(_('Table already exists'))
-      except hive_metastore.ttypes.NoSuchObjectException:
-        pass
-    return tbl
+  def clean(self):
+    cleaned_data = super(SaveResultsForm, self).clean()
+
+    if cleaned_data.get('save_target') == SaveResultsForm.SAVE_TYPE_TBL:
+      tbl = cleaned_data.get('target_table')
+      if tbl:
+        try:
+          if self.db is not None:
+            self.db.get_table('default', tbl) # Assumes 'default' DB
+          self._errors['target_table'] = self.error_class([_('Table already exists')])
+          del cleaned_data['target_table']
+        except hive_metastore.ttypes.NoSuchObjectException:
+          pass
+
+    return cleaned_data
 
 
 class HQLForm(forms.Form):

+ 6 - 4
apps/beeswax/src/beeswax/server/beeswax_lib.py

@@ -38,7 +38,6 @@ from beeswax.server.dbms import Table, DataTable
 LOG = logging.getLogger(__name__)
 
 
-
 class BeeswaxTable(Table):
 
   def __init__(self, table_obj):
@@ -111,10 +110,10 @@ class BeeswaxClient:
   def make_query(self, hql_query):
     # HUE-535 without having to modify Beeswaxd, add 'use database' as first option
     if self.query_server['server_name'] == 'impala':
-      configuration = []
+      configuration = [','.join(['%(key)s=%(value)s' % setting for setting in hql_query.settings])]
     else:
       configuration = ['use ' + hql_query.query.get('database', 'default')]
-    configuration.extend(hql_query.get_configuration())
+      configuration.extend(hql_query.get_configuration())
 
     thrift_query = BeeswaxService.Query(query=hql_query.query['query'], configuration=configuration)
     thrift_query.hadoop_user = self.user.username
@@ -122,7 +121,10 @@ class BeeswaxClient:
 
 
   def get_databases(self, *args, **kwargs):
-    return self.meta_client.get_all_databases()
+    if self.query_server['server_name'] == 'impala':
+      return ['default']
+    else:
+      return self.meta_client.get_all_databases()
 
 
   def get_tables(self, *args, **kwargs):

+ 1 - 1
apps/beeswax/src/beeswax/templates/execute.mako

@@ -42,7 +42,7 @@
     </div>
 
     <div class="actions">
-        <a id="executeQuery" class="btn btn-primary">${_('Execute')}</a>
+        <a id="executeQuery" class="btn btn-primary" tabindex="0">${_('Execute')}</a>
         % if design and not design.is_auto and design.name:
         <a id="saveQuery" class="btn">${_('Save')}</a>
         % endif

+ 1 - 0
apps/beeswax/src/beeswax/templates/list_history.mako

@@ -176,4 +176,5 @@ ${layout.menubar(section='history')}
         $("a[data-row-selector='true']").jHueRowSelector();
     });
 </script>
+
 ${ commonfooter(messages) | n,unicode }

+ 22 - 44
apps/beeswax/src/beeswax/templates/save_results.mako

@@ -15,6 +15,8 @@
 ## limitations under the License.
 <%!
 from desktop.views import commonheader, commonfooter
+from desktop.lib.django_util import extract_field_data
+
 from django.utils.translation import ugettext as _
 %>
 
@@ -23,7 +25,9 @@ from django.utils.translation import ugettext as _
 <%namespace name="util" file="util.mako" />
 
 ${ commonheader(_('Create table from file'), app_name, user, '100px') | n,unicode }
-${layout.menubar(section='history')}
+${layout.menubar(section='query')}
+
+<script src="/static/ext/js/knockout-2.1.0.js" type="text/javascript" charset="utf-8"></script>
 
 <div class="container-fluid">
 % if error_msg:
@@ -36,19 +40,23 @@ ${layout.menubar(section='history')}
       <div class="control-group">
         <div class="controls">
           <label class="radio">
-            <input id="id_save_target_0" type="radio" name="save_target" value="to a new table" checked="checked"/>
-            &nbsp;${_('In a new table')}
+            <input id="id_save_target_0" type="radio" name="save_target" value="${ form.SAVE_TYPE_TBL }" data-bind="checked: toWhere"/>
+            &nbsp;${ _('In a new table') }
           </label>
-          ${comps.field(form['target_table'], notitle=True, placeholder="Table Name")}
+          <span data-bind="visible: toWhere() == 'to a new table'">
+            ${ comps.field(form['target_table'], notitle=True, placeholder='Table Name') }
+          </span>
         </div>
       </div>
       <div class="control-group">
         <div class="controls">
           <label class="radio">
-            <input id="id_save_target_1" type="radio" name="save_target" value="to HDFS directory">
-            &nbsp;${_('In an HDFS directory')}
+            <input id="id_save_target_1" type="radio" name="save_target" value="${ form.SAVE_TYPE_DIR }" data-bind="checked: toWhere">
+            &nbsp;${ _('In an HDFS directory') }
           </label>
-          ${comps.field(form['target_dir'], notitle=True, hidden=True, placeholder=_('Results location'), klass="pathChooser")}
+          <span data-bind="visible: toWhere() == 'to HDFS directory'">
+            ${ comps.field(form['target_dir'], notitle=True, placeholder=_('Results location'), klass='pathChooser') }
+          </span>
         </div>
       </div>
       <div id="fileChooserModal" class="smallModal well hide">
@@ -62,48 +70,12 @@ ${layout.menubar(section='history')}
   </form>
 </div>
 
-
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function () {
-    $("input[name='save_target']").change(function () {
-      $(".errorlist").addClass("hide");
-      $(".control-group.error").removeClass("error");
-      $("input[name='target_table']").removeClass("fieldError");
-      if ($(this).val().indexOf("HDFS") > -1) {
-        $("input[name='target_table']").addClass("hide").val("");
-        $("input[name='target_dir']").removeClass("hide");
-        $(".fileChooserBtn").removeClass("hide");
-      }
-      else {
-        $("input[name='target_table']").removeClass("hide");
-        $("input[name='target_dir']").addClass("hide");
-        $(".fileChooserBtn").addClass("hide");
-      }
-    });
-
-    $("input[name='save_target']").change();
-
-    $("#saveForm").submit(function (e) {
-      if ($("input[name='save_target']:checked").val().indexOf("HDFS") > -1) {
-        if ($.trim($("input[name='target_dir']").val()) == "") {
-          $("input[name='target_dir']").parents(".control-group").addClass("error");
-          $("input[name='target_dir']").parents(".control-group").find(".fileChooserBtn").addClass("btn-danger");
-          return false;
-        }
-      }
-      else {
-        if ($.trim($("input[name='target_table']").val()) == "") {
-          $("input[name='target_table']").parents(".control-group").addClass("error");
-          return false;
-        }
-      }
-      return true;
-    });
-
     $("input[name='target_dir']").after(getFileBrowseButton($("input[name='target_dir']")));
 
     function getFileBrowseButton(inputElement) {
-      return $("<a>").addClass("btn").addClass("fileChooserBtn").addClass("hide").text("..").click(function (e) {
+      return $("<a>").addClass("btn").addClass("fileChooserBtn").text("..").click(function (e) {
         e.preventDefault();
         $("#fileChooserModal").jHueFileChooser({
           onFolderChange:function (filePath) {
@@ -123,6 +95,12 @@ ${layout.menubar(section='history')}
         $("input[name='target_dir']").parents(".control-group").find(".fileChooserBtn").removeClass("btn-danger");
       });
     }
+
+    var viewModel = {
+      toWhere: ko.observable("${ extract_field_data(form['save_target']) }")
+    };
+
+    ko.applyBindings(viewModel);
   });
 </script>
 

+ 51 - 36
apps/beeswax/src/beeswax/templates/watch_results.mako

@@ -59,48 +59,50 @@ ${layout.menubar(section='query')}
 </style>
 
 <div class="container-fluid">
-	<h1>${_('Query Results:')} ${util.render_query_context(query_context)}</h1>
+  <h1>${_('Query Results:')} ${ util.render_query_context(query_context) }</h1>
   <div id="expand"><i class="icon-chevron-right icon-white"></i></div>
-	<div class="row-fluid">
-		<div class="span3">
-			<div class="well sidebar-nav">
+    <div class="row-fluid">
+        <div class="span3">
+            <div class="well sidebar-nav">
         <a id="collapse" class="btn btn-small"><i class="icon-chevron-left" rel="tooltip" title="${_('Collapse this panel')}"></i></a>
-				<ul class="nav nav-list">
-					% if download_urls:
-					<li class="nav-header">${_('Downloads')}</li>
-					<li><a target="_blank" href="${download_urls["csv"]}">${_('Download as CSV')}</a></li>
-					<li><a target="_blank" href="${download_urls["xls"]}">${_('Download as XLS')}</a></li>
-					% endif
-					%if can_save:
-					<li><a data-toggle="modal" href="#saveAs">${_('Save')}</a></li>
-					% endif
-					<%
-			          n_jobs = hadoop_jobs and len(hadoop_jobs) or 0
-			          mr_jobs = (n_jobs == 1) and _('MR Job') or _('MR Jobs')
-			        %>
-				 	% if n_jobs > 0:
-						<li class="nav-header">${mr_jobs} (${n_jobs})</li>
-						% for jobid in hadoop_jobs:
-						    <li><a href="${url("jobbrowser.views.single_job", job=jobid.replace('application', 'job'))}">${ jobid.replace("application_", "") }</a></li>
-						% endfor
-					% else:
-						<li class="nav-header">${mr_jobs}</li>
-						<li>${_('No Hadoop jobs were launched in running this query.')}</li>
-					% endif
-				</ul>
-			</div>
+                <ul class="nav nav-list">
+                    % if download_urls:
+                    <li class="nav-header">${_('Downloads')}</li>
+                    <li><a target="_blank" href="${download_urls["csv"]}">${_('Download as CSV')}</a></li>
+                    <li><a target="_blank" href="${download_urls["xls"]}">${_('Download as XLS')}</a></li>
+                    % endif
+                    %if can_save:
+                    <li><a data-toggle="modal" href="#saveAs">${_('Save')}</a></li>
+                    % endif
+                    <%
+                      n_jobs = hadoop_jobs and len(hadoop_jobs) or 0
+                      mr_jobs = (n_jobs == 1) and _('MR Job') or _('MR Jobs')
+                    %>
+                     % if n_jobs > 0:
+                        <li class="nav-header">${mr_jobs} (${n_jobs})</li>
+                        % for jobid in hadoop_jobs:
+                            <li><a href="${url("jobbrowser.views.single_job", job=jobid.replace('application', 'job'))}">${ jobid.replace("application_", "") }</a></li>
+                        % endfor
+                    % else:
+                        <li class="nav-header">${mr_jobs}</li>
+                        <li>${_('No Hadoop jobs were launched in running this query.')}</li>
+                    % endif
+                </ul>
+            </div>
+
       <div id="jumpToColumnAlert" class="alert hide">
         <button type="button" class="close" data-dismiss="alert">&times;</button>
         <strong>${_('Did you know?')}</strong> ${_('You can click on a row to select a column you want to jump to.')}
       </div>
-		</div>
-		<div class="span9">
+        </div>
+
+        <div class="span9">
       <ul class="nav nav-tabs">
         <li class="active"><a href="#results" data-toggle="tab">
             %if error:
-			            ${_('Error')}
+                  ${_('Error')}
             %else:
-						${_('Results')}
+                  ${_('Results')}
             %endif
         </a></li>
         <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
@@ -116,6 +118,12 @@ ${layout.menubar(section='query')}
               <div class="alert alert-error">
                 <h3>${_('Error!')}</h3>
                 <pre>${ error_message }</pre>
+                % if expired and query_context:
+                    <div class="well">
+                        ${ _('The query result has expired.') }
+                        ${ _('You can rerun it from ') } ${ util.render_query_context(query_context) }
+                    </div>
+                % endif
               </div>
             % else:
             % if expected_first_row != start_row:
@@ -144,10 +152,12 @@ ${layout.menubar(section='query')}
             <div class="pagination pull-right">
               <ul>
               % if start_row != 0:
-                  <li class="prev"><a title="${_('Beginning of List')}" href="${ url(app_name + ':view_results', query.id, 0) }${'?context=' + context_param or '' | n}">&larr; ${_('Beginning of List')}</a></li>
+                  % if app_name != 'impala':
+                      <li class="prev"><a title="${_('Beginning of List')}" href="${ url(app_name + ':view_results', query.id, 0) }${'?context=' + context_param or '' | n}">&larr; ${_('Beginning of List')}</a></li>
+                  % endif
               % endif
               % if has_more and len(results) == 100:
-                  <li><a title="${_('Next page')}" href="${ url(app_name + ':view_results', query.id, next_row) }${'?context=' + context_param or '' | n}">${_('Next Page')} &rarr;</a></li>
+                  <li><a title="${_('Next page')}" href= "${ url(app_name + ':view_results', query.id, next_row) }${'?context=' + context_param or '' | n }">${_('Next Page')} &rarr;</a></li>
               % endif
               </ul>
             </div>
@@ -178,11 +188,12 @@ ${layout.menubar(section='query')}
         % endif
       </div>
 
-		</div>
-	</div>
+        </div>
+    </div>
 </div>
 
 %if can_save:
+## duplication from save_results.mako
 <div id="saveAs" class="modal hide fade">
   <form id="saveForm" action="${url(app_name + ':save_results', query.id) }" method="POST"
         class="form form-inline form-padding-fix">
@@ -324,6 +335,10 @@ ${layout.menubar(section='query')}
         $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
       }
 
+      % if app_name == 'impala':
+          $("#collapse").click();
+          $(".sidebar-nav, #expand").hide();
+      % endif
     });
 </script>
 

+ 13 - 9
apps/beeswax/src/beeswax/views.py

@@ -544,10 +544,6 @@ def watch_query(request, id):
   handle, state = _get_query_handle_and_state(query_history)
   query_history.save_state(state)
 
-
-  # Query finished?
-#  if state == models.QueryHistory.STATE.expired:
-#    raise PopupException(_("The result of this query has expired."))
   if query_history.is_success():
     return format_preserving_redirect(request, on_success_url, request.GET)
   elif query_history.is_failure():
@@ -610,6 +606,7 @@ def view_results(request, id, first_row=0):
   fetch_error = False
   error_message = ''
   log = ''
+  app_name = get_app_name(request)
 
   query_history = authorized_get_history(request, id, must_exist=True)
   db = dbms.get(request.user, query_history.get_query_server_config())
@@ -618,6 +615,13 @@ def view_results(request, id, first_row=0):
   context_param = request.GET.get('context', '')
   query_context = _parse_query_context(context_param)
 
+  # Update the status as expired should not be accessible
+  # Impala does not support startover for now
+  expired = state == models.QueryHistory.STATE.expired
+  if expired or app_name == 'impala':
+    state = models.QueryHistory.STATE.expired
+    query_history.save_state(state)
+
   # Retrieve query results
   try:
     results = db.fetch(handle, start_over, 100)
@@ -631,7 +635,7 @@ def view_results(request, id, first_row=0):
     error_message, log = expand_exception(ex, db)
 
   # Handle errors
-  error = fetch_error or results is None
+  error = fetch_error or results is None or expired
 
   context = {
     'error': error,
@@ -644,13 +648,15 @@ def view_results(request, id, first_row=0):
     'query_context': query_context,
     'can_save': False,
     'context_param': context_param,
+    'expired': expired,
+    'app_name': app_name
   }
 
   if not error:
     download_urls = {}
     if downloadable:
       for format in common.DL_FORMATS:
-        download_urls[format] = reverse(get_app_name(request) + ':download', kwargs=dict(id=str(id), format=format))
+        download_urls[format] = reverse(app_name + ':download', kwargs=dict(id=str(id), format=format))
 
     save_form = beeswax.forms.SaveResultsForm()
     results.start_row = first_row
@@ -684,9 +690,7 @@ def save_results(request, id):
     # Make sure the result is available.
     # Note that we may still hit errors during the actual save
     if not query_history.is_success():
-    #if state != models.QueryHistory.STATE.available:
       if query_history.is_failure():
-      #if state in (models.QueryHistory.STATE.failed, models.QueryHistory.STATE.expired):
         msg = _('This query has %(state)s. Results unavailable.') % {'state': state}
       else:
         msg = _('The result of this query is not available yet.')
@@ -1214,7 +1218,7 @@ def _get_query_handle_and_state(query_history):
   handle = query_history.get_handle()
 
   if handle is None:
-    raise PopupException(_("Failed to retrieve query state from the Beeswax Server."))
+    raise PopupException(_("Failed to retrieve query state from the Query Server."))
 
   state = dbms.get(query_history.owner, query_history.get_query_server_config()).get_state(handle)