Browse Source

[oozie] Fix several inconsistencies

Get job from History now return the fulle Workflow or Coordinator instead of Job
Create coordinator Editor should set the Workflow name in the left panel
The labels for the controls used in uploading files in File Browser should be consistent
beeswax server_interface setting shouldn't list hiveserver2 because it's not supported yet
Fix Beeswax email notification
Fix 101% coordinator progress
Romain Rigaux 13 years ago
parent
commit
1178545c6d

+ 2 - 1
apps/beeswax/src/beeswax/conf.py

@@ -24,7 +24,8 @@ from desktop.lib.conf import Config, coerce_bool
 SERVER_INTERFACE = Config(
 SERVER_INTERFACE = Config(
   key="server_interface",
   key="server_interface",
   help=_("Beeswax or Hive Server 2 Thrift API used. Choices are: 'beeswax' or 'hiveserver2'."),
   help=_("Beeswax or Hive Server 2 Thrift API used. Choices are: 'beeswax' or 'hiveserver2'."),
-  default="beeswax")
+  default="beeswax",
+  private=True)
 
 
 
 
 BEESWAX_SERVER_HOST = Config(
 BEESWAX_SERVER_HOST = Config(

+ 3 - 0
apps/beeswax/src/beeswax/models.py

@@ -117,6 +117,9 @@ class QueryHistory(models.Model):
   def set_to_failed(self):
   def set_to_failed(self):
     self.last_state = QueryHistory.STATE.failed.index
     self.last_state = QueryHistory.STATE.failed.index
 
 
+  def set_to_available(self):
+    self.last_state = QueryHistory.STATE.available.index
+
 
 
 class HiveServerQueryHistory(QueryHistory):
 class HiveServerQueryHistory(QueryHistory):
   # Map from (thrift) server state
   # Map from (thrift) server state

+ 2 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -175,7 +175,7 @@ class Dbms:
     return None
     return None
 
 
 
 
-  def execute_and_watch(self, query, design=None, notify=False):
+  def execute_and_watch(self, query, design=None):
     """
     """
     Run query and return a QueryHistory object in order to see its progress on a Web page.
     Run query and return a QueryHistory object in order to see its progress on a Web page.
     """
     """
@@ -189,7 +189,7 @@ class Dbms:
                                 server_type=self.server_type,
                                 server_type=self.server_type,
                                 last_state=QueryHistory.STATE.submitted.index,
                                 last_state=QueryHistory.STATE.submitted.index,
                                 design=design,
                                 design=design,
-                                notify=notify)
+                                notify=query.query.get('email_notify', False))
     query_history.save()
     query_history.save()
 
 
     LOG.debug("Made new QueryHistory id %s user %s query: %s..." % (query_history.id, self.client.user, query_history.query[:25]))
     LOG.debug("Made new QueryHistory id %s user %s query: %s..." % (query_history.id, self.client.user, query_history.query[:25]))

+ 10 - 7
apps/beeswax/src/beeswax/templates/execute.mako

@@ -210,13 +210,16 @@ ${layout.menubar(section='query')}
                                 ${_("Enable Parameterization")}
                                 ${_("Enable Parameterization")}
                             </label>
                             </label>
                         </li>
                         </li>
-                        <ul
-                        % if app_name == 'impala':
-                            class="hide"
-                        % endif
-                        >
-                          <li class="nav-header">${_('Email Notification')}</li>
-                          <li>
+                          <li class="nav-header
+                            % if app_name == 'impala':
+                                hide
+                            % endif
+                          ">${_('Email Notification')}</li>
+                          <li
+	                        % if app_name == 'impala':
+	                            class="hide"
+	                        % endif
+                          >
                             <label class="checkbox" rel="tooltip" data-original-title="${_("If checked, you will receive an email notification when the query completes.")}">
                             <label class="checkbox" rel="tooltip" data-original-title="${_("If checked, you will receive an email notification when the query completes.")}">
                                 <input type="checkbox" id="id_${form.query["email_notify"].html_name | n}" name="${form.query["email_notify"].html_name | n}" ${extract_field_data(form.query["email_notify"]) and "CHECKED" or ""}/>
                                 <input type="checkbox" id="id_${form.query["email_notify"].html_name | n}" name="${form.query["email_notify"].html_name | n}" ${extract_field_data(form.query["email_notify"]) and "CHECKED" or ""}/>
                                 ${_("Email me on completion")}
                                 ${_("Email me on completion")}

+ 2 - 2
apps/beeswax/src/beeswax/test_base.py

@@ -16,7 +16,6 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 #
 #
-from django.core.urlresolvers import reverse
 
 
 """
 """
 Common infrastructure for beeswax tests
 Common infrastructure for beeswax tests
@@ -201,7 +200,7 @@ def wait_for_query_to_finish(client, response, max=30.0):
 def make_query(client, query, submission_type="Execute",
 def make_query(client, query, submission_type="Execute",
                udfs=None, settings=None, resources=None,
                udfs=None, settings=None, resources=None,
                wait=False, name=None, desc=None, local=True,
                wait=False, name=None, desc=None, local=True,
-               is_parameterized=True, max=30.0, database='default', **kwargs):
+               is_parameterized=True, max=30.0, database='default', email_notify=False, **kwargs):
   """
   """
   Prepares arguments for the execute view.
   Prepares arguments for the execute view.
 
 
@@ -219,6 +218,7 @@ def make_query(client, query, submission_type="Execute",
     'query-query': query,
     'query-query': query,
     'query-is_parameterized': is_parameterized and "on",
     'query-is_parameterized': is_parameterized and "on",
     'query-database': database,
     'query-database': database,
+    'query-email_notify': email_notify and "on",
   }
   }
 
 
   if submission_type == 'Execute':
   if submission_type == 'Execute':

+ 21 - 2
apps/beeswax/src/beeswax/tests.py

@@ -59,11 +59,11 @@ CSV_LINK_PAT = re.compile('/beeswax/download/\d+/csv')
 def _make_query(client, query, submission_type="Execute",
 def _make_query(client, query, submission_type="Execute",
                 udfs=None, settings=None, resources=[],
                 udfs=None, settings=None, resources=[],
                 wait=False, name=None, desc=None, local=True,
                 wait=False, name=None, desc=None, local=True,
-                is_parameterized=True, max=30.0, database='default', **kwargs):
+                is_parameterized=True, max=30.0, database='default', email_notify=False, **kwargs):
   """Wrapper around the real make_query"""
   """Wrapper around the real make_query"""
   res = make_query(client, query, submission_type,
   res = make_query(client, query, submission_type,
                    udfs, settings, resources,
                    udfs, settings, resources,
-                   wait, name, desc, local, is_parameterized, max, database, **kwargs)
+                   wait, name, desc, local, is_parameterized, max, database, email_notify, **kwargs)
 
 
   # Should be in the history if it's submitted.
   # Should be in the history if it's submitted.
   if submission_type == 'Execute':
   if submission_type == 'Execute':
@@ -504,6 +504,25 @@ for x in sys.stdin:
     csv_resp = download(handle, 'csv', self.db)
     csv_resp = download(handle, 'csv', self.db)
     assert_equal(len(csv_resp.content.strip().split('\n')), limit + 1)
     assert_equal(len(csv_resp.content.strip().split('\n')), limit + 1)
 
 
+  def test_query_done_cb(self):
+    hql = 'SELECT * FROM test'
+    query = hql_query(hql)
+    query._data_dict['query']['email_notify'] = False
+    query_history = self.db.execute_and_watch(query)
+
+    response = self.client.get('/beeswax/query_cb/done/%s' % query_history.server_id)
+    assert_true('email_notify is false' in response.content, response.content)
+
+    query = hql_query(hql)
+    query._data_dict['query']['email_notify'] = True
+    query_history = self.db.execute_and_watch(query)
+
+    response = self.client.get('/beeswax/query_cb/done/%s' % query_history.server_id,)
+    assert_true('sent' in response.content, response.content)
+
+    response = self.client.get('/beeswax/query_cb/done/blahblahblah')
+    assert_true('QueryHistory matching query does not exist' in response.content, response.content)
+
   def test_data_export(self):
   def test_data_export(self):
     hql = 'SELECT * FROM test'
     hql = 'SELECT * FROM test'
     query = hql_query(hql)
     query = hql_query(hql)

+ 29 - 25
apps/beeswax/src/beeswax/views.py

@@ -477,9 +477,8 @@ def execute_query(request, design_id=None):
           if to_explain:
           if to_explain:
             return explain_directly(request, query, design, query_server)
             return explain_directly(request, query, design, query_server)
           else:
           else:
-            notify = form.query.cleaned_data.get('email_notify', False)
             download = request.POST.has_key('download')
             download = request.POST.has_key('download')
-            return execute_directly(request, query, query_server, design, on_success_url=on_success_url, notify=notify, download=download)
+            return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
         except BeeswaxException, ex:
         except BeeswaxException, ex:
           print ex.errorCode
           print ex.errorCode
           print ex.SQLState
           print ex.SQLState
@@ -912,36 +911,41 @@ def query_done_cb(request, server_id):
   This view should always return a 200 response, to reflect that the
   This view should always return a 200 response, to reflect that the
   notification is delivered to the right view.
   notification is delivered to the right view.
   """
   """
-  res = HttpResponse('<html><head></head><body></body></html>')
+  message_template = '<html><head></head>%(message)s<body></body></html>'
+  message = {'message': 'error'}
 
 
-  history = models.QueryHistory.objects.get(server_id=server_id)
+  try:
+    query_history = models.QueryHistory.objects.get(server_id=server_id)
 
 
-  if history is None:
-    LOG.error('Processing query completion email: Cannot find query matching id %s' % (server_id,))
-    return res
+    # Update the query status
+    query_history.set_to_available()
 
 
-  # Update the query status
-  history.save_state(models.QueryHistory.STATE.available)
+    # Find out details about the query
+    if not query_history.notify:
+      message['message'] = 'email_notify is false'
+      return HttpResponse(message_template % message)
 
 
-  # Find out details about the query
-  if not history.notify:
-    return res
-  design = history.design
-  user = history.owner
-  subject = _("Beeswax query completed")
-  if design:
-    subject += ": %s" % (design.name,)
+    design = query_history.design
+    user = query_history.owner
+    subject = _("Beeswax query completed")
+
+    if design:
+      subject += ": %s" % (design.name,)
+
+    link = "%s/#launch=Beeswax:%s" % \
+              (get_desktop_uri_prefix(),
+               reverse(get_app_name(request) + ':watch_query', kwargs={'id': query_history.id}))
+    body = _("%(subject)s. You may see the results here: %(link)s\n\nQuery:\n%(query)s") % {
+               'subject': subject, 'link': link, 'query': query_history.query
+             }
 
 
-  link = "%s/#launch=Beeswax:%s" % \
-            (get_desktop_uri_prefix(),
-             reverse(get_app_name(request) + ':watch_query', kwargs={'id': history.id}))
-  body = _("%(subject)s. You may see the results here: %(link)s\n\nQuery:\n%(query)s") % {'subject': subject, 'link': link, 'query': history.query}
-  try:
     user.email_user(subject, body)
     user.email_user(subject, body)
+    message['message'] = 'sent'
   except Exception, ex:
   except Exception, ex:
-    LOG.error("Failed to send query completion notification via e-mail to %s: %s" %
-              (user.username, ex))
-  return res
+    msg = "Failed to send query completion notification via e-mail: %s" % (ex)
+    LOG.error(msg)
+    message['message'] = msg
+  return HttpResponse(message_template % message)
 
 
 
 
 
 

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -922,7 +922,7 @@ from django.utils.translation import ugettext as _
                     action: action,
                     action: action,
                     template: '<div class="qq-uploader">' +
                     template: '<div class="qq-uploader">' +
                             '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
                             '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
-                            '<div class="qq-upload-button">${_('Upload a file')}</div>' +
+                            '<div class="qq-upload-button">${_('Select files')}</div>' +
                             '<ul class="qq-upload-list"></ul>' +
                             '<ul class="qq-upload-list"></ul>' +
                             '</div>',
                             '</div>',
                     fileTemplate: '<li>' +
                     fileTemplate: '<li>' +

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/jobs.mako

@@ -136,7 +136,7 @@ ${ commonheader(_('Job Browser'), "jobbrowser", user) | n,unicode }
                 % endif
                 % endif
             </td>
             </td>
             <td data-sort-value="${job.startTimeMs}">${job.startTimeFormatted}</td>
             <td data-sort-value="${job.startTimeMs}">${job.startTimeFormatted}</td>
-            <td>
+            <td data-row-selector-exclude="true" style="padding-right: 60px">
                 % if (job.status.lower() == 'running' or job.status.lower() == 'pending') and not job.is_mr2:
                 % if (job.status.lower() == 'running' or job.status.lower() == 'pending') and not job.is_mr2:
                   % if request.user.is_superuser or request.user.username == job.user:
                   % if request.user.is_superuser or request.user.username == job.user:
                     <a href="#" title="${_('Kill this job')}" kill-action="${url('jobbrowser.views.kill_job', job=job.jobId)}?next=${request.get_full_path() | urlencode}"
                     <a href="#" title="${_('Kill this job')}" kill-action="${url('jobbrowser.views.kill_job', job=job.jobId)}?next=${request.get_full_path() | urlencode}"

+ 2 - 2
apps/oozie/src/oozie/models.py

@@ -1443,11 +1443,11 @@ class History(models.Model):
 
 
   def get_workflow(self):
   def get_workflow(self):
     if self.oozie_job_id.endswith('W'):
     if self.oozie_job_id.endswith('W'):
-      return self.job
+      return self.job.get_full_node()
 
 
   def get_coordinator(self):
   def get_coordinator(self):
     if self.oozie_job_id.endswith('C'):
     if self.oozie_job_id.endswith('C'):
-      return self.job
+      return self.job.get_full_node()
 
 
   @classmethod
   @classmethod
   def get_workflow_from_config(self, conf_dict):
   def get_workflow_from_config(self, conf_dict):

+ 4 - 1
apps/oozie/src/oozie/templates/editor/create_coordinator.mako

@@ -215,9 +215,12 @@ ${ layout.menubar(section='coordinators') }
     });
     });
 
 
     $("#id_workflow").change(function () {
     $("#id_workflow").change(function () {
-      $("#workflowName").text($("#id_workflow option[value='" + $(this).val() + "']").text());
+      if ($(this).val()) {
+        $("#workflowName").text($("#id_workflow option[value='" + $(this).val() + "']").text());
+      }
     });
     });
 
 
+    $("#id_workflow").change();
   });
   });
 </script>
 </script>
 
 

+ 1 - 1
desktop/libs/liboozie/src/liboozie/types.py

@@ -353,7 +353,7 @@ class Coordinator(Job):
     end = mktime(self.endTime)
     end = mktime(self.endTime)
 
 
     if end != start:
     if end != start:
-      return int((1 - (end - next) / (end - start)) * 100)
+      return min(int((1 - (end - next) / (end - start)) * 100), 100)
     else:
     else:
       return 100
       return 100