Эх сурвалжийг харах

HUE-1887 [oozie] Support dashboard Coordinator and Workflow action SLA

Pass #1
SLA main page:
- add one more day in the end date
- sort list by nominal time
- combine id and parent_id
- make date filtering easier to disable
- make SLA properties parameterizable

All:
Support Coordinator SLA, Workflow action SLA
Romain Rigaux 12 жил өмнө
parent
commit
bd76ca188f

+ 25 - 1
apps/oozie/src/oozie/models.py

@@ -126,7 +126,7 @@ class Job(models.Model):
   is_trashed = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is trashed'), blank=True, # Deprecated
                                    help_text=_t('If this job is trashed.'))
   doc = generic.GenericRelation(Document, related_name='oozie_doc')
-  data = models.TextField(blank=True, default=json.dumps({}))  # e.g. data=json.dump({'sla': [python data], ...})
+  data = models.TextField(blank=True, default=json.dumps({}))  # e.g. data=json.dumps({'sla': [python data], ...})
 
   objects = JobManager()
   unique_together = ('owner', 'name')
@@ -433,6 +433,10 @@ class Workflow(Job):
   def find_parameters(self):
     params = set()
 
+    if self.sla_enabled:
+      for param in find_json_parameters(self.sla):
+        params.add(param)
+
     for node in self.node_list:
       if hasattr(node, 'find_parameters'):
         params.update(node.find_parameters())
@@ -1523,6 +1527,10 @@ class Coordinator(Job):
     for param in find_parameters(self, ['job_properties']):
       params[param] = ''
 
+    if self.sla_enabled:
+      for param in find_json_parameters(self.sla):
+        params.add(param)
+
     for dataset in self.dataset_set.all():
       for param in find_parameters(dataset, ['uri']):
         if param not in set(DATASET_FREQUENCY):
@@ -1882,6 +1890,20 @@ class History(models.Model):
 
     return history
 
+
+def get_link(oozie_id):
+  link = ''
+
+  if 'W@' in oozie_id:
+    link = reverse('oozie:list_oozie_workflow_action', kwargs={'action': oozie_id})
+  elif oozie_id.endswith('W'):
+    link = reverse('oozie:list_oozie_workflow', kwargs={'job_id': oozie_id})
+  elif oozie_id.endswith('C'):
+    link = reverse('oozie:list_oozie_coordinator', kwargs={'job_id': oozie_id})
+
+  return link
+
+
 def find_parameters(instance, fields=None):
   """Find parameters in the given fields"""
   if fields is None:
@@ -1890,6 +1912,8 @@ def find_parameters(instance, fields=None):
   params = []
   for field in fields:
     data = getattr(instance, field)
+    if field == 'sla' and not instance.sla_enabled:
+      continue
     if isinstance(data, list):
       params.extend(find_json_parameters(data))
     elif isinstance(data, basestring):

+ 38 - 15
apps/oozie/src/oozie/templates/dashboard/list_oozie_sla.mako

@@ -58,23 +58,27 @@ ${ layout.menubar(section='sla', dashboard=True) }
 <div class="container-fluid">
   <div class="card card-small">
     <h1 class="card-heading simple">
-    <div class="pull-left" style="margin-right: 20px;margin-top: 2px">${_('Search SLA')}</div>
+    <div class="pull-left" style="margin-right: 20px;margin-top: 2px">${_('Search')}</div>
     <form class="form-inline" id="searchForm" method="GET" action="." style="margin-bottom: 4px">
       <label>
         ${_('Name or Id')}
         <input type="text" name="job_name" class="searchFilter input-xlarge search-query" placeholder="${_('Job Name or Id (required)')}">
       </label>
+      <span style="padding-left:25px">
+        <label class="label-with-margin">
+          ${ _('Start') }
+          <input type="text" name="start_0" class="input-small date" value="" placeholder="${_('Date in GMT')}"  data-bind="enable: useDates">
+          <input type="text" name="start_1" class="input-small time" value="" data-bind="enable: useDates">
+        </label>
+        <label>
+          ${ _('End') }
+          <input type="text" name="end_0" class="input-small date" value="" placeholder="${_('Date in GMT')}" data-bind="enable: useDates">
+          <input type="text" name="end_1" class="input-small time" value="" data-bind="enable: useDates">
+        </label>
+      </span>
       <label class="checkbox label-with-margin">
-        <input type="checkbox" name="isParent" class="searchFilter">
-        ${ _('This is the parent ID') }
-      </label>
-      <label class="label-with-margin">
-        ${ _('Start') }
-        <input type="text" name="start_0" class="input-small date" value="" placeholder="${_('Date in GMT')}"><input type="text" name="start_1" class="input-small time" value="">
-      </label>
-      <label>
-        ${ _('End') }
-        <input type="text" name="end_0" class="input-small date" value="" placeholder="${_('Date in GMT')}"><input type="text" name="end_1" class="input-small time" value="">
+        <input type="checkbox" name="useDates" class="searchFilter" data-bind="checked: useDates, click: performSearch()">
+        ${ _('Date filter') }
       </label>
     </form>
     </h1>
@@ -146,14 +150,19 @@ ${ layout.menubar(section='sla', dashboard=True) }
     </div>
   </div>
 </div>
+
 <script src="/oozie/static/js/bundles.utils.js" type="text/javascript" charset="utf-8"></script>
 <script src="/oozie/static/js/sla.utils.js" type="text/javascript" charset="utf-8"></script>
+
+<script src="/static/ext/js/knockout-min.js" type="text/javascript" charset="utf-8"></script>
+
 <script src="/static/ext/js/jquery/plugins/jquery.flot.min.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/jquery/plugins/jquery.flot.selection.min.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/jquery/plugins/jquery.flot.time.min.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/js/jquery.blueprint.js"></script>
 
 <script type="text/javascript" charset="utf-8">
+
   function performSearch(id) {
     if ((id != null || $("input[name='job_name']").val().trim()) != "" && slaTable) {
       window.location.hash = (id != null ? id : $("input[name='job_name']").val().trim());
@@ -166,8 +175,8 @@ ${ layout.menubar(section='sla', dashboard=True) }
         "job_name": id != null ? id : $("input[name='job_name']").val()
       };
 
-      if ($("input[name='isParent']").is("checked")) {
-        _postObj.isParent = true;
+      if (window.viewModel.useDates()) {
+        _postObj.useDates = true;
       }
       if ($("input[name='start_0']").val() != "" && $("input[name='start_1']").val() != "") {
         _postObj.start = moment($("input[name='start_0']").val() + " " + $("input[name='start_1']").val(), IN_DATETIME_FORMAT).format(OUT_DATETIME_FORMAT);
@@ -204,15 +213,26 @@ ${ layout.menubar(section='sla', dashboard=True) }
   }
 
   var slaTable;
+
   $(document).ready(function () {
+    var ViewModel = function () {
+      var self = this;
+
+      self.useDates = ko.observable(false);
+    };
+
+    window.viewModel = new ViewModel([]);
+    ko.applyBindings(window.viewModel);
+
+
     $("a[data-row-selector='true']").jHueRowSelector();
 
     $("*[rel=tooltip]").tooltip();
 
     $("input[name='start_0']").val(moment().subtract('days', 7).format("MM/DD/YYYY"));
     $("input[name='start_1']").val(moment().subtract('days', 7).format("hh:mm A"));
-    $("input[name='end_0']").val(moment().format("MM/DD/YYYY"));
-    $("input[name='end_1']").val(moment().format("hh:mm A"));
+    $("input[name='end_0']").val(moment().add('days', 1).format("MM/DD/YYYY"));
+    $("input[name='end_1']").val(moment().add('days', 1).format("hh:mm A"));
 
 
     $.getJSON("${url('oozie:list_oozie_workflows')}?format=json&justsla=true", function (data) {
@@ -244,6 +264,9 @@ ${ layout.menubar(section='sla', dashboard=True) }
         "sEmptyTable": "${_('No data available')}",
         "sZeroRecords": "${_('No matching records')}"
       },
+      "aaSorting":[
+        [4, "desc"]
+      ],
       "fnDrawCallback": function (oSettings) {
         $("a[data-row-selector='true']").jHueRowSelector();
       }

+ 18 - 3
apps/oozie/src/oozie/tests.py

@@ -774,18 +774,33 @@ class TestEditor(OozieMockBase):
 
 
   def test_find_parameters(self):
+    data = json.dumps({'sla': [
+        {'key': 'enabled', 'value': True},
+        {'key': 'nominal-time', 'value': '${time}'},]}
+    )
     jobs = [Job(name="$a"),
             Job(name="foo ${b} $$"),
-            Job(name="${foo}", description="xxx ${foo}")]
+            Job(name="${foo}", description="xxx ${food}", data=data)]
 
-    result = [find_parameters(job, ['name', 'description']) for job in jobs]
-    assert_equal(set(["b", "foo"]), reduce(lambda x, y: x | set(y), result, set()))
+    result = [find_parameters(job, ['name', 'description', 'sla']) for job in jobs]
+    assert_equal(set(["b", "foo", "food", "time"]), reduce(lambda x, y: x | set(y), result, set()))
 
 
   def test_find_all_parameters(self):
+    self.wf.data = json.dumps({'sla': [
+        {'key': 'enabled', 'value': False},
+        {'key': 'nominal-time', 'value': '${time}'},]}
+    )
     assert_equal([{'name': u'output', 'value': u''}, {'name': u'SLEEP', 'value': ''}, {'name': u'market', 'value': u'US'}],
                  self.wf.find_all_parameters())
 
+    self.wf.data = json.dumps({'sla': [
+        {'key': 'enabled', 'value': True},
+        {'key': 'nominal-time', 'value': '${time}'},]}
+    )
+    assert_equal([{'name': u'time', 'value': u''}, {'name': u'output', 'value': u''}, {'name': u'SLEEP', 'value': ''}, {'name': u'market', 'value': u'US'}],
+                 self.wf.find_all_parameters())
+
 
   def test_workflow_has_cycle(self):
     action1 = Node.objects.get(workflow=self.wf, name='action-name-1')

+ 20 - 18
apps/oozie/src/oozie/views/dashboard.py

@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import time
 
 from django.forms.formsets import formset_factory
@@ -41,7 +42,7 @@ from oozie.conf import OOZIE_JOBS_COUNT
 from oozie.forms import RerunForm, ParameterForm, RerunCoordForm,\
   RerunBundleForm
 from oozie.models import History, Job, Workflow, utc_datetime_format, Bundle,\
-  Coordinator
+  Coordinator, get_link
 from oozie.settings import DJANGO_APPS
 
 
@@ -212,7 +213,8 @@ def list_oozie_workflow(request, job_id, coordinator_job_id=None, bundle_job_id=
   if oozie_workflow.has_sla:
     api = get_oozie(request.user, api_version="v2")
     params = {
-      'id': oozie_workflow.id
+      'id': oozie_workflow.id,
+      'parent_id': oozie_workflow.id
     }
     oozie_slas = api.get_oozie_slas(**params)
 
@@ -268,7 +270,8 @@ def list_oozie_coordinator(request, job_id, bundle_job_id=None):
   if oozie_coordinator.has_sla:
     api = get_oozie(request.user, api_version="v2")
     params = {
-      'id': oozie_coordinator.id
+      'id': oozie_coordinator.id,
+      'parent_id': oozie_coordinator.id
     }
     oozie_slas = api.get_oozie_slas(**params)
 
@@ -318,8 +321,7 @@ def list_oozie_workflow_action(request, action, coordinator_job_id=None, bundle_
     action = get_oozie(request.user).get_action(action)
     workflow = check_job_access_permission(request, action.id.split('@')[0])
   except RestException, ex:
-    raise PopupException(_("Error accessing Oozie action %s.") % (action,),
-                         detail=ex.message)
+    raise PopupException(_("Error accessing Oozie action %s.") % (action,), detail=ex.message)
 
   oozie_coordinator = None
   if coordinator_job_id is not None:
@@ -360,22 +362,21 @@ def list_oozie_sla(request):
   api = get_oozie(request.user, api_version="v2")
 
   if request.method == 'POST':
-    # filter=nominal_start=2013-06-18T00:01Z;nominal_end=2013-06-23T00:01Z;app_name=my-sla-app
     params = {}
 
     job_name = request.POST.get('job_name')
-    if job_name.endswith('-oozie-oozi-W'):
-      if 'isParent' in request.POST:
-        params['parent_id'] = job_name
-      else:
-        params['id'] = job_name
+
+    if re.match('.*-oozie-oozi-[WCB]', job_name):
+      params['id'] = job_name
+      params['parent_id'] = job_name
     else:
       params['app_name'] = job_name
 
-    if request.POST.get('start'):
-      params['nominal_start'] = request.POST.get('start')
-    if request.POST.get('end'):
-      params['nominal_end'] = request.POST.get('end')
+    if 'useDates' in request.POST:
+      if request.POST.get('start'):
+        params['nominal_start'] = request.POST.get('start')
+      if request.POST.get('end'):
+        params['nominal_end'] = request.POST.get('end')
 
     oozie_slas = api.get_oozie_slas(**params)
 
@@ -385,7 +386,7 @@ def list_oozie_sla(request):
   if request.REQUEST.get('format') == 'json':
     massaged_slas = []
     for sla in oozie_slas:
-      massaged_slas.append(massaged_sla_for_json(sla))
+      massaged_slas.append(massaged_sla_for_json(sla, request))
 
     return HttpResponse(json.dumps({'oozie_slas': massaged_slas}), content_type="text/json")
 
@@ -393,13 +394,14 @@ def list_oozie_sla(request):
     'oozie_slas': oozie_slas
   })
 
-def massaged_sla_for_json(sla):
+
+def massaged_sla_for_json(sla, request):
   massaged_sla = {
     'slaStatus': sla['slaStatus'],
     'id': sla['id'],
     'appType': sla['appType'],
     'appName': sla['appName'],
-    'appUrl': reverse(sla['appType'] == 'WORKFLOW_JOB' and 'oozie:list_oozie_workflow' or 'oozie:list_oozie_coordinator', kwargs={'job_id': sla['id']}),
+    'appUrl': get_link(sla['id']),
     'user': sla['user'],
     'nominalTime': sla['nominalTime'],
     'expectedStart': sla['expectedStart'],