Browse Source

[jb] Add status filter to the tasks listing of an Oozie schedule in the Job Browser

The implementation of this was left todo many years ago, back when we made the switch from the dedicated Oozie status pages to the Job Browser in Hue 4.

Given that the Job Browser pages are generic in the sense that the same code is used to display many types of "jobs" I've deliberately kept this change as non-intrusive as possible. Existing patterns, variables and values have been re-used even if there's quite some room for improvement and the filter will only appear for the tasks listing in the schedules tab.
Johan Åhlén 2 years ago
parent
commit
d193a45a10

+ 4 - 1
apps/jobbrowser/src/jobbrowser/api2.py

@@ -94,8 +94,11 @@ def job(request, interface=None):
     app_id = json.loads(request.POST.get('app_id'))
     app_id = json.loads(request.POST.get('app_id'))
 
 
   if interface == 'schedules':
   if interface == 'schedules':
+    filters = dict([(key, value) for _filter in json.loads(
+        request.POST.get('filters', '[]')) for key, value in list(_filter.items()) if value
+    ])
     offset = json.loads(request.POST.get('pagination', '{"offset": 1}')).get('offset')
     offset = json.loads(request.POST.get('pagination', '{"offset": 1}')).get('offset')
-    response_app = get_api(request.user, interface, cluster=cluster).app(app_id, offset=offset)
+    response_app = get_api(request.user, interface, cluster=cluster).app(app_id, offset=offset, filters=filters)
   else:
   else:
     response_app = get_api(request.user, interface, cluster=cluster).app(app_id)
     response_app = get_api(request.user, interface, cluster=cluster).app(app_id)
 
 

+ 45 - 23
apps/jobbrowser/src/jobbrowser/apis/schedule_api.py

@@ -55,7 +55,7 @@ class ScheduleApi(Api):
     jobs = oozie_api.get_coordinators(**kwargs)
     jobs = oozie_api.get_coordinators(**kwargs)
 
 
     return {
     return {
-      'apps':[{
+      'apps': [{
         'id': app['id'],
         'id': app['id'],
         'name': app['appName'],
         'name': app['appName'],
         'status': app['status'],
         'status': app['status'],
@@ -72,12 +72,34 @@ class ScheduleApi(Api):
     }
     }
 
 
 
 
-  def app(self, appid, offset=1):
+  def app(self, appid, offset=1, filters={}):
     oozie_api = get_oozie(self.user)
     oozie_api = get_oozie(self.user)
     coordinator = oozie_api.get_coordinator(jobid=appid)
     coordinator = oozie_api.get_coordinator(jobid=appid)
 
 
     mock_get = MockGet()
     mock_get = MockGet()
     mock_get.update('offset', offset)
     mock_get.update('offset', offset)
+
+    """ 
+      The Oozie job api supports one or more "status" parameters. The valid status values are:
+      
+      WAITING, READY, SUBMITTED, RUNNING, SUSPENDED, TIMEDOUT, SUCCEEDED, KILLED, FAILED, IGNORED, SKIPPED
+      
+      The job browser UI has a generic filter mechanism that is re-used across all different type of jobs, that
+      parameter is called "states" and it only has three possible values: completed, running or failed
+      
+      Here we adapt this to fit the API requirements, "state" becomes "status" and the values are translated
+      based on how it's been done historically (for instance list_oozie_coordinator.mako around line 725).
+    """
+    if 'states' in filters:
+      statusFilters = []
+      for stateFilter in filters.get('states'):
+        if stateFilter == 'completed':
+          statusFilters.append('SUCCEEDED')
+        elif stateFilter == 'running':
+          statusFilters.extend(['RUNNING', 'READY', 'SUBMITTED', 'SUSPENDED', 'WAITING'])
+        elif stateFilter == 'failed':
+          statusFilters.extend(['KILLED', 'FAILED', 'TIMEDOUT', 'SKIPPED'])
+      mock_get.update('status', statusFilters)
     request = MockDjangoRequest(self.user, get=mock_get)
     request = MockDjangoRequest(self.user, get=mock_get)
     response = list_oozie_coordinator(request, job_id=appid)
     response = list_oozie_coordinator(request, job_id=appid)
 
 
@@ -133,36 +155,36 @@ class ScheduleApi(Api):
       return coordinator['properties']['tasks']
       return coordinator['properties']['tasks']
 
 
   _API_STATUSES = {
   _API_STATUSES = {
-    'PREP':               'RUNNING',
-    'RUNNING':            'RUNNING',
-    'RUNNINGWITHERROR':   'RUNNING',
-    'PREPSUSPENDED':      'PAUSED',
-    'SUSPENDED':          'PAUSED',
+    'PREP': 'RUNNING',
+    'RUNNING': 'RUNNING',
+    'RUNNINGWITHERROR': 'RUNNING',
+    'PREPSUSPENDED': 'PAUSED',
+    'SUSPENDED': 'PAUSED',
     'SUSPENDEDWITHERROR': 'PAUSED',
     'SUSPENDEDWITHERROR': 'PAUSED',
-    'PREPPAUSED':         'PAUSED',
-    'PAUSED':             'PAUSED',
-    'PAUSEDWITHERROR':    'PAUSED',
-    'SUCCEEDED':          'SUCCEEDED',
-    'DONEWITHERROR':      'FAILED',
-    'KILLED':             'FAILED',
-    'FAILED':             'FAILED',
+    'PREPPAUSED': 'PAUSED',
+    'PAUSED': 'PAUSED',
+    'PAUSEDWITHERROR': 'PAUSED',
+    'SUCCEEDED': 'SUCCEEDED',
+    'DONEWITHERROR': 'FAILED',
+    'KILLED': 'FAILED',
+    'FAILED': 'FAILED',
   }
   }
 
 
   def _api_status(self, status):
   def _api_status(self, status):
     return self._API_STATUSES.get(status, 'FAILED')
     return self._API_STATUSES.get(status, 'FAILED')
 
 
   _TASK_API_STATUSES = {
   _TASK_API_STATUSES = {
-    'WAITING':   'RUNNING',
-    'READY':     'RUNNING',
+    'WAITING': 'RUNNING',
+    'READY': 'RUNNING',
     'SUBMITTED': 'RUNNING',
     'SUBMITTED': 'RUNNING',
-    'RUNNING':   'RUNNING',
+    'RUNNING': 'RUNNING',
     'SUSPENDED': 'PAUSED',
     'SUSPENDED': 'PAUSED',
     'SUCCEEDED': 'SUCCEEDED',
     'SUCCEEDED': 'SUCCEEDED',
-    'TIMEDOUT':  'FAILED',
-    'KILLED':    'FAILED',
-    'FAILED':    'FAILED',
-    'IGNORED':   'FAILED',
-    'SKIPPED':   'FAILED',
+    'TIMEDOUT': 'FAILED',
+    'KILLED': 'FAILED',
+    'FAILED': 'FAILED',
+    'IGNORED': 'FAILED',
+    'SKIPPED': 'FAILED',
   }
   }
 
 
   def _task_api_status(self, status):
   def _task_api_status(self, status):
@@ -192,4 +214,4 @@ class MockGet(object):
       return self._prop.get(prop, default)
       return self._prop.get(prop, default)
 
 
   def getlist(self, prop):
   def getlist(self, prop):
-    return []
+    return self._prop.get(prop)

+ 25 - 12
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2444,13 +2444,17 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <!-- ko with: coordinatorActions() -->
           <!-- ko with: coordinatorActions() -->
           <form class="form-inline">
           <form class="form-inline">
             ##<input data-bind="value: textFilter" type="text" class="input-xlarge search-query" placeholder="${_('Filter by name')}">
             ##<input data-bind="value: textFilter" type="text" class="input-xlarge search-query" placeholder="${_('Filter by name')}">
-
-            ##<span data-bind="foreach: statesValuesFilter">
-            ##  <label class="checkbox">
-            ##    <div class="pull-left margin-left-5 status-border status-content" data-bind="css: value, hueCheckbox: checked"></div>
-            ##    <div class="inline-block" data-bind="text: name, toggle: checked"></div>
-            ##  </label>
-            ##</span>
+            <!-- ko with: $root.job() -->
+            <!-- ko if: type() === 'schedule' -->
+            <span data-bind="foreach: statesValuesFilter">
+              <label class="checkbox">
+                <div class="pull-left margin-left-5 status-border status-content" data-bind="css: value, hueCheckbox: checked"></div>
+                <div class="inline-block" data-bind="text: name, toggle: checked"></div>
+              </label>
+            </span>
+            <!-- ko hueSpinner: { spin: applyingFilters, inline: true } --><!-- /ko -->
+            <!-- /ko -->
+            <!-- /ko -->
             <div data-bind="template: { name: 'job-actions${ SUFFIX }' }" class="pull-right"></div>
             <div data-bind="template: { name: 'job-actions${ SUFFIX }' }" class="pull-right"></div>
           </form>
           </form>
 
 
@@ -2963,8 +2967,16 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           {'types': ko.mapping.toJS(self.typesFilter())},
           {'types': ko.mapping.toJS(self.typesFilter())},
         ];
         ];
       });
       });
-      self.filters.subscribe(function(value) {
-        self.fetchProfile('tasks');
+      self.applyingFilters = ko.observable(false);
+      self.filters.subscribe(function () {
+        if (self.type() === 'schedule') {
+          self.applyingFilters(true);
+          self.updateJob(false, true).always(function () {
+            self.applyingFilters(false);
+          });
+        } else {
+          self.fetchProfile('tasks');
+        }
       });
       });
       self.metadataFilter = ko.observable('');
       self.metadataFilter = ko.observable('');
       self.metadataFilter.subscribe(function(newValue) {
       self.metadataFilter.subscribe(function(newValue) {
@@ -3057,7 +3069,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           cluster: ko.mapping.toJSON(vm.compute),
           cluster: ko.mapping.toJSON(vm.compute),
           app_id: ko.mapping.toJSON(self.id),
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           interface: ko.mapping.toJSON(vm.interface),
-          pagination: ko.mapping.toJSON(self.pagination)
+          pagination: ko.mapping.toJSON(self.pagination),
+          filters: ko.mapping.toJSON(self.filters)
         }, function (data) {
         }, function (data) {
           if (data.status == 0) {
           if (data.status == 0) {
             if (data.app) {
             if (data.app) {
@@ -3250,10 +3263,10 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         });
         });
       };
       };
 
 
-      self.updateJob = function (updateLogs) {
+      self.updateJob = function (updateLogs, forceUpdate) {
         huePubSub.publish('graph.refresh.view');
         huePubSub.publish('graph.refresh.view');
         var deferred = $.Deferred();
         var deferred = $.Deferred();
-        if (vm.job() == self && self.apiStatus() == 'RUNNING') {
+        if (vm.job() == self && (self.apiStatus() == 'RUNNING' || forceUpdate)) {
           vm.apiHelper.cancelActiveRequest(lastUpdateJobRequest);
           vm.apiHelper.cancelActiveRequest(lastUpdateJobRequest);
           lastUpdateJobRequest = self._fetchJob(function (data) {
           lastUpdateJobRequest = self._fetchJob(function (data) {
             var requests = [];
             var requests = [];