Browse Source

[oozie] Add progress information on dashboard

For Worfklows and Coordinator and on their individual pages
Romain Rigaux 13 years ago
parent
commit
551b162

+ 8 - 8
apps/oozie/src/oozie/migrations/0001_initial.py

@@ -5,9 +5,9 @@ from south.v2 import SchemaMigration
 from django.db import models
 
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
-        
+
         # Adding model 'Job'
         db.create_table('oozie_job', (
             ('is_shared', self.gf('django.db.models.fields.BooleanField')(default=False, db_index=True, blank=True)),
@@ -194,10 +194,10 @@ class Migration(SchemaMigration):
             ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
         ))
         db.send_create_signal('oozie', ['History'])
-    
-    
+
+
     def backwards(self, orm):
-        
+
         # Deleting model 'Job'
         db.delete_table('oozie_job')
 
@@ -251,8 +251,8 @@ class Migration(SchemaMigration):
 
         # Deleting model 'History'
         db.delete_table('oozie_history')
-    
-    
+
+
     models = {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
@@ -443,5 +443,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
     }
-    
+
     complete_apps = ['oozie']

+ 1 - 1
apps/oozie/src/oozie/models.py

@@ -488,7 +488,7 @@ class Workflow(Job):
       else:
         flat.append(nodes)
       return flat
-    
+
     def from_iterable(iterables):
       # Python 2.6 chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
       for it in iterables:

+ 74 - 58
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinator.mako

@@ -31,73 +31,89 @@ ${ layout.menubar(section='dashboard') }
 
   <h1>${ _('Coordinator') } ${ oozie_coordinator.appName }</h1>
 
-  <div>
     <div class="tab-pane" id="details">
-      <table class="table table-condensed" cellpadding="0" cellspacing="0">
-        <tbody>
-          <tr>
-            <td>${ _('Coordinator') }</td>
-            <td>
+      <div class="container-fluid">
+        <div class="row-fluid">
+          <div class="span3">${ _('Coordinator') }</div>
+          <div class="span6">
               % if coordinator is not None:
                 <a href="${ coordinator.get_absolute_url() }">${ oozie_coordinator.appName }</a>
               % else:
                 ${ oozie_coordinator.appName }
               % endif
-            </td>
-          </tr>
-          <tr>
-            <td>${ _('Submitter') }</td>
-            <td>${ oozie_coordinator.user }</td>
-          </tr>
-          <tr>
-            <td>${ _('Frequency') }</td>
-            <td>${ oozie_coordinator.frequency } ${ oozie_coordinator.timeUnit }</td>
-          </tr>
-          <tr>
-            <td>${ _('Status') }</td>
-            <td><span class="label ${ utils.get_status(oozie_coordinator.status) }">${ oozie_coordinator.status }</span>&nbsp;</td>
-          </tr>
-          <tr>
-            <td>${ _('Next Materialized Time') }</td>
-            <td>${ utils.format_time(oozie_coordinator.nextMaterializedTime) }</td>
-          </tr>
-          % if coordinator:
-          <tr>
-            <td>${ _('Datasets') }</td>
-            <td></td>
-          </tr>
+          </div>
+        </div>
+
+        <div class="row-fluid">
+          <div class="span3">${ _('Submitter') }</div>
+          <div class="span6">${ oozie_coordinator.user }</div>
+        </div>
+
+        <div class="row-fluid">
+          <div class="span3">${ _('Status') }</div>
+          <div class="span6"><span class="label ${ utils.get_status(oozie_coordinator.status) }">${ oozie_coordinator.status }</span>&nbsp;</div>
+        </div>
+
+        <div class="row-fluid">
+          <div class="span3">
+            ${ _('Progress') }
+          </div>
+          <div class="span3">
+            ${ oozie_coordinator.get_progress() }%
+          </div>
+        </div>
+
+        <div class="row-fluid">
+          <div class="span3">${ _('Frequency') }</div>
+          <div class="span3">${ oozie_coordinator.frequency } ${ oozie_coordinator.timeUnit }</div>
+          <div class="span3">${ _('Next Materialized Time') }</div>
+          <div class="span3">${ utils.format_time(oozie_coordinator.nextMaterializedTime) }</div>
+        </div>
+
+
+        <div class="row-fluid">
+          <div class="span3">${ _('Start time') }</div><div class="span3">${ utils.format_time(oozie_coordinator.startTime) }</div>
+          <div class="span3">${ _('End time') }</div><div class="span3">${ utils.format_time(oozie_coordinator.endTime) }</div>
+        </div>
+
+        % if coordinator:
+          <div class="row-fluid">
+            <div class="row-fluid">
+              <div class="span3">${ _('Datasets') }</div>
+            </div>
             % for dataset in coordinator.dataset_set.all():
-              <tr>
-                <td></td>
-                <td>${ dataset.name } : ${ dataset.uri }</td>
-              </tr>
+              <div class="row-fluid">
+                <div class="span3"></div>
+                <div class="span6">${ dataset.name } : ${ dataset.uri }</div>
+              </div>
             % endfor
+          </div>
           % endif
-          <tr>
-            <td>${ _('Manage') }</td>
-            <td>
-            <form action="${ url('oozie:resubmit_coordinator', oozie_coord_id=oozie_coordinator.id) }" method="post">
-            % if oozie_coordinator.is_running():
-              <a title="${_('Kill %(coordinator)s') % dict(coordinator=oozie_coordinator.id)}"
-                id="kill-coordinator"
-                class="btn small confirmationModal"
-                alt="${ _('Are you sure you want to kill coordinator %s?') % oozie_coordinator.id }"
-                href="javascript:void(0)"
-                data-url="${ url('oozie:manage_oozie_jobs', job_id=oozie_coordinator.id, action='kill') }"
-                data-message="${ _('The coordinator was killed!') }"
-                data-confirmation-message="${ _('Are you sure you\'d like to kill this job?') }">
-                  ${_('Kill')}
-              </a>
-            % else:
-              <button type="submit" class="btn">
-                ${ _('Resubmit') }
-              </button>
-            % endif
-            </form>
-            </td>
-          </tr>
-        </tbody>
-      </table>
+
+          <div class="row-fluid">
+            <div class="span3">${ _('Manage') }</div>
+            <div class="span6">
+              <form action="${ url('oozie:resubmit_coordinator', oozie_coord_id=oozie_coordinator.id) }" method="post">
+              % if oozie_coordinator.is_running():
+                <a title="${_('Kill %(coordinator)s') % dict(coordinator=oozie_coordinator.id)}"
+                  id="kill-coordinator"
+                  class="btn small confirmationModal"
+                  alt="${ _('Are you sure you want to kill coordinator %s?') % oozie_coordinator.id }"
+                  href="javascript:void(0)"
+                  data-url="${ url('oozie:manage_oozie_jobs', job_id=oozie_coordinator.id, action='kill') }"
+                  data-message="${ _('The coordinator was killed!') }"
+                  data-confirmation-message="${ _('Are you sure you\'d like to kill this job?') }">
+                    ${_('Kill')}
+                </a>
+              % else:
+                <button type="submit" class="btn">
+                  ${ _('Resubmit') }
+                </button>
+              % endif
+              </form>
+            </div>
+          </div>
+      </div>
     </div>
 
     <ul class="nav nav-tabs">

+ 8 - 8
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako

@@ -27,15 +27,15 @@ ${ layout.menubar(section='dashboard') }
 
 
 <div class="container-fluid">
-    ${ layout.dashboard_sub_menubar(section='workflows') }
+  ${ layout.dashboard_sub_menubar(section='workflows') }
 
-    <h1>
-      % if oozie_coordinator:
-        ${ _('Coordinator') } <a href="${ oozie_coordinator.get_absolute_url() }">${ oozie_coordinator.appName }</a> :
-      % endif
+  <h1>
+    % if oozie_coordinator:
+      ${ _('Coordinator') } <a href="${ oozie_coordinator.get_absolute_url() }">${ oozie_coordinator.appName }</a> :
+    % endif
 
-      ${ _('Workflow') } ${ oozie_workflow.appName }
-    </h1>
+    ${ _('Workflow') } ${ oozie_workflow.appName }
+  </h1>
 
   <div class="row-fluid">
     <div class="span3">
@@ -96,7 +96,7 @@ ${ layout.menubar(section='dashboard') }
     </div>
     % for var, value in parameters.iteritems():
       <div class="row-fluid">
-      <div class="span3"></div>
+        <div class="span3"></div>
         <div class="span3">
           ${ var | h }
         </div>

+ 1 - 1
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflows.mako

@@ -90,7 +90,7 @@ ${ layout.menubar(section='dashboard') }
                 data-confirmation-message="${ _('Are you sure you\'d like to kill this job?') }">
                   ${ _('Kill') }
               </a>
-           </td>
+            </td>
           </tr>
         % endfor
       </tbody>

+ 4 - 0
apps/oozie/src/oozie/views/dashboard.py

@@ -174,6 +174,10 @@ def split_oozie_jobs(oozie_jobs):
 
   for job in oozie_jobs:
     if job.is_running():
+      if job.type == 'Workflow':
+        job = get_oozie().get_job(job.id)
+      else:
+        job = get_oozie().get_coordinator(job.id)
       jobs_running.append(job)
     else:
       jobs_completed.append(job)

+ 2 - 0
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -146,6 +146,7 @@ class OozieApi(object):
     return self.get_jobs('coord', offset, cnt, **kwargs)
 
 
+  # TODO: make get_job accept any jobid
   def get_job(self, jobid):
     """
     get_job(jobid) -> Workflow
@@ -155,6 +156,7 @@ class OozieApi(object):
     wf = Workflow(self, resp)
     return wf
 
+
   def get_coordinator(self, jobid):
     params = self._get_params()
     resp = self._root.get('job/%s' % (jobid,), params)

+ 20 - 5
desktop/libs/liboozie/src/liboozie/types.py

@@ -23,6 +23,7 @@ http://incubator.apache.org/oozie/docs/3.2.0-incubating/docs/WebServicesAPI.html
 """
 
 from cStringIO import StringIO
+from time import mktime
 
 from desktop.lib import i18n
 from desktop.lib.django_util import PopupException
@@ -45,7 +46,7 @@ class Action(object):
   def _fixup(self): pass
 
   def is_finished(self):
-    return self.status == 'OK'
+    return self.status in ('OK', 'SUCCEEDED')
 
   @classmethod
   def create(self, action_class, action_dict):
@@ -266,9 +267,6 @@ class Job(object):
     res.append('kill')
     return res
 
-  def get_progress(self):
-    return sum([action.is_finished() for action in self.actions]) / float(max(len(self.actions), 1)) * 100
-
   def check_request_permission(self, request):
     """Raise PopupException if request user doesn't have permission to modify workflow"""
     if not request.user.is_superuser and request.user.username != self.user:
@@ -317,8 +315,10 @@ class Coordinator(Job):
   def _fixup(self):
     super(Coordinator, self)._fixup()
 
-    if self.nextMaterializedTime:
+    if self.nextMaterializedTime is not None:
       self.nextMaterializedTime = parse_timestamp(self.nextMaterializedTime)
+    else:
+      self.nextMaterializedTime = self.startTime
 
     # For when listing/mixing all the jobs together
     self.id = self.coordJobId
@@ -331,6 +331,17 @@ class Coordinator(Job):
   def get_absolute_url(self):
     return reverse('oozie:list_oozie_coordinator', kwargs={'job_id': self.id})
 
+  def get_progress(self):
+    """How much more time before the final materialization."""
+    next = mktime(self.nextMaterializedTime)
+    start = mktime(self.startTime)
+    end = mktime(self.endTime)
+
+    if end != start:
+      return int((1 - (end - next) / (end - start)) * 100)
+    else:
+      return 100
+
 
 class Workflow(Job):
   _ATTRS = [
@@ -371,6 +382,10 @@ class Workflow(Job):
   def get_absolute_url(self):
     return reverse('oozie:list_oozie_workflow', kwargs={'job_id': self.id})
 
+  def get_progress(self):
+    """How many actions are finished on the total of actions."""
+    return int(sum([action.is_finished() for action in self.actions]) / float(max(len(self.actions), 1)) * 100)
+
 
 class JobList(object):
   """