Bladeren bron

HUE-820 [oozie] Display any workflow as a graph

Import a workflow from the Oozie XML definition from the API if there is no
Hue Worflow model associated with it
If the import fail no graph is showned and a message is added in the logs
Refactor views and model
Reuse import workflow
Make sure to delete the imported workflow
Make definition tab higher
Improve variable outputs HDFs link with guess_link
Add a negative and positive test
Romain Rigaux 13 jaren geleden
bovenliggende
commit
517312b76e

+ 1 - 3
apps/oozie/src/oozie/import_workflow.py

@@ -35,10 +35,8 @@ try:
   import json
 except ImportError:
   import simplejson as json
-from collections import deque
+
 import logging
-import re
-import sys
 from lxml import etree
 
 from django.core import serializers

+ 25 - 6
apps/oozie/src/oozie/models.py

@@ -15,15 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-import time
-from django.template.defaultfilters import escapejs
-
 try:
   import json
 except ImportError:
   import simplejson as json
+import logging
 import re
+import time
+
 from datetime import datetime,  timedelta
 from string import Template
 from itertools import chain
@@ -32,6 +31,8 @@ from django.db import models
 from django.core.urlresolvers import reverse
 from django.core.validators import RegexValidator
 from django.contrib.auth.models import User
+from django.forms.models import inlineformset_factory
+from django.template.defaultfilters import escapejs
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
 from desktop.log.access import access_warn
@@ -42,8 +43,8 @@ from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.hadoopfs import Hdfs
 from liboozie.submittion import Submission
 
-from oozie.management.commands import oozie_setup
 from oozie.conf import REMOTE_SAMPLE_DIR, SHARE_JOBS
+from oozie.management.commands import oozie_setup
 from timezones import TIMEZONES
 
 
@@ -408,7 +409,11 @@ class Workflow(Job):
       child = Link.objects.filter(parent=node).exclude(name__in=['related', 'kill', 'error'])[0].child
       return [node] + self.get_hierarchy_rec(child)
 
-  def gen_status_graph(self, forms, actions):
+  def gen_status_graph(self, oozie_workflow):
+    from oozie.forms import NodeForm  # Circular dependency
+    actions = oozie_workflow.get_working_actions()
+    WorkflowFormSet = inlineformset_factory(Workflow, Node, form=NodeForm, max_num=0, can_order=False, can_delete=False)
+    forms = WorkflowFormSet(instance=self).forms
     template='editor/gen/workflow-graph-status.xml.mako'
 
     index = dict([(form.instance.id, form) for form in forms])
@@ -416,6 +421,20 @@ class Workflow(Job):
 
     return django_mako.render_to_string(template, {'nodes': self.get_hierarchy(), 'index': index, 'actions': actions_index})
 
+  @classmethod
+  def gen_status_graph_from_xml(cls, user, oozie_workflow):
+    from oozie.import_workflow import import_workflow # Circular dependency
+    try:
+      workflow = Workflow.objects.new_workflow(user)
+      workflow.save()
+      try:
+        import_workflow(workflow, oozie_workflow.definition)
+        return workflow.gen_status_graph(oozie_workflow)
+      except Exception, e:
+        LOG.info('Workflow %s could not be converted to a graph: %s' % (oozie_workflow.id, e))
+    finally:
+      workflow.delete()
+
   def to_xml(self):
     tmpl = 'editor/gen/workflow.xml.mako'
     return re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {'workflow': self}))

+ 109 - 113
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako

@@ -143,7 +143,7 @@ ${ layout.menubar(section='dashboard') }
   <br/><br/>
 
     <ul class="nav nav-tabs">
-      % if hue_workflow:
+      % if workflow_graph:
         <li class="active"><a href="#graph" data-toggle="tab">${ _('Graph') }</a></li>
         <li><a href="#actions" data-toggle="tab">${ _('Actions') }</a></li>
       % else:
@@ -156,129 +156,119 @@ ${ layout.menubar(section='dashboard') }
     </ul>
 
     <div id="workflow-tab-content" class="tab-content" style="min-height:200px">
-     % if hue_workflow:
+
+     % if workflow_graph:
        <div id="graph" class="tab-pane active">
-         % if hue_workflow is not None:
-         <%
-           from oozie.forms import NodeForm
-           from oozie.models import Workflow, Node
-           from django.forms.models import inlineformset_factory
-
-           WorkflowFormSet = inlineformset_factory(Workflow, Node, form=NodeForm, max_num=0, can_order=False, can_delete=False)
-           forms = WorkflowFormSet(instance=hue_workflow.get_full_node()).forms
-         %>
-
-           ${ hue_workflow.get_full_node().gen_status_graph(forms, oozie_workflow.get_working_actions()) | n,unicode }
-         % endif
+           ${ workflow_graph | n,unicode }
        </div>
      % endif
 
-    <div class="tab-pane ${ utils.if_false(hue_workflow, 'active') }" id="actions">
-      % if oozie_workflow.get_working_actions():
-        <table class="table table-striped table-condensed selectable">
-          <thead>
-            <tr>
-              <th>${ _('Logs') }</th>
-              <th>${ _('Id') }</th>
-              <th>${ _('Name') }</th>
-              <th>${ _('Type') }</th>
-              <th>${ _('Status') }</th>
-              <th>${ _('External Id') }</th>
-
-              <th>${ _('Start Time') }</th>
-              <th>${ _('End Time') }</th>
-
-              <th>${ _('Retries') }</th>
-              <th>${ _('Error Message') }</th>
-              <th>${ _('Transition') }</th>
-
-              <th>${ _('Data') }</th>
-            </tr>
-          </thead>
-          <tbody>
-            % for i, action in enumerate(oozie_workflow.get_working_actions()):
+      <div id="actions" class="tab-pane ${ utils.if_false(workflow_graph, 'active') }">
+        % if oozie_workflow.get_working_actions():
+          <table class="table table-striped table-condensed selectable">
+            <thead>
               <tr>
-                <td>
-                  % if action.externalId:
-                    <a href="${ url('jobbrowser.views.job_single_logs', job=action.externalId) }" data-row-selector-exclude="true"><i class="icon-tasks"></i></a>
-                  % endif
-                </td>
-                <td>
-                  <a href="${ url('oozie:list_oozie_workflow_action', action=action.id) }" data-row-selector='true'>${ action.id }</a>
-                </td>
-                <td>
-                  % if design_link:
-                    <a href="${ design_link }">${ action.name }</a>
-                  % else:
-                    ${ action.name }
-                  % endif
-                </td>
-                <td>${ action.type }</td>
-                <td><span class="label ${ utils.get_status(action.status) }">${ action.status }</span></td>
-                <td>
-                  % if action.externalId:
-                    <a href="${ url('jobbrowser.views.single_job', job=action.externalId) }">${ "_".join(action.externalId.split("_")[-2:]) }</a>
-                  % endif
-                </td>
-
-                <td>${ utils.format_time(action.startTime) }</td>
-                <td>${ utils.format_time(action.endTime) }</td>
-
-                <td>${ action.retries }</td>
-                <td>${ action.errorMessage }</td>
-                <td>${ action.transition }</td>
-
-                <td>${ action.data }</td>
+                <th>${ _('Logs') }</th>
+                <th>${ _('Id') }</th>
+                <th>${ _('Name') }</th>
+                <th>${ _('Type') }</th>
+                <th>${ _('Status') }</th>
+                <th>${ _('External Id') }</th>
+
+                <th>${ _('Start Time') }</th>
+                <th>${ _('End Time') }</th>
+
+                <th>${ _('Retries') }</th>
+                <th>${ _('Error Message') }</th>
+                <th>${ _('Transition') }</th>
+
+                <th>${ _('Data') }</th>
               </tr>
-            % endfor
-          <tbody>
-        </table>
+            </thead>
+            <tbody>
+              % for i, action in enumerate(oozie_workflow.get_working_actions()):
+                <tr>
+                  <td>
+                    % if action.externalId:
+                      <a href="${ url('jobbrowser.views.job_single_logs', job=action.externalId) }" data-row-selector-exclude="true"><i class="icon-tasks"></i></a>
+                    % endif
+                  </td>
+                  <td>
+                    <a href="${ url('oozie:list_oozie_workflow_action', action=action.id) }" data-row-selector='true'>${ action.id }</a>
+                  </td>
+                  <td>
+                    % if design_link:
+                      <a href="${ design_link }">${ action.name }</a>
+                    % else:
+                      ${ action.name }
+                    % endif
+                  </td>
+                  <td>${ action.type }</td>
+                  <td><span class="label ${ utils.get_status(action.status) }">${ action.status }</span></td>
+                  <td>
+                    % if action.externalId:
+                      <a href="${ url('jobbrowser.views.single_job', job=action.externalId) }">${ "_".join(action.externalId.split("_")[-2:]) }</a>
+                    % endif
+                  </td>
+
+                  <td>${ utils.format_time(action.startTime) }</td>
+                  <td>${ utils.format_time(action.endTime) }</td>
+
+                  <td>${ action.retries }</td>
+                  <td>${ action.errorMessage }</td>
+                  <td>${ action.transition }</td>
+
+                  <td>${ action.data }</td>
+                </tr>
+              % endfor
+            <tbody>
+          </table>
         % endif
-      </div>
+       </div>
 
-      <div class="tab-pane" id="details">
-        <table class="table table-condensed">
-          <tbody>
-            <tr>
-              <td>${ _('Group') }</td>
-              <td>${ oozie_workflow.group or '-' }</td>
-            </tr>
-            <tr>
-              <td>${ _('External Id') }</td>
-              <td>${ oozie_workflow.externalId or '-' }</td>
-            </tr>
-            <tr>
-              <td>${ _('Start Time') }</td>
-              <td>${ utils.format_time(oozie_workflow.startTime) }</td>
-            </tr>
-            <tr>
-              <td>${ _('Created Time') }</td>
-              <td>${ utils.format_time(oozie_workflow.createdTime) }</td>
-            </tr>
-            <tr>
-              <td>${ _('End Time') }</td>
-              <td>${  utils.format_time(oozie_workflow.endTime) }</td>
-            </tr>
-            <tr>
-              <td>${ _('Application Path') }</td>
-              <td>${  utils.hdfs_link(oozie_workflow.appPath) }</td>
-            </tr>
-          </tbody>
-        </table>
-      </div>
+       <div class="tab-pane" id="details">
+          <table class="table table-condensed">
+            <tbody>
+              <tr>
+                <td>${ _('Group') }</td>
+                <td>${ oozie_workflow.group or '-' }</td>
+              </tr>
+              <tr>
+                <td>${ _('External Id') }</td>
+                <td>${ oozie_workflow.externalId or '-' }</td>
+              </tr>
+              <tr>
+                <td>${ _('Start Time') }</td>
+                <td>${ utils.format_time(oozie_workflow.startTime) }</td>
+              </tr>
+              <tr>
+                <td>${ _('Created Time') }</td>
+                <td>${ utils.format_time(oozie_workflow.createdTime) }</td>
+              </tr>
+              <tr>
+                <td>${ _('End Time') }</td>
+                <td>${  utils.format_time(oozie_workflow.endTime) }</td>
+              </tr>
+              <tr>
+                <td>${ _('Application Path') }</td>
+                <td>${  utils.hdfs_link(oozie_workflow.appPath) }</td>
+              </tr>
+            </tbody>
+          </table>
+       </div>
 
-      <div class="tab-pane" id="configuration">
-          ${ utils.display_conf(oozie_workflow.conf_dict) }
-      </div>
+       <div class="tab-pane" id="configuration">
+            ${ utils.display_conf(oozie_workflow.conf_dict) }
+       </div>
 
-      <div class="tab-pane" id="log">
-          <pre>${ oozie_workflow.log }</pre>
-      </div>
+       <div class="tab-pane" id="log">
+            <pre>${ oozie_workflow.log }</pre>
+       </div>
 
-      <div class="tab-pane" id="definition">
-          <textarea id="definitionEditor">${ oozie_workflow.definition }</textarea>
-      </div>
-  </div>
+       <div class="tab-pane" id="definition" style="min-height:400px">
+            <textarea id="definitionEditor">${ oozie_workflow.definition }</textarea>
+       </div>
+    </div>
 
   <div style="margin-top: 20px; margin-bottom: 20px">
     <a class="btn" onclick="history.back()">${ _('Back') }</a>
@@ -304,6 +294,12 @@ ${ layout.menubar(section='dashboard') }
 <link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <script src="/static/ext/js/codemirror-xml.js"></script>
 
+<style>
+.CodeMirror.cm-s-default {
+   height:500px;
+}
+</style>
+
 <script type="text/javascript">
   $(document).ready(function() {
 

+ 3 - 1
apps/oozie/src/oozie/templates/utils.inc.mako

@@ -126,7 +126,9 @@
 
 <%def name="guess_hdfs_link(name, path)">
   <%
-    if name.endswith('dir') or name.endswith('path') or path.startswith('/') or path.startswith('hdfs://'):
+    import re
+
+    if re.search('(dir|path|output|input)', name, re.I) or path.startswith('/') or path.startswith('hdfs://'):
       return hdfs_link(path)
     else:
       return path

+ 46 - 3
apps/oozie/src/oozie/tests.py

@@ -23,7 +23,6 @@ import logging
 import re
 import os
 
-from nose.plugins.skip import SkipTest
 from nose.tools import raises, assert_true, assert_false, assert_equal, assert_not_equal
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
@@ -111,10 +110,37 @@ class MockOozieApi:
     return 'Done'
 
   def get_job_definition(self, jobid):
-    return '<xml></xml>'
+    if jobid == MockOozieApi.WORKFLOW_IDS[0]:
+      return """<workflow-app name="MapReduce" xmlns="uri:oozie:workflow:0.4">
+      <start to="Sleep"/>
+      <action name="Sleep">
+          <map-reduce>
+              <job-tracker>${jobTracker}</job-tracker>
+              <name-node>${nameNode}</name-node>
+              <configuration>
+                  <property>
+                      <name>mapred.reduce.tasks</name>
+                      <value>1</value>
+                  </property>
+                  <property>
+                      <name>sleep.job.reduce.sleep.time</name>
+                      <value>${REDUCER_SLEEP_TIME}</value>
+                  </property>
+              </configuration>
+          </map-reduce>
+          <ok to="end"/>
+          <error to="kill"/>
+      </action>
+      <kill name="kill">
+          <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+      </kill>
+      <end name="end"/>
+  </workflow-app>"""
+    else:
+      return """<workflow-app name="MapReduce" xmlns="uri:oozie:workflow:0.4">BAD</workflow-app>"""
 
   def get_job_log(self, jobid):
-    return '<xml></xml>'
+    return '2013-01-08 16:28:06,487  INFO ActionStartXCommand:539 - USER[romain] GROUP[-] TOKEN[] APP[MapReduce] JOB[0000002-130108101138395-oozie-oozi-W] ACTION[0000002-130108101138395-oozie-oozi-W@:start:] Start action [0000002-130108101138395-oozie-oozi-W@:start:] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10]'
 
 
 class OozieMockBase(object):
@@ -129,6 +155,7 @@ class OozieMockBase(object):
     oozie_api.OozieApi = MockOozieApi
     oozie_api._api_cache = None
 
+    History.objects.all().delete()
     Coordinator.objects.all().delete()
 
     self.c = make_logged_in_client(is_superuser=False)
@@ -1870,6 +1897,22 @@ class TestDashboard(OozieMockBase):
     response = client_not_me.get(reverse('oozie:list_oozie_coordinator', args=[MockOozieApi.COORDINATOR_IDS[0]]))
     assert_false('Permission denied' in response.content, response.content)
 
+  def test_good_workflow_status_graph(self):
+    workflow_count = Workflow.objects.count()
+
+    response = self.c.get(reverse('oozie:list_oozie_workflow', args=[MockOozieApi.WORKFLOW_IDS[0]]), {})
+
+    assert_true(response.context['workflow_graph'])
+    assert_equal(Workflow.objects.count(), workflow_count)
+
+  def test_bad_workflow_status_graph(self):
+    workflow_count = Workflow.objects.count()
+
+    response = self.c.get(reverse('oozie:list_oozie_workflow', args=[MockOozieApi.WORKFLOW_IDS[1]]), {})
+
+    assert_true(response.context['workflow_graph'] is None)
+    assert_equal(Workflow.objects.count(), workflow_count)
+
 
 class TestUtils(OozieMockBase):
 

+ 7 - 1
apps/oozie/src/oozie/views/dashboard.py

@@ -38,7 +38,7 @@ from oozie.forms import RerunForm, ParameterForm
 
 
 from oozie.conf import OOZIE_JOBS_COUNT
-from oozie.models import History, Job
+from oozie.models import History, Job, Workflow
 from oozie.settings import DJANGO_APPS
 
 
@@ -137,6 +137,11 @@ def list_oozie_workflow(request, job_id, coordinator_job_id=None):
 
   parameters = oozie_workflow.conf_dict.copy()
 
+  if hue_workflow:
+    workflow_graph = hue_workflow.gen_status_graph(oozie_workflow)
+  else:
+    workflow_graph = Workflow.gen_status_graph_from_xml(request.user, oozie_workflow)
+
   return render('dashboard/list_oozie_workflow.mako', request, {
     'history': history,
     'oozie_workflow': oozie_workflow,
@@ -145,6 +150,7 @@ def list_oozie_workflow(request, job_id, coordinator_job_id=None):
     'hue_coord': hue_coord,
     'parameters': parameters,
     'has_job_edition_permission': has_job_edition_permission,
+    'workflow_graph': workflow_graph
   })
 
 

+ 4 - 0
apps/oozie/static/css/workflow.css

@@ -1,3 +1,7 @@
+ul {
+  list-style-type: none;
+}
+
 #modal-window .modal-content {
   height: 300px;
   overflow: auto;