Explorar el Código

[oozie] import jobsub actions

abec hace 13 años
padre
commit
b0cfd86

+ 7 - 0
apps/oozie/src/oozie/forms.py

@@ -41,6 +41,13 @@ class WorkflowForm(forms.ModelForm):
     }
 
 
+class ImportJobsubDesignForm(forms.Form):
+  """Used for specifying what oozie actions to import"""
+  def __init__(self, choices=[], *args, **kwargs):
+    super(ImportJobsubDesignForm, self).__init__(*args, **kwargs)
+    self.fields['action_id'] = forms.ChoiceField(choices=choices, widget=forms.RadioSelect(attrs={'class':'radio'}))
+
+
 class NodeForm(forms.ModelForm):
   class Meta:
     ALWAYS_HIDE = ('workflow', 'children', 'node_type')

+ 55 - 0
apps/oozie/src/oozie/import_jobsub.py

@@ -0,0 +1,55 @@
+import re
+
+from jobsub.models import OozieDesign, OozieMapreduceAction, OozieStreamingAction,\
+  OozieJavaAction
+
+from oozie.models import Mapreduce, Java, Streaming
+
+def convert_jobsub_design(jobsub_design):
+  """Creates an oozie action from a jobsub design"""
+  action = jobsub_design.get_root_action()
+  if action is None:
+    return None
+  if action.action_type == OozieMapreduceAction.ACTION_TYPE:
+    action = _convert_jobsub_mapreduce_action(action)
+  elif action.action_type == OozieStreamingAction.ACTION_TYPE:
+    action = _convert_jobsub_streaming_action(action)
+  elif action.action_type == OozieJavaAction.ACTION_TYPE:
+    action = _convert_jobsub_java_action(action)
+  else:
+    return None
+  action.name = jobsub_design.name
+  action.description = jobsub_design.description
+  return action
+
+VARIABLE_NAME_REGEX = re.compile('(?<!\$)\$(\w+)')
+def _translate_jobsub_contents(contents):
+  return VARIABLE_NAME_REGEX.sub(r'${\1}', contents)
+
+def _convert_jobsub_mapreduce_action(jobsub_action):
+  action = Mapreduce(files=_translate_jobsub_contents(jobsub_action.files),
+    archives=_translate_jobsub_contents(jobsub_action.archives),
+    jar_path=_translate_jobsub_contents(jobsub_action.jar_path),
+    job_properties=_translate_jobsub_contents(jobsub_action.job_properties))
+  action.node_type = Mapreduce.node_type
+  return action
+
+def _convert_jobsub_streaming_action(jobsub_action):
+  action = Streaming(files=_translate_jobsub_contents(jobsub_action.files),
+    archives=_translate_jobsub_contents(jobsub_action.archives),
+    job_properties=_translate_jobsub_contents(jobsub_action.job_properties),
+    mapper=_translate_jobsub_contents(jobsub_action.mapper),
+    reducer=_translate_jobsub_contents(jobsub_action.reducer))
+  action.node_type = Streaming.node_type
+  return action
+
+def _convert_jobsub_java_action(jobsub_action):
+  action = Java(files=_translate_jobsub_contents(jobsub_action.files),
+    archives=_translate_jobsub_contents(jobsub_action.archives),
+    jar_path=_translate_jobsub_contents(jobsub_action.jar_path),
+    main_class=_translate_jobsub_contents(jobsub_action.main_class),
+    args=_translate_jobsub_contents(jobsub_action.args),
+    java_opts=_translate_jobsub_contents(jobsub_action.java_opts),
+    job_properties=_translate_jobsub_contents(jobsub_action.job_properties))
+  action.node_type = Java.node_type
+  return action

+ 0 - 1
apps/oozie/src/oozie/models.py

@@ -153,7 +153,6 @@ class WorkflowManager(models.Manager):
 
     return workflow
 
-
 class Workflow(Job):
   """
   http://incubator.apache.org/oozie/docs/3.2.0-incubating/docs/WorkflowFunctionalSpec.html

+ 39 - 2
apps/oozie/src/oozie/templates/editor/edit_workflow.mako

@@ -60,6 +60,9 @@ ${ layout.menubar(section='workflows') }
               <li class="active">
                 <a href="#add" data-toggle="tab">${ _('Add') }</a>
               </li>
+              <li>
+                <a href="#import" data-toggle="tab">${ _('Import') }</a>
+              </li>
              </ul>
 
             <div class="tab-content">
@@ -88,7 +91,16 @@ ${ layout.menubar(section='workflows') }
                   <i class="icon-plus"></i> ${ _('Pig') }
                 </a>
                 <p/>
-             </div>
+              </div>
+              <div class="tab-pane" id="import">
+                <p>
+                <a href="javascript:void(0);" title="${ _('Click to add to the end') }" class="btn modalWindow"
+                  data-modal-url="${ url('oozie:import_action', workflow=workflow.id, parent_action_id=workflow.end.get_parents()[0].id) }"
+                  data-modal-id="#modal-window">
+                  <i class="icon-plus"></i> ${ _('Import Jobsub') }
+                </a>
+                <p/>
+              </div>
             </div>
           </div>
 
@@ -162,6 +174,16 @@ ${ layout.menubar(section='workflows') }
 </div>
 
 
+<div id="modal-window" class="modal hide fade"></div>
+
+
+<style type="text/css">
+#modal-window .modal-content {
+  height: 300px;
+  overflow: auto;
+}
+</style>
+
 <script src="/static/ext/js/knockout-2.0.0.js" type="text/javascript" charset="utf-8"></script>
 
 <script type="text/javascript">
@@ -170,8 +192,23 @@ ${ layout.menubar(section='workflows') }
       window.location = $(this).attr('data-edit');
     });
 
-    $("a[data-row-selector='true']").jHueRowSelector();
+  $(".modalWindow").click(function(){
+    var _this = $(this);
+    $.ajax({
+      url: _this.attr("data-modal-url"),
+      beforeSend: function(xhr){
+        xhr.setRequestHeader("X-Requested-With", "Hue");
+      },
+      dataType: "html",
+      success: function(data){
+        var id = _this.attr("data-modal-id");
+        $(id).html(data);
+        $(id).modal("show");
+      }
+    });
   });
+
+  $("a[data-row-selector='true']").jHueRowSelector();
 </script>
 
 ${ utils.path_chooser_libs(True) }

+ 73 - 0
apps/oozie/src/oozie/templates/editor/import_workflow_action.mako

@@ -0,0 +1,73 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+  from django.utils.translation import ugettext as _
+%>
+
+<form id="actionForm" action="${ form_url }" method="POST">
+  <div class="modal-header">
+    <a href="#" class="close" data-dismiss="modal">&times;</a>
+    <h3>${_("Select Job Design to Import")}</h3>
+  </div>
+  <div class="modal-content">
+    <div class="container-fluid">
+      <table id="jobdesignerActionsTable" class="table datatables">
+        <thead>
+          <tr>
+            <th></th>
+            <th>${ _('Name') }</th>
+            <th>${ _('Description') }</th>
+          </tr>
+        </thead>
+        <tbody>
+          %for action in available_actions:
+            <tr class="action-row">
+              <td class=".btn-large action-column" data-row-selector-exclude="true" style="background-color: white;">
+                <input type="radio" name="action_id" value="${ action.id }" />
+              </td>
+              <td>
+                ${ action.name }
+              </td>
+              <td>${ action.description }</td>
+            </tr>
+          %endfor
+        </tbody>
+      </table>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <a href="#" class="btn secondary" data-dismiss="modal">${_('Cancel')}</a>
+    <input type="submit" class="btn primary" value="${_('Import')}"/>
+  </div>
+</form>
+
+
+<script src="/static/ext/js/datatables-paging-0.1.js" type="text/javascript" charset="utf-8"></script>
+
+<script type="text/javascript" charset="utf-8">
+  $(document).ready(function() {
+    $(".action-row").click(function(e){
+      var select_btn = $(this).find('input');
+      select_btn.prop("checked", true);
+
+      $(".action-row").css("background-color", "");
+      $(this).css("background-color", "#ECF4F8");
+    });
+
+    $("a[data-row-selector='true']").jHueRowSelector();
+  });
+</script>

+ 24 - 0
apps/oozie/src/oozie/tests.py

@@ -20,6 +20,7 @@ try:
 except ImportError:
   import simplejson as json
 import logging
+import re
 
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
@@ -27,6 +28,8 @@ from django.core.urlresolvers import reverse
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
+from jobsub.management.commands import jobsub_setup
+from jobsub.models import OozieDesign
 from liboozie import oozie_api
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   CoordinatorList, WorkflowAction
@@ -231,6 +234,27 @@ class TestEditor:
     assert_equal(node_count + 1, self.wf.actions.count())
 
 
+  def test_import_action(self):
+    # Setup jobsub examples
+    if not jobsub_setup.Command().has_been_setup():
+      jobsub_setup.Command().handle()
+
+    # There should be 3 from examples
+    jobsub_design = OozieDesign.objects.all()[0]
+    node_size = len(Node.objects.all())
+    kwargs = dict(workflow=self.wf.id, parent_action_id=self.wf.end.get_parents()[0].id)
+    response = self.c.post(reverse('oozie:import_action', kwargs=kwargs), {'action_id': jobsub_design.id})
+    assert_equal(302, response.status_code)
+    assert_equal(node_size + 1, len(Node.objects.all()))
+
+    # There should now be an imported action at the end of Node list
+    # Need to test properties to make sure we got it right
+    # Must also make sure that jobsub field values are translated
+    translation_regex = re.compile('(?<!\$)\$(\w+)')
+    node = Node.objects.all()[len(Node.objects.all())-1].get_full_node()
+    for field in node.PARAM_FIELDS:
+      assert_equal(translation_regex.sub(r'${\1}', getattr(jobsub_design.get_root_action(), field)), getattr(node, field))
+
   def test_decision_node(self):
     action1 = Node.objects.get(name='action-name-1')
     action2 = Node.objects.get(name='action-name-2')

+ 1 - 0
apps/oozie/src/oozie/urls.py

@@ -33,6 +33,7 @@ urlpatterns = patterns(
   url(r'^resubmit_workflow/(?P<oozie_wf_id>[-\w]+)$', 'resubmit_workflow', name='resubmit_workflow'),
 
   url(r'^new_action/(?P<workflow>\d+)/(?P<node_type>\w+)/(?P<parent_action_id>\d+)$', 'new_action', name='new_action'),
+  url(r'^import_action/(?P<workflow>\d+)/(?P<parent_action_id>\d+)$', 'import_action', name='import_action'),
   url(r'^edit_action/(?P<action>\d+)$', 'edit_action', name='edit_action'),
   url(r'^edit_workflow_fork/(?P<action>\d+)$', 'edit_workflow_fork', name='edit_workflow_fork'),
   url(r'^delete_action/(?P<action>\d+)$', 'delete_action', name='delete_action'),

+ 37 - 2
apps/oozie/src/oozie/views/editor.py

@@ -20,6 +20,7 @@ try:
 except ImportError:
   import simplejson as json
 import logging
+import re
 
 
 from django.core.urlresolvers import reverse
@@ -29,21 +30,26 @@ from django.forms.models import inlineformset_factory, modelformset_factory
 from django.http import HttpResponse
 from django.shortcuts import redirect
 from django.utils.functional import wraps
+from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import render, PopupException, extract_field_data
 from desktop.lib.rest.http_client import RestException
 from desktop.log.access import access_warn
 from hadoop.fs.exceptions import WebHdfsException
+from jobsub.models import OozieDesign, OozieMapreduceAction, OozieStreamingAction,\
+  OozieJavaAction
 from liboozie.submittion import Submission
 
 from oozie.conf import SHARE_JOBS
+from oozie.import_jobsub import convert_jobsub_design
 from oozie.management.commands import oozie_setup
 from oozie.models import Workflow, Node, Link, History, Coordinator,\
-  Dataset, DataInput, DataOutput, Job, _STD_PROPERTIES_JSON
+  Mapreduce, Java, Streaming, Dataset, DataInput, DataOutput, Job,\
+  _STD_PROPERTIES_JSON
 from oozie.forms import NodeForm, WorkflowForm, CoordinatorForm, DatasetForm,\
   DataInputForm, DataInputSetForm, DataOutputForm, DataOutputSetForm, LinkForm,\
-  DefaultLinkForm, design_form_by_type, ParameterForm
+  DefaultLinkForm, design_form_by_type, ImportJobsubDesignForm, ParameterForm
 
 
 LOG = logging.getLogger(__name__)
@@ -403,6 +409,35 @@ def edit_action(request, action):
   })
 
 
+@check_job_access_permission
+def import_action(request, workflow, parent_action_id):
+  available_actions = OozieDesign.objects.all()
+
+  if request.method == 'POST':
+    form = ImportJobsubDesignForm(data=request.POST, choices=[(action.id, action.name) for action in available_actions])
+    if form.is_valid():
+      try:
+        design = OozieDesign.objects.get(id=form.cleaned_data['action_id'])
+        action = convert_jobsub_design(design)
+        action.workflow = workflow
+        action.save()
+
+        workflow.add_action(action, parent_action_id)
+      except OozieDesign.DoesNotExist:
+        request.error(_('Jobsub design doesn\'t exist.'))
+      except (Mapreduce.DoesNotExist, Streaming.DoesNotExist, Java.DoesNotExist):
+        request.error(_('Could not convert jobsub design'))
+      except:
+        request.error(_('Could not convert jobsub design or add action to workflow'))
+
+    return redirect(reverse('oozie:edit_workflow', kwargs={'workflow': action.workflow.id}))
+
+  return render('editor/import_workflow_action.mako', request, {
+    'workflow': workflow,
+    'available_actions': available_actions,
+    'form_url': reverse('oozie:import_action', kwargs={'workflow': workflow.id, 'parent_action_id': parent_action_id}),
+  })
+
 @check_action_access_permission
 @check_action_edition_permission
 def edit_workflow_fork(request, action):

+ 2 - 2
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -192,9 +192,9 @@ class OozieApi(object):
     return self._root.put('job/%s' % jobid, params)
 
 
-  def submit_workflow(self, application_path):
+  def submit_workflow(self, application_path, properties=None):
     """
-    submit_workflow(application_path) -> jobid
+    submit_workflow(application_path, properties=None) -> jobid
 
     Submit a job to Oozie. May raise PopupException.
     """