Преглед на файлове

HUE-4055 [oozie] Upgrade oozie spark action to support spark-action:0.2 schema version

krish преди 9 години
родител
ревизия
50a9397c1e

+ 1 - 0
apps/oozie/src/oozie/importlib/xslt2/workflows/0.5/action.xslt

@@ -21,6 +21,7 @@
 <xsl:import href="extensions/ssh.0.1.xslt"/>
 <xsl:import href="extensions/ssh.0.2.xslt"/>
 <xsl:import href="extensions/spark.0.1.xslt"/>
+<xsl:import href="extensions/spark.0.2.xslt"/>
 <xsl:import href="nodes/fs.xslt"/>
 <xsl:import href="nodes/java.xslt"/>
 <xsl:import href="nodes/mapreduce.xslt"/>

+ 17 - 0
apps/oozie/src/oozie/importlib/xslt2/workflows/0.5/extensions/spark.0.2.xslt

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:spark="uri:oozie:spark-action:0.2">
+
+<xsl:template match="spark:spark">
+
+  ,"spark": {
+    "name": "<xsl:value-of select="*[local-name()='name']"/>",
+    "master": "<xsl:value-of select="*[local-name()='master']"/>",
+    "mode": "<xsl:value-of select="*[local-name()='mode']"/>",
+    "class": "<xsl:value-of select="*[local-name()='class']"/>",
+    "jar": "<xsl:value-of select="*[local-name()='jar']"/>"
+  }
+
+</xsl:template>
+
+</xsl:stylesheet>

+ 10 - 1
apps/oozie/src/oozie/models2.py

@@ -805,6 +805,8 @@ def _upgrade_older_node(node):
     node['properties']['content_type'] = 'text/plain'
     node['properties']['attachment'] = ''
 
+  if node['type'] == 'spark-widget' and 'files' not in node['properties']:
+    node['properties']['files'] = []
 
 class Action(object):
 
@@ -1797,6 +1799,13 @@ class SparkAction(Action):
           'help_text': _('The name of the spark application'),
           'type': ''
      },
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Files put in the running directory.'),
+          'type': ''
+     },
     'class': {
           'name': 'class',
           'label': _('Main class'),
@@ -1862,7 +1871,7 @@ class SparkAction(Action):
 
   @classmethod
   def get_mandatory_fields(cls):
-    return [cls.FIELDS['spark_master'], cls.FIELDS['mode'], cls.FIELDS['jars'], cls.FIELDS['class']]
+    return [cls.FIELDS['spark_master'], cls.FIELDS['mode'], cls.FIELDS['jars']]
 
 
 class KillAction(Action):

+ 19 - 13
apps/oozie/src/oozie/templates/editor2/common_workflow.mako

@@ -843,19 +843,25 @@
             ${ _('Arguments') } <i class="fa fa-plus"></i>
           </a>
         </h6>
-        <ul class="unstyled white sortable-arguments" data-bind="visible: properties.spark_arguments().length > 0, sortable: { data: properties.spark_arguments, options: { axis: 'y', containment: 'parent' }}">
-          <li>
-            <span class="muted move-widget">
-              <i class="fa fa-arrows"></i>
-            </span>
-            <input type="text" class="input-xlarge filechooser-input seventy" data-bind="filechooser: value, filechooserOptions: globalFilechooserOptions, hdfsAutocomplete: value, attr: { placeholder:  $root.workflow_properties.spark_arguments.help_text }" validate="nonempty"/>
-            <span data-bind='template: { name: "common-fs-link", data: {path: value, with_label: false}}'></span>
-            <a href="#" data-bind="click: function(){ $parent.properties.spark_arguments.remove(this); $(document).trigger('drawArrows') }">
-              <i class="fa fa-minus"></i>
-            </a>
-          </li>
-       </ul>
-       <em data-bind="visible: properties.spark_arguments().length == 0">${ _('No arguments defined.') }</em>
+
+        <div class="row-fluid">
+          <div>
+            <ul class="unstyled white sortable-arguments" data-bind="visible: properties.spark_arguments().length > 0, sortable: { data: properties.spark_arguments, options: { axis: 'y', containment: 'parent' }}">
+              <li>
+                <span class="muted move-widget">
+                  <i class="fa fa-arrows"></i>
+                </span>
+                <input type="text" class="input-xlarge filechooser-input seventy" data-bind="filechooser: value, filechooserOptions: globalFilechooserOptions, hdfsAutocomplete: value, attr: { placeholder:  $root.workflow_properties.spark_arguments.help_text }" validate="nonempty"/>
+                <span data-bind='template: { name: "common-fs-link", data: {path: value, with_label: false}}'></span>
+                <a href="#" data-bind="click: function(){ $parent.properties.spark_arguments.remove(this); $(document).trigger('drawArrows') }">
+                  <i class="fa fa-minus"></i>
+                </a>
+              </li>
+            </ul>
+            <em data-bind="visible: properties.spark_arguments().length == 0">${ _('No arguments defined.') }</em>
+          </div>
+          <div class="span6" data-bind="template: { name: 'common-properties-files' }"></div>
+        </div>
       </div>
     </div>
 

+ 3 - 1
apps/oozie/src/oozie/templates/editor2/gen/workflow-spark.xml.mako

@@ -18,7 +18,7 @@
 <%namespace name="common" file="workflow-common.xml.mako" />
 
     <action name="${ node['name'] }"${ common.credentials(node['properties']['credentials']) }${ common.retry_max(node['properties']['retry_max']) }${ common.retry_interval(node['properties']['retry_interval']) }>
-        <spark xmlns="uri:oozie:spark-action:0.1">
+        <spark xmlns="uri:oozie:spark-action:0.2">
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>
 
@@ -45,6 +45,8 @@
             % for argument in node['properties']['spark_arguments']:
               <arg>${ argument['value'] }</arg>
             % endfor
+
+            ${ common.distributed_cache(node['properties']['files'], node['properties']['archives']) }
         </spark>
         <ok to="${ node_mapping[node['children'][0]['to']].name }"/>
         <error to="${ node_mapping[node['children'][1]['error']].name }"/>

+ 32 - 1
apps/oozie/src/oozie/tests2.py

@@ -923,4 +923,35 @@ class TestExternalWorkflowGraph(object):
     assert_true(len(workflow_data['workflow']['nodes']) == 10)
     assert_equal(workflow_data['layout'][0]['rows'][6]['widgets'][0]['widgetType'], 'decision-widget')
     assert_equal(workflow_data['workflow']['nodes'][7]['type'], 'decision-widget')
-    assert_true(len(workflow_data['workflow']['nodes'][7]['children']) == 2)
+    assert_true(len(workflow_data['workflow']['nodes'][7]['children']) == 2)
+
+
+  def test_gen_workflow_data_from_xml_for_spark_schema02(self):
+    self.wf.definition = """<workflow-app name="My_Workflow" xmlns="uri:oozie:workflow:0.5">
+    <start to="spark-fa35"/>
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <action name="spark-fa35">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <master>local[*]</master>
+            <mode>client</mode>
+            <name>MySpark</name>
+            <jar>wordcount.py</jar>
+            <file>/user/admin/wordcount.py#wordcount.py</file>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+    <end name="End"/>
+    </workflow-app>
+    """
+
+    workflow_data = Workflow.gen_workflow_data_from_xml(self.user, self.wf)
+
+    assert_true(len(workflow_data['layout'][0]['rows']) == 4)
+    assert_true(len(workflow_data['workflow']['nodes']) == 4)
+    assert_equal(workflow_data['layout'][0]['rows'][1]['widgets'][0]['widgetType'], 'spark-widget')
+    assert_true(len(workflow_data['workflow']['nodes'][1]['children']) == 2)

+ 2 - 0
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -89,6 +89,8 @@ class Submission(object):
         properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
 
     self.properties['security_enabled'] = self.api.security_enabled
+    if 'oozie.use.system.libpath' not in properties:
+        properties['oozie.use.system.libpath'] = 'true'
 
   def __str__(self):
     if self.oozie_id: