Эх сурвалжийг харах

HUE-4720 [oozie] Drag & Drop saved Spark app into a workflow

Romain Rigaux 9 жил өмнө
parent
commit
4e942229c6

+ 7 - 0
apps/oozie/src/oozie/conf.py

@@ -81,6 +81,13 @@ ENABLE_CRON_SCHEDULING = Config( # Until Hue 3.8
   help=_t('Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.')
 )
 
+ENABLE_DOCUMENT_ACTION = Config(
+  key="enable_document_action",
+  help=_t("Flag to enable the saved Editor queries to be dragged and dropped into a workflow."),
+  type=bool,
+  default=True
+)
+
 
 def config_validator(user):
   res = []

+ 89 - 5
apps/oozie/src/oozie/models2.py

@@ -759,10 +759,13 @@ class Node():
       self.data['properties']['app_jar'] = properties['app_jar'] # Not used here
       self.data['properties']['files'] = [{'value': f['path']} for f in properties['files']]
       self.data['properties']['arguments'] = [{'value': prop} for prop in properties['arguments']]
-    elif self.data['type'] == SparkDocumentAction.TYPE:
+    elif self.data['type'] == SparkDocumentAction.TYPE or self.data['type'] == 'spark-document':
       notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=self.data['properties']['uuid']))
       properties = notebook.get_data()['snippets'][0]['properties']
 
+      if self.data['type'] == 'spark-document': # Oozie Document Action
+        self.data['properties']['app_name'] = properties['app_name']
+
       self.data['properties']['class'] = properties['class']
       self.data['properties']['jars'] = os.path.basename(properties['jars'][0])
       self.data['properties']['files'] = [{'value': f} for f in properties['jars']] + [{'value': f['path']} for f in properties['files']]
@@ -1825,7 +1828,7 @@ class SparkAction(Action):
      'spark_master': {
           'name': 'spark_master',
           'label': _('Spark Master'),
-          'value': 'local[*]',
+          'value': 'yarn',
           'help_text': _('Ex: spark://host:port, mesos://host:port, yarn, or local.'),
           'type': ''
      },
@@ -1833,7 +1836,7 @@ class SparkAction(Action):
           'name': 'mode',
           'label': _('Mode'),
           'value': 'client',
-          'help_text': _('e.g. client,cluster'),
+          'help_text': _('e.g. Client cluster'),
           'type': ''
      },
      'app_name': {
@@ -2152,9 +2155,89 @@ class JavaDocumentAction(Action):
     return [cls.FIELDS['uuid']]
 
 
-class SparkDocumentAction(SparkAction):
+class SparkDocumentAction(Action):
   TYPE = 'spark2-document'
+  FIELDS = {
+    'uuid': {
+        'name': 'uuid',
+        'label': _('Spark program'),
+        'value': '',
+        'help_text': _('Select a saved Spark program you want to schedule.'),
+        'type': 'spark'
+     },
+     'spark_master': {
+          'name': 'spark_master',
+          'label': _('Spark Master'),
+          'value': 'yarn',
+          'help_text': _('Ex: spark://host:port, mesos://host:port, yarn, or local.'),
+          'type': ''
+     },
+     'mode': {
+          'name': 'mode',
+          'label': _('Mode'),
+          'value': 'client',
+          'help_text': _('e.g. Client cluster'),
+          'type': ''
+     },
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Path to file to put in the running directory.'),
+          'type': ''
+     },
+     'spark_arguments': {
+          'name': 'spark_arguments',
+          'label': _('Arguments'),
+          'value': [],
+          'help_text': _('Arguments, one by one, e.g. 1000, /path/a.')
+     },
+     'parameters': { # For Oozie Action Document
+          'name': 'parameters',
+          'label': _('Parameters'),
+          'value': [],
+          'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}')  % {'type': TYPE.title()},
+          'type': ''
+     },
+     # Common
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production')
+     },
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.')
+     },
+     'job_xml': {
+          'name': 'job_xml',
+          'label': _('Job XML'),
+          'value': '',
+          'help_text': _('Refer to a Hadoop JobConf job.xml'),
+          'type': ''
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
 
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
 
 
 class DecisionNode(Action):
@@ -2189,7 +2272,8 @@ NODES = {
   'spark-widget': SparkAction,
   'generic-widget': GenericAction,
   'hive-document-widget': HiveDocumentAction,
-  'java-document-widget': JavaDocumentAction
+  'java-document-widget': JavaDocumentAction,
+  'spark-document-widget': SparkDocumentAction
 }
 
 

+ 6 - 1
apps/oozie/src/oozie/static/oozie/js/workflow-editor.ko.js

@@ -166,7 +166,7 @@ var Node = function (node) {
     });
   }
 
-  if (type == 'hive-document-widget' && typeof self.properties.uuid != "undefined") {
+  if ((type == 'hive-document-widget' || type == 'spark-document-widget') && typeof self.properties.uuid != "undefined") {
     self.properties.uuid.subscribe(function () {
       self.actionParametersFetched(false);
       self.fetch_parameters();
@@ -509,6 +509,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
 
     self.getDocuments('query-hive', self.hiveQueries);
     self.getDocuments('query-java', self.javaQueries);
+    self.getDocuments('query-spark2', self.sparkApps);
   };
 
   self.getDocuments = function(type, destination) {
@@ -553,12 +554,15 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
   self.subworkflows = ko.observableArray(getOtherSubworkflows(self, subworkflows_json));
   self.hiveQueries = ko.observableArray();
   self.javaQueries = ko.observableArray();
+  self.sparkApps = ko.observableArray();
   self.history = ko.mapping.fromJS(history_json);
 
   self.getDocumentById = function (type, uuid) {
     var _query = null;
     if (type.indexOf('java') != -1) {
       data = self.javaQueries();
+    } else if (type.indexOf('spark') != -1) {
+      data = self.sparkApps();
     } else {
       data = self.hiveQueries();
     }
@@ -1284,6 +1288,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
   self.draggableGenericAction = ko.observable(bareWidgetBuilder("Generic", "generic-widget"));
   self.draggableHiveDocumentAction = ko.observable(bareWidgetBuilder("Hive", "hive-document-widget"));
   self.draggableJavaDocumentAction = ko.observable(bareWidgetBuilder("Java", "java-document-widget"));
+  self.draggableSparkDocumentAction = ko.observable(bareWidgetBuilder("Spark", "spark-document-widget"));
   self.draggableKillNode = ko.observable(bareWidgetBuilder("Kill", "kill-widget"));
 };
 

+ 77 - 0
apps/oozie/src/oozie/templates/editor2/common_workflow.mako

@@ -898,6 +898,83 @@
 </script>
 
 
+<script type="text/html" id="spark-document-widget">
+  <!-- ko if: $root.workflow.getNodeById(id()) -->
+  <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">
+
+    <div data-bind="visible: ! $root.isEditing()">
+      <span data-bind="template: { name: 'logs-icon' }"></span>
+      <!-- ko if: $root.getDocumentById('spark2', properties.uuid()) -->
+      <!-- ko with: $root.getDocumentById('spark2', properties.uuid()) -->
+        <a data-bind="attr: { href: absoluteUrl() }" target="_blank"><span data-bind='text: name'></span></a>
+        <br/>
+        <span data-bind='text: description' class="muted"></span>
+      <!-- /ko -->
+      <!-- /ko -->
+    </div>
+
+    <div data-bind="visible: $root.isEditing">
+      <div data-bind="visible: ! $parent.ooziePropertiesExpanded()" class="nowrap">
+        <!-- ko if: $root.getDocumentById(type(), properties.uuid()) -->
+        <!-- ko with: $root.getDocumentById(type(), properties.uuid()) -->
+          <select data-bind="options: $root.sparkApps, optionsText: 'name', optionsValue: 'uuid', value: $parent.properties.uuid, select2Version4:{ placeholder: '${ _ko('Java program name...')}'}"></select>
+          <a href="#" data-bind="attr: { href: absoluteUrl() }" target="_blank" title="${ _('Open') }">
+            <i class="fa fa-external-link-square"></i>
+          </a>
+          <div data-bind='text: description' style="padding: 3px; margin-top: 2px" class="muted"></div>
+        <!-- /ko -->
+        <!-- /ko -->
+
+        <div class="span6" data-bind="template: { name: 'common-properties-parameters' }"></div>
+      </div>
+    </div>
+
+    <div data-bind="visible: $parent.ooziePropertiesExpanded">
+      <ul class="nav nav-tabs">
+        <li class="active"><a data-bind="attr: { href: '#properties-' + id()}" data-toggle="tab">${ _('Properties') }</a></li>
+        <li><a data-bind="attr: { href: '#sla-' + id()}" href="#sla" data-toggle="tab">${ _('SLA') }</a></li>
+        <li><a data-bind="attr: { href: '#credentials-' + id()}" data-toggle="tab">${ _('Credentials') }</a></li>
+        <li><a data-bind="attr: { href: '#transitions-' + id()}" data-toggle="tab">${ _('Transitions') }</a></li>
+      </ul>
+      <div class="tab-content">
+        <div class="tab-pane active" data-bind="attr: { id: 'properties-' + id() }">
+          <div class="airy">
+            <span class="widget-label" data-bind="text: $root.workflow_properties.spark_master.label"></span>
+            <input type="text" class="input-medium" data-bind="value: properties.spark_master, attr: { placeholder: $root.workflow_properties.spark_master.help_text }" />
+          </div>
+
+          <div class="airy">
+            <span class="widget-label" data-bind="text: $root.workflow_properties.mode.label"></span>
+            <input type="text" class="input-medium" data-bind="value: properties.mode, attr: { placeholder: $root.workflow_properties.mode.help_text }" />
+          </div>
+
+          <div class="airy">
+            <span class="widget-label" data-bind="text: $root.workflow_properties.app_name.label"></span>
+            <input type="text" class="input-xlarge seventy" data-bind="value: properties.app_name, attr: { placeholder: $root.workflow_properties.app_name.help_text }" />
+          </div>
+
+          <br/>
+          <span data-bind="template: { name: 'common-action-properties' }"></span>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'sla-' + id() }">
+          <span data-bind="template: { name: 'common-action-sla' }"></span>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'credentials-' + id() }">
+          <span data-bind="template: { name: 'common-action-credentials' }"></span>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'transitions-' + id() }">
+          <span data-bind="template: { name: 'common-action-transition' }"></span>
+        </div>
+      </div>
+    </div>
+  </div>
+  <!-- /ko -->
+</script>
+
+
 <script type="text/html" id="generic-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">

+ 1 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-spark-document.xml.mako

@@ -0,0 +1 @@
+workflow-spark.xml.mako

+ 17 - 3
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -14,9 +14,12 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 <%!
+from django.utils.translation import ugettext as _
+
 from desktop.views import commonheader, commonfooter, commonshare, _ko
 from desktop import conf
-from django.utils.translation import ugettext as _
+
+from oozie.conf import ENABLE_DOCUMENT_ACTION
 %>
 
 <%namespace name="dashboard" file="/common_dashboard.mako" />
@@ -25,6 +28,7 @@ from django.utils.translation import ugettext as _
 <%namespace name="layout" file="../navigation-bar.mako" />
 
 ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
+
 <div id="editor">
 
 <%def name="buttons()">
@@ -115,6 +119,7 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons) }
          <a class="draggable-icon"><img src="${ static('oozie/art/icon_beeswax_48.png') }" class="app-icon"><sup style="color: #338bb8; margin-left: -4px; top: -14px; font-size: 12px">2</sup></a>
     </div>
 
+    % if ENABLE_DOCUMENT_ACTION.get():
     <div data-bind="css: { 'draggable-widget': true },
                     draggable: {data: draggableJavaDocumentAction(), isEnabled: true,
                     options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggableJavaDocumentAction());}}}"
@@ -122,6 +127,15 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons) }
          <a class="draggable-icon"><i class="fa fa-file-code-o"></i></a>
     </div>
 
+    <div data-bind="css: { 'draggable-widget': true },
+                    draggable: {data: draggableSparkDocumentAction(), isEnabled: true,
+                    options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggableSparkDocumentAction());}}}"
+         title="${_('Saved Spark program')}" rel="tooltip" data-placement="top">
+         <a class="draggable-icon"><img src="${ static('oozie/art/icon_spark_48.png') }" class="app-icon"></a>
+    </div>
+
+    % endif
+
     <div class="toolbar-label">${ _('ACTIONS') }</div>
 
     <div data-bind="css: { 'draggable-widget': true },
@@ -276,8 +290,8 @@ ${ workflow.render() }
           <!-- ko if: type() == 'workflow' -->
           <select data-bind="options: $root.subworkflows, optionsText: 'name', optionsValue: 'value', value: value"></select>
           <!-- /ko -->
-          <!-- ko if: type() == 'hive' || type() == 'java' -->
-          <select data-bind="options: type() == 'java' ? $root.javaQueries() : $root.hiveQueries(), optionsText: 'name', optionsValue: 'uuid', value: value, select2Version4:{ placeholder: '${ _ko('Document name...')}'}"></select>
+          <!-- ko if: type() == 'hive' || type() == 'java' || type() == 'spark' -->
+          <select data-bind="options: type() == 'java' ? $root.javaQueries() : (type() == 'spark' ? $root.sparkApps() : $root.hiveQueries()), optionsText: 'name', optionsValue: 'uuid', value: value, select2Version4:{ placeholder: '${ _ko('Document name...')}'}"></select>
           <!-- ko if: $root.getDocumentById(type(), value()) -->
             <!-- ko with: $root.getDocumentById(type(), value()) -->
               <a href="#" data-bind="attr: { href: $data.absoluteUrl() }" target="_blank" title="${ _('Open') }">

+ 5 - 1
apps/oozie/src/oozie/views/editor2.py

@@ -303,6 +303,10 @@ def action_parameters(request):
     elif node_data['type'] == 'hive-document':
       notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
       parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
+    elif node_data['type'] == 'spark-document':
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
+      for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']:
+        parameters = parameters.union(set(find_dollar_braced_variables(arg)))
 
     response['status'] = 0
     response['parameters'] = list(parameters)
@@ -384,7 +388,7 @@ def gen_xml_workflow(request):
 @check_editor_access_permission
 @check_document_access_permission()
 def submit_workflow(request, doc_id):
-  workflow = Workflow(document=Document2.objects.get(id=doc_id))
+  workflow = Workflow(document=Document2.objects.get(id=doc_id), user=request.user)
 
   return _submit_workflow_helper(request, workflow, submit_action=reverse('oozie:editor_submit_workflow', kwargs={'doc_id': workflow.id}))
 

+ 4 - 3
desktop/conf.dist/hue.ini

@@ -612,9 +612,6 @@
   ## Flag to enable the creation of a coordinator for the current SQL query.
   # enable_query_scheduling=true
 
-  ## Flag to enable the Java document in editor and workflow.
-  # enable_java_document=true
-
   ## Base URL to Remote GitHub Server
   # github_remote_url=https://github.com
 
@@ -990,6 +987,9 @@
   # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
   ## enable_cron_scheduling=true
 
+  ## Flag to enable the saved Editor queries to be dragged and dropped into a workflow.
+  # enable_document_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app
@@ -1142,6 +1142,7 @@
   # Offset in bytes where a negative offset will fetch the last N bytes for the given log file (default 1MB).
   ## log_offset=-1000000
 
+
 ###########################################################################
 # Settings to configure Sentry / Security App.
 ###########################################################################

+ 4 - 5
desktop/conf/pseudo-distributed.ini.tmpl

@@ -620,9 +620,6 @@
   ## Flag to enable the creation of a coordinator for the current SQL query.
   # enable_query_scheduling=true
 
-  ## Flag to enable the Java document in editor and workflow.
-  # enable_java_document=true
-
   ## Base URL to Remote GitHub Server
   # github_remote_url=https://github.com
 
@@ -998,8 +995,9 @@
   # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
   ## enable_cron_scheduling=true
 
-  # Offset in bytes where a negative offset will fetch the last N bytes for the given log file (default 1MB).
-  ## log_offset=-1000000
+  ## Flag to enable the saved Editor queries to be dragged and dropped into a workflow.
+  # enable_document_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app
@@ -1152,6 +1150,7 @@
   # Offset in bytes where a negative offset will fetch the last N bytes for the given log file (default 1MB).
   ## log_offset=-1000000
 
+
 ###########################################################################
 # Settings to configure Sentry / Security App.
 ###########################################################################

+ 0 - 7
desktop/libs/notebook/src/notebook/conf.py

@@ -109,13 +109,6 @@ ENABLE_BATCH_EXECUTE = Config(
   dynamic_default=is_oozie_enabled
 )
 
-ENABLE_JAVA_DOCUMENT = Config(
-  key="enable_java_document",
-  help=_t("Flag to enable the Java document in editor and workflow."),
-  type=bool,
-  dynamic_default=is_oozie_enabled
-)
-
 
 GITHUB_REMOTE_URL = Config(
     key="github_remote_url",