浏览代码

HUE-4750 [connector] Shell snippet

Romain Rigaux 9 年之前
父节点
当前提交
6d524fb

+ 127 - 4
apps/oozie/src/oozie/models2.py

@@ -804,6 +804,18 @@ class Node():
       self.data['properties']['source_path'] = action['properties']['source_path']
       self.data['properties']['destination_path'] = action['properties']['destination_path']
 
+    elif self.data['type'] == ShellDocumentAction.TYPE:
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=self.data['properties']['uuid']))
+      action = notebook.get_data()['snippets'][0]
+
+      name = '%s-%s' % (self.data['type'].split('-')[0], self.data['id'][:4])
+      self.data['properties']['shell_command'] = action['properties']['command_path']
+      self.data['properties']['env_var'] = [{'value': prop} for prop in action['properties']['env_var']]
+      self.data['properties']['capture_output'] = action['properties']['capture_output']
+
+      self.data['properties']['files'] = [{'value': action['properties']['command_path']}] + [{'value': prop} for prop in action['properties']['files']]
+      self.data['properties']['archives'] = [{'value': prop} for prop in action['properties']['archives']]
+
     data = {
       'node': self.data,
       'mapping': mapping,
@@ -2478,6 +2490,80 @@ class DistCpDocumentAction(Action):
     return [cls.FIELDS['uuid']]
 
 
+class ShellDocumentAction(Action):
+  TYPE = 'shell-document'
+  FIELDS = {
+    'uuid': {
+        'name': 'uuid',
+        'label': _('Shell program'),
+        'value': '',
+        'help_text': _('Select a saved Shell program you want to schedule.'),
+        'type': 'shell-doc'
+     },
+     'parameters': {
+          'name': 'parameters',
+          'label': _('Parameters'),
+          'value': [],
+          'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}')  % {'type': TYPE.title()},
+          'type': ''
+     },
+     # Common
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Files put in the running directory.'),
+          'type': ''
+     },
+     'archives': {
+          'name': 'archives',
+          'label': _('Archives'),
+          'value': [],
+          'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
+          'type': ''
+     },
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production'),
+          'type': ''
+     },
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.'),
+          'type': ''
+     },
+     'job_xml': {
+          'name': 'job_xml',
+          'label': _('Job XML'),
+          'value': '',
+          'help_text': _('Refer to a Hadoop JobConf job.xml'),
+          'type': ''
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
+
+
 class DecisionNode(Action):
   TYPE = 'decision'
   FIELDS = {}
@@ -2514,7 +2600,8 @@ NODES = {
   'spark-document-widget': SparkDocumentAction,
   'pig-document-widget': PigDocumentAction,
   'sqoop-document-widget': SqoopDocumentAction,
-  'distcp-document-widget': DistCpDocumentAction
+  'distcp-document-widget': DistCpDocumentAction,
+  'shell-document-widget': ShellDocumentAction
 }
 
 
@@ -3326,6 +3413,8 @@ class WorkflowBuilder():
         node = self.get_sqoop_document_node(document, user)
       elif document.type == 'query-distcp':
         node = self.get_distcp_document_node(document, user)
+      elif document.type == 'query-shell':
+        node = self.get_shell_document_node(document, user)
       else:
         raise PopupException(_('Snippet type %s is not supported in batch execution.') % document.type)
 
@@ -3502,7 +3591,6 @@ class WorkflowBuilder():
               "retry_max": [],
               "retry_interval": [],
               "job_properties": [],
-              "capture_output": False,
               "prepares": [],
               "credentials": credentials,
               "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
@@ -3539,7 +3627,43 @@ class WorkflowBuilder():
               "retry_max": [],
               "retry_interval": [],
               "job_properties": [],
-              "capture_output": False,
+              "prepares": [],
+              "credentials": credentials,
+              "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
+              "archives": []
+        },
+        "children": [
+            {"to": "33430f0f-ebfa-c3ec-f237-3e77efa03d0a"},
+            {"error": "17c9c895-5a16-7443-bb81-f34b30b21548"}
+        ],
+        "actionParameters": [],
+        "actionParametersFetched": False
+    }
+
+  def get_shell_document_node(self, document, user):
+    node = self._get_shell_node(document.uuid, is_document_node=True)
+
+    node['properties']['uuid'] = document.uuid
+
+    return node
+
+  def _get_shell_node(self, node_id, credentials=None, is_document_node=False):
+    if credentials is None:
+      credentials = []
+
+    return {
+        "id": node_id,
+        'name': 'shell-%s' % node_id[:4],
+        "type": "shell-document-widget",
+        "properties":{
+              "command_path": "",
+              "env_var": "",
+              "arguments": [],
+              "java_opts": [],
+              "retry_max": [],
+              "retry_interval": [],
+              "job_properties": [],
+              "capture_output": True,
               "prepares": [],
               "credentials": credentials,
               "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
@@ -3575,7 +3699,6 @@ class WorkflowBuilder():
               "retry_max": [],
               "retry_interval": [],
               "job_properties": [],
-              "capture_output": False,
               "prepares": [],
               "credentials": credentials,
               "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],

+ 1 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-shell-document.xml.mako

@@ -0,0 +1 @@
+workflow-shell.xml.mako

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -716,6 +716,10 @@
       name=Distcp
       interface=oozie
 
+    [[[shell]]]
+      name=Shell
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -720,6 +720,10 @@
       name=Distcp
       interface=oozie
 
+    [[[shell]]]
+      name=Shell
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -184,6 +184,10 @@ def _default_interpreters():
           'name': 'Spark', 'interface': 'oozie', 'options': {}
       })
       ,
+      ('shell', {
+          'name': 'Shell', 'interface': 'oozie', 'options': {}
+      })
+      ,
       ('text', {
           'name': 'Text', 'interface': 'text', 'options': {}
       }),

+ 7 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -178,7 +178,7 @@
       properties['files'] = [];
     }
 
-    if (snippetType == 'java') {
+    if (snippetType == 'java' || snippetType == 'shell') {
       properties['archives'] = [];
       properties['files'] = [];
     }
@@ -201,6 +201,11 @@
       properties['source_path'] = '';
       properties['destination_path'] = '';
     }
+    else if (snippetType == 'shell') {
+      properties['command_path'] = '';
+      properties['env_var'] = [];
+      properties['capture_output'] = true;
+    }
     else if (snippetType == 'py') {
       properties['py_file'] = '';
       properties['arguments'] = [];
@@ -714,6 +719,7 @@
       return (['jar', 'java', 'spark2', 'distcp'].indexOf(self.type()) == -1 && self.statement() !== '') ||
         (['jar', 'java'].indexOf(self.type()) != -1 && (self.properties().app_jar() != '' && self.properties().class() != '')) ||
         (['spark2'].indexOf(self.type()) != -1 && self.properties().jars().length > 0) ||
+        (['shell'].indexOf(self.type()) != -1 && self.properties().command_path().length > 0) ||
         (['distcp'].indexOf(self.type()) != -1 && self.properties().source_path().length > 0 && self.properties().destination_path().length > 0);
     });
     self.lastExecuted = ko.observable(typeof snippet.lastExecuted != "undefined" && snippet.lastExecuted != null ? snippet.lastExecuted : 0);

+ 25 - 5
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -311,11 +311,15 @@ ${ hueIcons.symbols() }
                 <i class="fa fa-files-o app-icon" style="vertical-align: middle"></i>
                 DistCp
               <!-- /ko -->
+              <!-- ko if: editorType() == 'shell' -->
+                <i class="fa fa-terminal app-icon" style="vertical-align: middle"></i>
+                Shell
+              <!-- /ko -->
               <!-- ko if: editorType() == 'beeswax' || editorType() == 'hive' -->
                 <img src="${ static('beeswax/art/icon_beeswax_48.png') }" class="app-icon" />
                 Hive
               <!-- /ko -->
-              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1', 'distcp'].indexOf(editorType()) == -1 -->
+              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1', 'distcp', 'shell'].indexOf(editorType()) == -1 -->
                 <img src="${ static('rdbms/art/icon_rdbms_48.png') }" class="app-icon" />
                 SQL
               <!-- /ko -->
@@ -1427,24 +1431,24 @@ ${ hueIcons.symbols() }
                 <!-- ko template: { if: $root.editorMode(), name: 'editor-snippet-header' } --><!-- /ko -->
                 <!-- ko template: { if: ! $root.editorMode(), name: 'notebook-snippet-header' } --><!-- /ko -->
               </h2>
-              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'distcp', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'distcp', 'shell', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'text', name: 'text-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'markdown', name: 'markdown-snippet-body' } --><!-- /ko -->
-              <!-- ko template: { if: ['java', 'distcp', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['java', 'distcp', 'shell', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
             </div>
             <div style="position: absolute; top:25px; margin-left:35px; width: calc(100% - 35px)" data-bind="style: { 'z-index': 400 - $index() }">
               <!-- ko template: 'snippet-settings' --><!-- /ko -->
             </div>
           </div>
           <!-- ko template: { if: ['text', 'markdown'].indexOf(type()) == -1, name: 'snippet-execution-status' } --><!-- /ko -->
-          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2', 'distcp'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
+          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2', 'distcp', 'shell'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
           <!-- ko if: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
           <!-- ko template: 'query-tabs' --><!-- /ko -->
           <!-- /ko -->
           <!-- ko ifnot: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
-          <!-- ko template: { if: ['text', 'jar', 'java', 'distcp', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
+          <!-- ko template: { if: ['text', 'jar', 'java', 'distcp', 'shell', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
           <!-- /ko -->
 
           <div class="clearfix"></div>
@@ -1474,6 +1478,8 @@ ${ hueIcons.symbols() }
         <!-- ko template: { if: typeof properties().parameters != 'undefined', name: 'property', data: { type: 'csv', label: '${ _ko('Parameters') }', value: properties().parameters, title: '${ _ko('Names and values of Pig parameters and options') }', placeholder: '${ _ko('e.g. input /user/data, -param input=/user/data, -optimizer_off SplitFilter, -verbose') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().hadoopProperties != 'undefined', name: 'property', data: { type: 'csv', label: '${ _ko('Hadoop properties') }', value: properties().hadoopProperties, title: '${ _ko('Name and values of Hadoop properties') }', placeholder: '${ _ko('e.g. mapred.job.queue.name=production, mapred.map.tasks.speculative.execution=false') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().resources != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Resources') }', value: properties().resources, title: '${ _ko('HDFS Files or compressed files') }', placeholder: '${ _ko('e.g. /tmp/file, /tmp.file.zip') }'}} --><!-- /ko -->
+
+        <!-- ko template: { if: typeof properties().capture_output != 'undefined', name: 'property', data: { type: 'boolean', label: '${ _ko('Capture output') }', value: properties().capture_output, title: '${ _ko('If capturing the output of the shell script') }' }} --><!-- /ko -->
       </form>
     </div>
     <a class="pointer demi-modal-chevron" data-bind="click: function() { settingsVisible(! settingsVisible()) }"><i class="fa fa-chevron-up"></i></a>
@@ -1810,6 +1816,20 @@ ${ hueIcons.symbols() }
         </div>
       </div>
       <!-- /ko -->
+      <!-- ko if: type() == 'shell' -->
+      <div class="control-group">
+        <label class="control-label">${_('Script path')}</label>
+        <div class="controls">
+          <input type="text" class="input-xxlarge filechooser-input" data-bind="value: properties().command_path, valueUpdate: 'afterkeydown', filechooser: properties().command_path" placeholder="${ _('Source path to the command') }"/>
+        </div>
+      </div>
+      <div class="control-group">
+        <label class="control-label">${_('Environment variables')}</label>
+        <div class="controls">
+          <!-- ko template: { if: typeof properties().env_var != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Arguments') }', value: properties().env_var, title: '${ _ko('Environment variable for the scritp') }', placeholder: '${ _ko('e.g. MAX=10, PATH=$PATH:/user/path') }'}} --><!-- /ko -->
+        </div>
+      </div>
+      <!-- /ko -->
       <!-- ko if: type() == 'jar' || type() == 'java' -->
       <div class="control-group">
         <label class="control-label">${_('Path')}</label>