Pārlūkot izejas kodu

HUE-4797 [connector] DistCp snippet

Romain Rigaux 9 gadi atpakaļ
vecāks
revīzija
aac4a0d

+ 105 - 1
apps/oozie/src/oozie/models2.py

@@ -796,6 +796,13 @@ class Node():
       self.data['properties']['command'] = action['statement']
       self.data['properties']['arguments'] = []
 
+    elif self.data['type'] == DistCpDocumentAction.TYPE:
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=self.data['properties']['uuid']))
+      action = notebook.get_data()['snippets'][0]
+
+      name = '%s-%s' % (self.data['type'].split('-')[0], self.data['id'][:4])
+      self.data['properties']['source_path'] = action['properties']['source_path']
+      self.data['properties']['destination_path'] = action['properties']['destination_path']
 
     data = {
       'node': self.data,
@@ -2404,6 +2411,63 @@ class SqoopDocumentAction(Action):
     return [cls.FIELDS['uuid']]
 
 
+class DistCpDocumentAction(Action):
+  TYPE = 'distcp-document'
+  FIELDS = {
+    'uuid': {
+        'name': 'uuid',
+        'label': _('DistCp program'),
+        'value': '',
+        'help_text': _('Select a saved DistCp program you want to schedule.'),
+        'type': 'distcp'
+     },
+     'parameters': {
+          'name': 'parameters',
+          'label': _('Parameters'),
+          'value': [],
+          'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}')  % {'type': TYPE.title()},
+          'type': ''
+     },
+      # Common
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.')
+     },
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production')
+     },
+     'java_opts': {
+          'name': 'java_opts',
+          'label': _('Java options'),
+          'value': '',
+          'help_text': _('Parameters for the JVM, e.g. -Dprop1=a -Dprop2=b')
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
+
+
 class DecisionNode(Action):
   TYPE = 'decision'
   FIELDS = {}
@@ -2439,7 +2503,8 @@ NODES = {
   'java-document-widget': JavaDocumentAction,
   'spark-document-widget': SparkDocumentAction,
   'pig-document-widget': PigDocumentAction,
-  'sqoop-document-widget': SqoopDocumentAction
+  'sqoop-document-widget': SqoopDocumentAction,
+  'distcp-document-widget': DistCpDocumentAction
 }
 
 
@@ -3249,6 +3314,8 @@ class WorkflowBuilder():
         node = self.get_pig_document_node(document, user)
       elif document.type == 'query-sqoop1':
         node = self.get_sqoop_document_node(document, user)
+      elif document.type == 'query-distcp':
+        node = self.get_distcp_document_node(document, user)
       else:
         raise PopupException(_('Snippet type %s is not supported in batch execution.') % document.type)
 
@@ -3437,6 +3504,43 @@ class WorkflowBuilder():
         "actionParametersFetched": False
     }
 
+  def get_distcp_document_node(self, document, user):
+    node = self._get_distcp_node(document.uuid, is_document_node=True)
+
+    node['properties']['uuid'] = document.uuid
+
+    return node
+
+  def _get_distcp_node(self, node_id, credentials=None, is_document_node=False):
+    if credentials is None:
+      credentials = []
+
+    return {
+        "id": node_id,
+        'name': 'distcp-%s' % node_id[:4],
+        "type": "distcp-document-widget",
+        "properties":{
+              "source_path": "",
+              "destination_path": "",
+              "arguments": [],
+              "java_opts": [],
+              "retry_max": [],
+              "retry_interval": [],
+              "job_properties": [],
+              "capture_output": False,
+              "prepares": [],
+              "credentials": credentials,
+              "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
+              "archives": []
+        },
+        "children": [
+            {"to": "33430f0f-ebfa-c3ec-f237-3e77efa03d0a"},
+            {"error": "17c9c895-5a16-7443-bb81-f34b30b21548"}
+        ],
+        "actionParameters": [],
+        "actionParametersFetched": False
+    }
+
   def get_pig_document_node(self, document, user):
     node = self._get_pig_node(document.uuid, is_document_node=True)
 

+ 42 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-distcp-document.xml.mako

@@ -0,0 +1,42 @@
+## -*- coding: utf-8 -*-
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+  from oozie.utils import smart_path
+%>
+
+<%namespace name="common" file="workflow-common.xml.mako" />
+
+    <action name="${ node['name'] }"${ common.credentials(node['properties']['credentials']) }${ common.retry_max(node['properties']['retry_max']) }${ common.retry_interval(node['properties']['retry_interval']) }>
+        <distcp xmlns="uri:oozie:distcp-action:0.1">
+            <job-tracker>${'${'}jobTracker}</job-tracker>
+            <name-node>${'${'}nameNode}</name-node>
+
+            ${ common.prepares(node['properties']['prepares']) }
+            ${ common.configuration(node['properties']['job_properties']) }
+
+            % if node['properties']['java_opts']:
+              <java-opts>${ node['properties']['java_opts'] }</java-opts>
+            % endif
+
+            <arg>${ smart_path(node['properties']['source_path']) }</arg>
+            <arg>${ smart_path(node['properties']['destination_path']) }</arg>
+        </distcp>
+        <ok to="${ node_mapping[node['children'][0]['to']].name }"/>
+        <error to="${ node_mapping[node['children'][1]['error']].name }"/>
+        ${ common.sla(node) }
+    </action>

+ 4 - 1
apps/oozie/src/oozie/utils.py

@@ -97,11 +97,14 @@ def workflow_to_dict(workflow):
   return workflow_dict
 
 
-def smart_path(path, mapping, is_coordinator=False):
+def smart_path(path, mapping=None, is_coordinator=False):
   # Try to prepend home_dir and FS scheme if needed.
   # If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
   # This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
   # The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
+  if mapping is None:
+    mapping = {}
+
   if not path.startswith('$') and not path.startswith('/') and not urlparse.urlsplit(path).scheme:
     path = '/user/%(username)s/%(path)s' % {
         'username': '${coord:user()}' if is_coordinator else '${wf:user()}',

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -712,6 +712,10 @@
       name=Sqoop1
       interface=oozie
 
+    [[[distcp]]]
+      name=Distcp
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -716,6 +716,10 @@
       name=Sqoop1
       interface=oozie
 
+    [[[distcp]]]
+      name=Distcp
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -176,6 +176,10 @@ def _default_interpreters():
           'name': 'Sqoop 1', 'interface': 'oozie', 'options': {}
       })
       ,
+      ('distcp', {
+          'name': 'Distcp', 'interface': 'oozie', 'options': {}
+      })
+      ,
       ('spark2', {
           'name': 'Spark', 'interface': 'oozie', 'options': {}
       })

+ 7 - 2
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -197,6 +197,10 @@
       properties['class'] = '';
       properties['arguments'] = [];
     }
+    else if (snippetType == 'distcp') {
+      properties['source_path'] = '';
+      properties['destination_path'] = '';
+    }
     else if (snippetType == 'py') {
       properties['py_file'] = '';
       properties['arguments'] = [];
@@ -706,9 +710,10 @@
 
     self.wasBatchExecuted = ko.observable(typeof snippet.wasBatchExecuted != "undefined" && snippet.wasBatchExecuted != null ? snippet.wasBatchExecuted : false);
     self.isReady = ko.computed(function() {
-      return (['jar', 'java', 'spark2'].indexOf(self.type()) == -1 && self.statement() !== '') ||
+      return (['jar', 'java', 'spark2', 'distcp'].indexOf(self.type()) == -1 && self.statement() !== '') ||
         (['jar', 'java'].indexOf(self.type()) != -1 && (self.properties().app_jar() != '' && self.properties().class() != '')) ||
-        (['spark2'].indexOf(self.type()) != -1 && self.properties().jars().length > 0);
+        (['spark2'].indexOf(self.type()) != -1 && self.properties().jars().length > 0) ||
+        (['distcp'].indexOf(self.type()) != -1 && self.properties().source_path().length > 0 && self.properties().destination_path().length > 0);
     });
     self.lastExecuted = ko.observable(typeof snippet.lastExecuted != "undefined" && snippet.lastExecuted != null ? snippet.lastExecuted : 0);
 

+ 23 - 5
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -307,11 +307,15 @@ ${ hueIcons.symbols() }
                 <img src="${ static('oozie/art/icon_sqoop_48.png') }" class="app-icon" />
                 Sqoop 1
               <!-- /ko -->
+              <!-- ko if: editorType() == 'distcp' -->
+                <i class="fa fa-files-o app-icon" style="vertical-align: middle"></i>
+                DistCp
+              <!-- /ko -->
               <!-- ko if: editorType() == 'beeswax' || editorType() == 'hive' -->
                 <img src="${ static('beeswax/art/icon_beeswax_48.png') }" class="app-icon" />
                 Hive
               <!-- /ko -->
-              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1'].indexOf(editorType()) == -1 -->
+              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1', 'distcp'].indexOf(editorType()) == -1 -->
                 <img src="${ static('rdbms/art/icon_rdbms_48.png') }" class="app-icon" />
                 SQL
               <!-- /ko -->
@@ -1423,24 +1427,24 @@ ${ hueIcons.symbols() }
                 <!-- ko template: { if: $root.editorMode(), name: 'editor-snippet-header' } --><!-- /ko -->
                 <!-- ko template: { if: ! $root.editorMode(), name: 'notebook-snippet-header' } --><!-- /ko -->
               </h2>
-              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'distcp', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'text', name: 'text-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'markdown', name: 'markdown-snippet-body' } --><!-- /ko -->
-              <!-- ko template: { if: ['java', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['java', 'distcp', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
             </div>
             <div style="position: absolute; top:25px; margin-left:35px; width: calc(100% - 35px)" data-bind="style: { 'z-index': 400 - $index() }">
               <!-- ko template: 'snippet-settings' --><!-- /ko -->
             </div>
           </div>
           <!-- ko template: { if: ['text', 'markdown'].indexOf(type()) == -1, name: 'snippet-execution-status' } --><!-- /ko -->
-          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
+          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2', 'distcp'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
           <!-- ko if: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
           <!-- ko template: 'query-tabs' --><!-- /ko -->
           <!-- /ko -->
           <!-- ko ifnot: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
-          <!-- ko template: { if: ['text', 'jar', 'java', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
+          <!-- ko template: { if: ['text', 'jar', 'java', 'distcp', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
           <!-- /ko -->
 
           <div class="clearfix"></div>
@@ -1792,6 +1796,20 @@ ${ hueIcons.symbols() }
 <script type="text/html" id="executable-snippet-body">
   <div style="padding:10px;">
     <form class="form-horizontal">
+      <!-- ko if: type() == 'distcp' -->
+      <div class="control-group">
+        <label class="control-label">${_('Source')}</label>
+        <div class="controls">
+          <input type="text" class="input-xxlarge filechooser-input" data-bind="value: properties().source_path, valueUpdate: 'afterkeydown', filechooser: properties().source_path" placeholder="${ _('Source path to copy, e.g. ${nameNode1}/path/to/input.txt') }"/>
+        </div>
+      </div>
+      <div class="control-group">
+        <label class="control-label">${_('Destination')}</label>
+        <div class="controls">
+          <input type="text" class="input-xxlarge filechooser-input" data-bind="value: properties().destination_path, valueUpdate: 'afterkeydown', filechooser: properties().destination_path" placeholder="${ _('Destination path, e.g. ${nameNode2}/path/to/output.txt') }"/>
+        </div>
+      </div>
+      <!-- /ko -->
       <!-- ko if: type() == 'jar' || type() == 'java' -->
       <div class="control-group">
         <label class="control-label">${_('Path')}</label>