فهرست منبع

HUE-3226 [connector] MR snippet

Romain Rigaux 9 سال پیش
والد
کامیت
08e8e73a8e

+ 132 - 2
apps/oozie/src/oozie/models2.py

@@ -785,7 +785,7 @@ class Node():
       for prop in action['properties']['hadoopProperties']:
         name, value = prop.split('=', 1)
         job_properties.append({'name': name, 'value': value})
-        self.data['properties']['job_properties'] = job_properties
+      self.data['properties']['job_properties'] = job_properties
       self.data['properties']['files'] = [{'value': prop} for prop in action['properties']['parameters']]
 
     elif self.data['type'] == SqoopDocumentAction.TYPE:
@@ -817,6 +817,24 @@ class Node():
       self.data['properties']['files'] = [{'value': action['properties']['command_path']}] + [{'value': prop} for prop in action['properties']['files']]
       self.data['properties']['archives'] = [{'value': prop} for prop in action['properties']['archives']]
 
+    elif self.data['type'] == MapReduceDocumentAction.TYPE:
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=self.data['properties']['uuid']))
+      action = notebook.get_data()['snippets'][0]
+
+      name = '%s-%s' % (self.data['type'].split('-')[0], self.data['id'][:4])
+      self.data['properties']['app_jar'] = action['properties']['app_jar']
+      self.data['properties']['arguments'] = []
+      self.data['properties']['parameters'] = []
+
+      job_properties = []
+      for prop in action['properties']['hadoopProperties']:
+        name, value = prop.split('=', 1)
+        job_properties.append({'name': name, 'value': value})
+      self.data['properties']['job_properties'] = job_properties
+
+      self.data['properties']['files'] = [{'value': prop} for prop in action['properties']['files']]
+      self.data['properties']['archives'] = [{'value': prop} for prop in action['properties']['archives']]
+
     data = {
       'node': self.data,
       'mapping': mapping,
@@ -2565,6 +2583,80 @@ class ShellDocumentAction(Action):
     return [cls.FIELDS['uuid']]
 
 
+class MapReduceDocumentAction(Action):
+  TYPE = 'mapreduce-document'
+  FIELDS = {
+    'uuid': {
+        'name': 'uuid',
+        'label': _('MapReduce program'),
+        'value': '',
+        'help_text': _('Select a saved MapReduce program you want to schedule.'),
+        'type': 'mapreduce-doc'
+     },
+     'parameters': {
+          'name': 'parameters',
+          'label': _('Parameters'),
+          'value': [],
+          'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}')  % {'type': TYPE.title()},
+          'type': ''
+     },
+     # Common
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Files put in the running directory.'),
+          'type': ''
+     },
+     'archives': {
+          'name': 'archives',
+          'label': _('Archives'),
+          'value': [],
+          'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
+          'type': ''
+     },
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production'),
+          'type': ''
+     },
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.'),
+          'type': ''
+     },
+     'job_xml': {
+          'name': 'job_xml',
+          'label': _('Job XML'),
+          'value': '',
+          'help_text': _('Refer to a Hadoop JobConf job.xml'),
+          'type': ''
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
+
+
 class DecisionNode(Action):
   TYPE = 'decision'
   FIELDS = {}
@@ -2602,7 +2694,8 @@ NODES = {
   'pig-document-widget': PigDocumentAction,
   'sqoop-document-widget': SqoopDocumentAction,
   'distcp-document-widget': DistCpDocumentAction,
-  'shell-document-widget': ShellDocumentAction
+  'shell-document-widget': ShellDocumentAction,
+  'mapreduce-document-widget': MapReduceDocumentAction
 }
 
 
@@ -3416,6 +3509,8 @@ class WorkflowBuilder():
         node = self.get_distcp_document_node(document, user)
       elif document.type == 'query-shell':
         node = self.get_shell_document_node(document, user)
+      elif document.type == 'query-mapreduce':
+        node = self.get_mapreduce_document_node(document, user)
       else:
         raise PopupException(_('Snippet type %s is not supported in batch execution.') % document.type)
 
@@ -3678,6 +3773,41 @@ class WorkflowBuilder():
         "actionParametersFetched": False
     }
 
+  def get_mapreduce_document_node(self, document, user):
+    node = self._get_mapreduce_node(document.uuid, is_document_node=True)
+
+    node['properties']['uuid'] = document.uuid
+
+    return node
+
+  def _get_mapreduce_node(self, node_id, credentials=None, is_document_node=False):
+    if credentials is None:
+      credentials = []
+
+    return {
+        "id": node_id,
+        'name': 'mapreduce-%s' % node_id[:4],
+        "type": "mapreduce-document-widget",
+        "properties":{
+              "jar_path": "",
+              "arguments": [],
+              "java_opts": [],
+              "retry_max": [],
+              "retry_interval": [],
+              "job_properties": [],
+              "prepares": [],
+              "credentials": credentials,
+              "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
+              "archives": []
+        },
+        "children": [
+            {"to": "33430f0f-ebfa-c3ec-f237-3e77efa03d0a"},
+            {"error": "17c9c895-5a16-7443-bb81-f34b30b21548"}
+        ],
+        "actionParameters": [],
+        "actionParametersFetched": False
+    }
+
   def get_pig_document_node(self, document, user):
     node = self._get_pig_node(document.uuid, is_document_node=True)
 

+ 1 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-mapreduce-document.xml.mako

@@ -0,0 +1 @@
+workflow-mapreduce.xml.mako

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -708,6 +708,10 @@
       name=Spark
       interface=oozie
 
+    [[[mapreduce]]]
+      name=MapReduce
+      interface=oozie
+
     [[[sqoop1]]]
       name=Sqoop1
       interface=oozie

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -712,6 +712,10 @@
       name=Spark
       interface=oozie
 
+    [[[mapreduce]]]
+      name=MapReduce
+      interface=oozie
+
     [[[sqoop1]]]
       name=Sqoop1
       interface=oozie

+ 2 - 2
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -209,8 +209,8 @@ class Submission(object):
           if statements is not None:
             self._create_file(deployment_dir, action.data['name'] + '.sql', statements)
 
-        elif action.data['type'] == 'java-document' or action.data['type'] == 'java':
-          if action.data['type'] == 'java-document':
+        elif action.data['type'] in ('java-document', 'java', 'mapreduce-document'):
+          if action.data['type'] == 'java-document' or action.data['type'] == 'mapreduce-document':
             from notebook.models import Notebook
             notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
             properties = notebook.get_data()['snippets'][0]['properties']

+ 4 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -184,6 +184,10 @@ def _default_interpreters():
           'name': 'Spark', 'interface': 'oozie', 'options': {}
       })
       ,
+      ('mapreduce', {
+          'name': 'MapReduce', 'interface': 'oozie', 'options': {}
+      })
+      ,
       ('shell', {
           'name': 'Shell', 'interface': 'oozie', 'options': {}
       })

+ 6 - 0
desktop/libs/notebook/src/notebook/connectors/oozie_batch.py

@@ -47,6 +47,7 @@ class OozieApi(Api):
   LOG_END_PATTERN = '<<< Invocation of Main class completed <<<'
   RESULTS_PATTERN = "(?P<results>>>> Invoking Beeline command line now >>>.+<<< Invocation of Beeline command completed <<<)"
   RESULTS_PATTERN_GENERIC = "(?P<results>>>> Invoking Main class now >>>.+<<< Invocation of Main class completed <<<)"
+  RESULTS_PATTERN_MAPREDUCE = "(?P<results>.+)"
   RESULTS_PATTERN_PIG = "(?P<results>>>> Invoking Pig command line now >>>.+<<< Invocation of Pig command completed <<<)"
 
   def __init__(self, *args, **kwargs):
@@ -97,6 +98,8 @@ class OozieApi(Api):
         results = self._get_results(log_output, snippet['type'])
         if results:
           response['status'] = 'available'
+        else:
+          LOG.warn('No log result could be matched for %s' % job_id)
       else:
         response['status'] = 'failed'
 
@@ -197,10 +200,13 @@ class OozieApi(Api):
       pattern = self.RESULTS_PATTERN
     elif action_type == 'pig':
       pattern = self.RESULTS_PATTERN_PIG
+    elif action_type == 'mapreduce':
+      pattern = self.RESULTS_PATTERN_MAPREDUCE
     else:
       pattern = self.RESULTS_PATTERN_GENERIC
 
     re_results = re.compile(pattern, re.M | re.DOTALL)
     if re_results.search(log_output):
       results = re.search(re_results, log_output).group('results').strip()
+
     return results

+ 16 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -178,9 +178,23 @@
       properties['files'] = [];
     }
 
-    if (snippetType == 'java' || snippetType == 'shell') {
+    if (snippetType == 'java') {
       properties['archives'] = [];
       properties['files'] = [];
+      properties['capture_output'] = false;
+    }
+
+    if (snippetType == 'shell') {
+      properties['archives'] = [];
+      properties['files'] = [];
+    }
+
+    if (snippetType == 'mapreduce') {
+      properties['app_jar'] = '';
+      properties['hadoopProperties'] = [];
+      properties['jars'] = [];
+      properties['files'] = [];
+      properties['archives'] = [];
     }
 
     if (snippetType == 'spark2') {
@@ -720,6 +734,7 @@
         (['jar', 'java'].indexOf(self.type()) != -1 && (self.properties().app_jar() != '' && self.properties().class() != '')) ||
         (['spark2'].indexOf(self.type()) != -1 && self.properties().jars().length > 0) ||
         (['shell'].indexOf(self.type()) != -1 && self.properties().command_path().length > 0) ||
+        (['mapreduce'].indexOf(self.type()) != -1 && self.properties().app_jar().length > 0) ||
         (['distcp'].indexOf(self.type()) != -1 && self.properties().source_path().length > 0 && self.properties().destination_path().length > 0);
     });
     self.lastExecuted = ko.observable(typeof snippet.lastExecuted != "undefined" && snippet.lastExecuted != null ? snippet.lastExecuted : 0);

+ 24 - 6
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -315,11 +315,15 @@ ${ hueIcons.symbols() }
                 <i class="fa fa-terminal app-icon" style="vertical-align: middle"></i>
                 Shell
               <!-- /ko -->
+              <!-- ko if: editorType() == 'mapreduce' -->
+                <i class="fa fa-file-archive-o app-icon" style="vertical-align: middle"></i>
+                MapReduce
+              <!-- /ko -->
               <!-- ko if: editorType() == 'beeswax' || editorType() == 'hive' -->
                 <img src="${ static('beeswax/art/icon_beeswax_48.png') }" class="app-icon" />
                 Hive
               <!-- /ko -->
-              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1', 'distcp', 'shell'].indexOf(editorType()) == -1 -->
+              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1', 'distcp', 'shell', 'mapreduce'].indexOf(editorType()) == -1 -->
                 <img src="${ static('rdbms/art/icon_rdbms_48.png') }" class="app-icon" />
                 SQL
               <!-- /ko -->
@@ -1431,24 +1435,24 @@ ${ hueIcons.symbols() }
                 <!-- ko template: { if: $root.editorMode(), name: 'editor-snippet-header' } --><!-- /ko -->
                 <!-- ko template: { if: ! $root.editorMode(), name: 'notebook-snippet-header' } --><!-- /ko -->
               </h2>
-              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'distcp', 'shell', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['text', 'jar', 'java', 'spark2', 'distcp', 'shell', 'mapreduce', 'py', 'markdown'].indexOf(type()) == -1, name: 'code-editor-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'text', name: 'text-snippet-body' } --><!-- /ko -->
               <!-- ko template: { if: type() == 'markdown', name: 'markdown-snippet-body' } --><!-- /ko -->
-              <!-- ko template: { if: ['java', 'distcp', 'shell', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
+              <!-- ko template: { if: ['java', 'distcp', 'shell', 'mapreduce', 'jar', 'py', 'spark2'].indexOf(type()) != -1, name: 'executable-snippet-body' } --><!-- /ko -->
             </div>
             <div style="position: absolute; top:25px; margin-left:35px; width: calc(100% - 35px)" data-bind="style: { 'z-index': 400 - $index() }">
               <!-- ko template: 'snippet-settings' --><!-- /ko -->
             </div>
           </div>
           <!-- ko template: { if: ['text', 'markdown'].indexOf(type()) == -1, name: 'snippet-execution-status' } --><!-- /ko -->
-          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2', 'distcp', 'shell'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
+          <!-- ko template: { if: $root.editorMode() && ['java', 'spark2', 'distcp', 'shell', 'mapreduce'].indexOf(type()) == -1, name: 'snippet-code-resizer' } --><!-- /ko -->
           <!-- ko if: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
           <!-- ko template: 'query-tabs' --><!-- /ko -->
           <!-- /ko -->
           <!-- ko ifnot: $root.editorMode() -->
           <!-- ko template: 'snippet-log' --><!-- /ko -->
-          <!-- ko template: { if: ['text', 'jar', 'java', 'distcp', 'shell', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
+          <!-- ko template: { if: ['text', 'jar', 'java', 'distcp', 'shell', 'mapreduce', 'py', 'markdown'].indexOf(type()) == -1, name: 'snippet-results' } --><!-- /ko -->
           <!-- /ko -->
 
           <div class="clearfix"></div>
@@ -1826,7 +1830,21 @@ ${ hueIcons.symbols() }
       <div class="control-group">
         <label class="control-label">${_('Environment variables')}</label>
         <div class="controls">
-          <!-- ko template: { if: typeof properties().env_var != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Arguments') }', value: properties().env_var, title: '${ _ko('Environment variable for the scritp') }', placeholder: '${ _ko('e.g. MAX=10, PATH=$PATH:/user/path') }'}} --><!-- /ko -->
+          <!-- ko template: { if: typeof properties().env_var != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Arguments') }', value: properties().env_var, title: '${ _ko('Environment variable for the script') }', placeholder: '${ _ko('e.g. MAX=10, PATH=$PATH:/user/path') }'}} --><!-- /ko -->
+        </div>
+      </div>
+      <!-- /ko -->
+      <!-- ko if: type() == 'mapreduce' -->
+      <div class="control-group">
+        <label class="control-label">${_('Jar path')}</label>
+        <div class="controls">
+          <input type="text" class="input-xxlarge filechooser-input" data-bind="value: properties().app_jar, valueUpdate: 'afterkeydown', filechooser: properties().app_jar" placeholder="${ _('Source path to the main MapReduce jar') }"/>
+        </div>
+      </div>
+      <div class="control-group">
+        <label class="control-label">${_('Hadoop properties')}</label>
+        <div class="controls">
+          <!-- ko template: { if: typeof properties().hadoopProperties != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Properties') }', value: properties().hadoopProperties, title: '${ _ko('Name and values of Hadoop properties') }', placeholder: '${ _ko('e.g. mapred.job.queue.name=production, mapred.map.tasks.speculative.execution=false') }'}} --><!-- /ko -->
         </div>
       </div>
       <!-- /ko -->