Browse Source

HUE-8500 [oozie] Remote cluster action

Romain Rigaux 7 years ago
parent
commit
f60d3f8

+ 7 - 0
apps/oozie/src/oozie/conf.py

@@ -108,6 +108,13 @@ ENABLE_IMPALA_ACTION = Config(
   default=False
 )
 
+ENABLE_ALTUS_ACTION = Config(
+  key="enable_altus_action",
+  help=_t("Flag to enable the Altus action."),
+  type=bool,
+  default=False
+)
+
 
 def config_validator(user):
   res = []

+ 54 - 0
apps/oozie/src/oozie/models2.py

@@ -905,6 +905,19 @@ class Node():
 
       self.data['properties']['files'] = files
       self.data['properties']['archives'] = []
+    elif self.data['type'] == AltusAction.TYPE:
+      shell_command_name = self.data['name'] + '.sh'
+      self.data['properties']['shell_command'] = shell_command_name
+      self.data['properties']['env_var'] = []
+      self.data['properties']['arguments'] = []
+      self.data['properties']['job_properties'] = []
+      self.data['properties']['capture_output'] = True
+
+      files = [{'value': shell_command_name}, {'value': 'altus.py'}]
+
+      self.data['properties']['files'] = files
+      self.data['properties']['archives'] = []
+
 
     data = {
       'node': self.data,
@@ -978,6 +991,8 @@ class Node():
       node_type = JavaAction.TYPE
     elif self.data['type'] == ImpalaAction.TYPE or self.data['type'] == ImpalaDocumentAction.TYPE:
       node_type = ShellAction.TYPE
+    elif self.data['type'] == AltusAction.TYPE:
+      node_type = ShellAction.TYPE
 
     return 'editor2/gen/workflow-%s.xml.mako' % node_type
 
@@ -2131,6 +2146,44 @@ class SparkAction(Action):
     return [cls.FIELDS['files'], cls.FIELDS['jars']]
 
 
+
+class AltusAction(Action):
+  TYPE = 'altus'
+  FIELDS = {
+     'service': {
+          'name': 'service',
+          'label': _('Service'),
+          'value': '',
+          'help_text': _('e.g. dataeng, iam, dataware...'),
+          'type': ''
+     },
+     'command': {
+          'name': 'command',
+          'label': _('Command'),
+          'value': '',
+          'help_text': _('e.g. listClusters, listJobs...'),
+          'type': ''
+     },
+     'parameters': {
+          'name': 'parameters',
+          'label': _('Arguments'),
+          'value': [],
+          'help_text': _('List of parameters provided to the command. e.g. jobId=xxx'),
+     },
+     'capture_output': {
+          'name': 'capture_output',
+          'label': _('Capture output'),
+          'value': True,
+          'help_text': _('Capture output of the stdout of the command execution.'),
+          'type': ''
+     },
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['service'], cls.FIELDS['command']]
+
+
 class KillAction(Action):
   TYPE = 'kill'
   FIELDS = {
@@ -2867,6 +2920,7 @@ NODES = {
   'email-widget': EmailAction,
   'streaming-widget': StreamingAction,
   'distcp-widget': DistCpAction,
+  'altus-widget': AltusAction,
   'kill-widget': KillAction,
   'join-widget': JoinAction,
   'fork-widget': ForkNode,

+ 1 - 0
apps/oozie/src/oozie/static/oozie/js/workflow-editor.ko.js

@@ -1351,6 +1351,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
   self.draggableHiveAction = ko.observable(bareWidgetBuilder("Hive Script", "hive-widget"));
   self.draggableHive2Action = ko.observable(bareWidgetBuilder("HiveServer2 Script", "hive2-widget"));
   self.draggableImpalaAction = ko.observable(bareWidgetBuilder("Impala Script", "impala-widget"));
+  self.draggableAltusAction = ko.observable(bareWidgetBuilder("Altus Command", "altus-widget"));
   self.draggablePigAction = ko.observable(bareWidgetBuilder("Pig Script", "pig-widget"));
   self.draggableJavaAction = ko.observable(bareWidgetBuilder("Java program", "java-widget"));
   self.draggableMapReduceAction = ko.observable(bareWidgetBuilder("MapReduce job", "mapreduce-widget"));

+ 66 - 0
apps/oozie/src/oozie/templates/editor2/common_workflow.mako

@@ -178,6 +178,10 @@
       <img src="${ static('oozie/art/icon_impala_48.png') }" class="widget-icon" alt="${ _('Impala icon') }">
       <!-- /ko -->
 
+      <!-- ko if: widgetType() == 'altus-widget' -->
+      <a class="widget-icon"><i class="fa fa-cloud"></i></a>
+      <!-- /ko -->
+
       <!-- ko if: widgetType() == 'pig-widget' || widgetType() == 'pig-document-widget'  -->
       <img src="${ static('oozie/art/icon_pig_48.png') }" class="widget-icon" alt="${ _('Pig icon') }">
       <!-- /ko -->
@@ -845,6 +849,68 @@
 </script>
 
 
+<script type="text/html" id="altus-widget">
+  <!-- ko if: $root.workflow.getNodeById(id()) -->
+  <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">
+
+    <div data-bind="visible: !$root.isEditing()">
+      <span data-bind="template: { name: 'logs-icon' }"></span>
+      <span data-bind="text: properties.service"></span>
+      <span data-bind="text: properties.command"></span>
+      <span data-bind="text: properties.parameters"></span>
+    </div>
+
+    <div data-bind="visible: $root.isEditing">
+      <div data-bind="visible: ! $parent.ooziePropertiesExpanded()" class="nowrap">
+        <div class="airy">
+          <span class="widget-label" data-bind="text: $root.workflow_properties.service.label"></span>
+          <input type="text" data-bind="value: properties.service, valueUpdate:'afterkeydown', attr: { placeholder:  $root.workflow_properties.service.help_text }" validate="nonempty"/>
+        </div>
+        <div class="airy">
+          <span class="widget-label" data-bind="text: $root.workflow_properties.command.label"></span>
+          <input type="text" data-bind="value: properties.command, valueUpdate:'afterkeydown', attr: { placeholder:  $root.workflow_properties.command.help_text }" validate="nonempty"/>
+        </div>
+        <div class="row-fluid">
+          <div class="span12" data-bind="template: { name: 'common-properties-parameters' }"></div>
+        </div>
+      </div>
+    </div>
+
+    <div data-bind="visible: $parent.ooziePropertiesExpanded">
+      <ul class="nav nav-tabs">
+        <li class="active"><a data-bind="attr: { href: '#properties-' + id()}" data-toggle="tab">${ _('Properties') }</a></li>
+        <li><a data-bind="attr: { href: '#sla-' + id()}" href="#sla" data-toggle="tab">${ _('SLA') }</a></li>
+        <li><a data-bind="attr: { href: '#credentials-' + id()}" data-toggle="tab">${ _('Credentials') }</a></li>
+        <li><a data-bind="attr: { href: '#transitions-' + id()}" data-toggle="tab">${ _('Transitions') }</a></li>
+      </ul>
+      <div class="tab-content">
+        <div class="tab-pane active" data-bind="attr: { id: 'properties-' + id() }">
+          <div class="row-fluid">
+            <span data-bind="text: $root.workflow_properties.capture_output.label"></span>
+            <input type="checkbox" data-bind="checked: properties.capture_output" />
+
+            <div class="span12" data-bind="template: { name: 'common-properties-parameters' }"></div>
+          </div>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'sla-' + id() }">
+          <span data-bind="template: { name: 'common-action-sla' }"></span>
+        </div>
+
+        ##<div class="tab-pane" data-bind="attr: { id: 'credentials-' + id() }">
+        ##  <span data-bind="template: { name: 'common-action-credentials' }"></span>
+        ##</div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'transitions-' + id() }">
+          <span data-bind="template: { name: 'common-action-transition' }"></span>
+        </div>
+      </div>
+    </div>
+  </div>
+  <!-- /ko -->
+</script>
+
+
 <script type="text/html" id="pig-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">

+ 13 - 2
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -19,7 +19,7 @@ from django.utils.translation import ugettext as _
 from desktop import conf
 from desktop.views import commonheader, commonfooter, commonshare, _ko
 
-from oozie.conf import ENABLE_DOCUMENT_ACTION, ENABLE_IMPALA_ACTION
+from oozie.conf import ENABLE_DOCUMENT_ACTION, ENABLE_IMPALA_ACTION, ENABLE_ALTUS_ACTION
 %>
 
 <%namespace name="dashboard" file="/common_dashboard.mako" />
@@ -141,11 +141,22 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons, is_emb
     </ul>
     % endif
   </%def>
+
   <%def name="widgets()">
     % if ENABLE_DOCUMENT_ACTION.get():
+
     <!-- ko if: $root.currentDraggableSection() === 'documents' -->
     <div class="draggable-documents">
 
+    % if ENABLE_ALTUS_ACTION.get():
+    <div data-bind="css: { 'draggable-widget': true },
+                    draggable: {data: draggableAltusAction(), isEnabled: true,
+                    options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggableAltusAction());}}}"
+         title="${_('Altus Command')}" rel="tooltip" data-placement="top">
+         <a class="draggable-icon"><i class="fa fa-cloud"></i></a>
+    </div>
+    % endif
+
     <!-- ko if: $root.availableActions().length == 0 || $root.availableActions().indexOf('hive') != -1 -->
     <div data-bind="css: { 'draggable-widget': true },
                     draggable: {data: draggableHiveDocumentAction(), isEnabled: true,
@@ -418,7 +429,7 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons, is_emb
       <div data-bind="component: { name: 'hue-drop-down', params: { value: compute, entries: availableComputes, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active compute') }' } }"></div>
     <!-- /ko -->
     </span>
-      
+
     <div class="row-fluid">
       %if is_embeddable:
       <div class="span12 margin-top-20">

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1250,6 +1250,9 @@
   # Flag to enable the Impala action.
   ## enable_impala_action=false
 
+  # Flag to enable the Altus action.
+  ## enable_altus_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1252,6 +1252,9 @@
   # Flag to enable the Impala action.
   ## enable_impala_action=false
 
+  # Flag to enable the Altus action.
+  ## enable_altus_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app

+ 37 - 13
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -210,16 +210,39 @@ class Submission(object):
           self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace
 
         elif action.data['type'] == 'altus':
-          service = 'dataeng' # action.data['properties'].get('script_path')
-          auth_key_id = ALTUS.AUTH_KEY_ID.get()
-          auth_key_secret = ALTUS.AUTH_KEY_SECRET.get().replace('\\n', '\n')
+          self._create_file(deployment_dir, action.data['name'] + '.sh', '''#!/usr/bin/env bash
+
+export PYTHONPATH=`pwd`
+
+echo 'Starting Altus command...'
+
+python altus.py
+
+          ''')
+
           shell_script = self._generate_altus_action_script(
-            service=service,
-            auth_key_id=auth_key_id,
-            auth_key_secret=auth_key_secret
+            service=action.data['properties'].get('service'),
+            command=action.data['properties'].get('command'),
+            arguments=dict([arg.split('=', 1) for arg in action.data['properties'].get('arguments', [])]),
+            auth_key_id=ALTUS.AUTH_KEY_ID.get(),
+            auth_key_secret=ALTUS.AUTH_KEY_SECRET.get().replace('\\n', '\n')
           )
-          self._create_file(deployment_dir, action.data['name'] + '.py', shell_script)
-          self.fs.do_as_user(self.user, self.fs.copyFromLocal, os.path.join(get_desktop_root(), 'core', 'ext-py', 'navoptapi-0.1.0'), self.job.deployment_dir)
+          self._create_file(deployment_dir, 'altus.py', shell_script)
+
+          ext_py_lib_path = os.path.join(get_desktop_root(), 'core', 'ext-py')
+          lib_dir_path = os.path.join(self.job.deployment_dir, 'lib')
+          libs = [
+            (os.path.join(ext_py_lib_path, 'navoptapi-0.1.0'), 'navoptapi'),
+            (os.path.join(ext_py_lib_path, 'navoptapi-0.1.0'), 'altuscli'),
+            (os.path.join(ext_py_lib_path, 'asn1crypto-0.24.0'), 'asn1crypto'),
+            (os.path.join(ext_py_lib_path, 'rsa-3.4.2'), 'rsa'),
+            (os.path.join(ext_py_lib_path, 'pyasn1-0.1.8'), 'pyasn1'),
+          ]
+          for source_path, name in libs:
+            destination_path = os.path.join(lib_dir_path, name)
+            if not self.fs.do_as_user(self.user, self.fs.exists, destination_path):
+              # Note: would be much faster to have only one zip archive
+              self.fs.do_as_user(self.user, self.fs.copyFromLocal, os.path.join(source_path, name), destination_path)
 
         elif action.data['type'] == 'impala' or action.data['type'] == 'impala-document':
           from oozie.models2 import _get_impala_url
@@ -527,7 +550,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
       self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data))
     LOG.debug("Created/Updated %s" % (file_path,))
 
-  def _generate_altus_action_script(self, service,  auth_key_id, auth_key_secret):
+  def _generate_altus_action_script(self, service, command, arguments, auth_key_id, auth_key_secret):
     if service == 'analyticdb' or service == 'dataware':
       hostname = ALTUS.HOSTNAME_ANALYTICDB.get()
     elif service == 'dataeng':
@@ -543,7 +566,7 @@ from navoptapi.api_lib import ApiLib
 
 hostname = '%(hostname)s'
 auth_key_id = '%(auth_key_id)s'
-auth_key_secret = '%(auth_key_secret)s'
+auth_key_secret = '''%(auth_key_secret)s'''
 
 def _exec(service, command, parameters=None):
   if parameters is None:
@@ -554,15 +577,16 @@ def _exec(service, command, parameters=None):
     resp = api.call_api(command, parameters)
     return resp.json()
   except Exception, e:
+    print e
     raise e
 
-_exec('%(service)s', '%(command)s', %(args)s)
+print _exec('%(service)s', '%(command)s', %(args)s)
 
 """ % {
       'hostname': hostname,
       'service': service,
-      'command': 'listJobs',
-      'args': {},
+      'command': command,
+      'args': arguments,
       'auth_key_id': auth_key_id,
       'auth_key_secret': auth_key_secret
     }

+ 18 - 12
desktop/libs/liboozie/src/liboozie/submittion2_tests.py

@@ -121,7 +121,7 @@ def test_copy_files():
     else:
       list_dir_workspace = cluster.fs.listdir(deployment_dir)
       list_dir_deployement = cluster.fs.listdir(external_deployment_dir)
-  
+
       # All destinations there
       assert_true(cluster.fs.exists(deployment_dir + '/udf1.jar'), list_dir_workspace)
       assert_true(cluster.fs.exists(deployment_dir + '/udf2.jar'), list_dir_workspace)
@@ -129,23 +129,23 @@ def test_copy_files():
       assert_true(cluster.fs.exists(deployment_dir + '/udf4.jar'), list_dir_workspace)
       assert_true(cluster.fs.exists(deployment_dir + '/udf5.jar'), list_dir_workspace)
       assert_true(cluster.fs.exists(deployment_dir + '/udf6.jar'), list_dir_workspace)
-  
+
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf1.jar'), list_dir_deployement)
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf2.jar'), list_dir_deployement)
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf3.jar'), list_dir_deployement)
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf4.jar'), list_dir_deployement)
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf5.jar'), list_dir_deployement)
       assert_true(cluster.fs.exists(external_deployment_dir + '/udf6.jar'), list_dir_deployement)
-  
+
       stats_udf1 = cluster.fs.stats(deployment_dir + '/udf1.jar')
       stats_udf2 = cluster.fs.stats(deployment_dir + '/udf2.jar')
       stats_udf3 = cluster.fs.stats(deployment_dir + '/udf3.jar')
       stats_udf4 = cluster.fs.stats(deployment_dir + '/udf4.jar')
       stats_udf5 = cluster.fs.stats(deployment_dir + '/udf5.jar')
       stats_udf6 = cluster.fs.stats(deployment_dir + '/udf6.jar')
-  
+
       submission._copy_files('%s/workspace' % prefix, "<xml>My XML</xml>", {'prop1': 'val1'})
-  
+
       assert_not_equal(stats_udf1['fileId'], cluster.fs.stats(deployment_dir + '/udf1.jar')['fileId'])
       assert_not_equal(stats_udf2['fileId'], cluster.fs.stats(deployment_dir + '/udf2.jar')['fileId'])
       assert_not_equal(stats_udf3['fileId'], cluster.fs.stats(deployment_dir + '/udf3.jar')['fileId'])
@@ -422,13 +422,21 @@ oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
     user = User.objects.get(username='test')
     submission = Submission(user, job=TestJob(), fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname'))
 
+    command = submission._generate_altus_action_script(
+      service='dataeng',
+      command='listClusters',
+      arguments={},
+      auth_key_id='altus_auth_key_id',
+      auth_key_secret='altus_auth_key_secret'
+    )
+
     assert_true('''#!/usr/bin/env python
 
 from navoptapi.api_lib import ApiLib
 
 hostname = 'dataengapi.us-west-1.altus.cloudera.com'
 auth_key_id = 'altus_auth_key_id'
-auth_key_secret = 'altus_auth_key_secret'
+auth_key_secret = \'\'\'altus_auth_key_secret\'\'\'
 
 def _exec(service, command, parameters=None):
   if parameters is None:
@@ -439,12 +447,10 @@ def _exec(service, command, parameters=None):
     resp = api.call_api(command, parameters)
     return resp.json()
   except Exception, e:
+    print e
     raise e
 
-_exec('dataeng', 'listJobs', {})
-''' in submission._generate_altus_action_script(
-        service='dataeng',
-        auth_key_id='altus_auth_key_id',
-        auth_key_secret='altus_auth_key_secret'
-      )
+print _exec('dataeng', 'listClusters', {})
+''' in command,
+      command
     )