Explorar o código

HUE-5863 [oozie] Add an Impala action

Romain Rigaux %!s(int64=8) %!d(string=hai) anos
pai
achega
10fa3ea

+ 8 - 0
apps/oozie/src/oozie/conf.py

@@ -19,6 +19,7 @@ import os.path
 
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
+from desktop.conf import is_hue4
 from desktop.lib.conf import Config, coerce_bool
 from desktop.lib import paths
 from liboozie.conf import get_oozie_status
@@ -95,6 +96,13 @@ ENABLE_DOCUMENT_ACTION = Config(
   default=False
 )
 
+ENABLE_IMPALA_ACTION = Config(
+  key="enable_impala_action",
+  help=_t("Flag to enable the Impala action."),
+  type=bool,
+  default=is_hue4()
+)
+
 
 def config_validator(user):
   res = []

+ 59 - 2
apps/oozie/src/oozie/models2.py

@@ -757,6 +757,7 @@ class Node():
     if self.data['type'] in ('hive2', 'hive-document') and not self.data['properties']['jdbc_url']:
       self.data['properties']['jdbc_url'] = _get_hiveserver2_url()
 
+
     if self.data['type'] == 'fork':
       links = [link for link in self.data['children'] if link['to'] in node_mapping]
       if len(links) != len(self.data['children']):
@@ -849,6 +850,19 @@ class Node():
       self.data['properties']['files'] = [{'value': prop} for prop in action['properties']['files']]
       self.data['properties']['archives'] = [{'value': prop} for prop in action['properties']['archives']]
 
+    elif self.data['type'] == ImpalaAction.TYPE:
+      self.data['properties']['shell_command'] = 'impala.sh'
+      self.data['properties']['env_var'] = []
+      self.data['properties']['capture_output'] = False
+      self.data['properties']['arguments'] = []
+
+      files = [{'value': 'impala.sh'}]
+      if self.data['properties']['key_tab_path']:
+        files.append({'value': self.data['properties']['key_tab_path']})
+
+      self.data['properties']['files'] = files
+      self.data['properties']['archives'] = []
+
     data = {
       'node': self.data,
       'mapping': mapping,
@@ -919,6 +933,8 @@ class Node():
     node_type = self.data['type']
     if self.data['type'] == JavaDocumentAction.TYPE:
       node_type = JavaAction.TYPE
+    elif self.data['type'] == ImpalaAction.TYPE:
+      node_type = ShellAction.TYPE
 
     return 'editor2/gen/workflow-%s.xml.mako' % node_type
 
@@ -1237,10 +1253,20 @@ def _get_hiveserver2_url():
     return hiveserver2_jdbc_url()
   except Exception, e:
     # Might fail is Hive is disabled
-    LOG.warn('Could not guess HiveServer2 URL: %s' % smart_str(e))
+    LOG.exception('Could not guess HiveServer2 URL: %s' % smart_str(e))
     return 'jdbc:hive2://localhost:10000/default'
 
 
+def _get_impala_url():
+  try:
+    from impala.dbms import get_query_server_config
+    return get_query_server_config()['server_host']
+  except Exception, e:
+    # Might fail is Impala is disabled
+    LOG.exception('Could not get Impalad URL: %s' % smart_str(e))
+    return 'localhost'
+
+
 class HiveServer2Action(Action):
   TYPE = 'hive2'
   DEFAULT_CREDENTIALS = 'hive2'
@@ -1266,7 +1292,6 @@ class HiveServer2Action(Action):
           'help_text': _('Arguments for beeline. E.g. --showHeader=true, -Djavax.net.ssl.trustStore=/etc/cdep-ssl-conf/CA_STANDARD/truststore.jks'),
           'type': []
      },
-     # Common
      'jdbc_url': {
           'name': 'jdbc_url',
           'label': _('HiveServer2 URL'),
@@ -1282,6 +1307,7 @@ class HiveServer2Action(Action):
                          'something requiring a password (e.g. LDAP); non-secured Hive Server 2 or Kerberized Hive Server 2 don\'t require a password.'),
           'type': ''
      },
+     # Common
      'files': {
           'name': 'files',
           'label': _('Files'),
@@ -1338,6 +1364,36 @@ class HiveServer2Action(Action):
     return [cls.FIELDS['script_path']]
 
 
+class ImpalaAction(HiveServer2Action):
+  TYPE = 'impala'
+  DEFAULT_CREDENTIALS = 'impala' # None at this time
+
+  FIELDS = HiveServer2Action.FIELDS.copy()
+  del FIELDS['jdbc_url']
+  del FIELDS['password']
+  FIELDS['impalad_host'] = {
+      'name': 'impalad_host',
+      'label': _('Impalad hostname'),
+      'value': "",
+      'help_text': _('e.g. impalad-001.cluster.com. The hostname of the Impalad to send the query to.'),
+      'type': ''
+  }
+  FIELDS['key_tab_path'] = {
+      'name': 'key_tab_path',
+      'label': _('Keytab path'),
+      'value': '',
+      'help_text': _('Path to the keytab to use when on a secure cluster, e.g. /user/joe/joe.keytab.'),
+      'type': ''
+  }
+  FIELDS['user_principal'] = {
+      'name': 'user_principal',
+      'label': _('User principal'),
+      'value': 'joe@PROD.EDH',
+      'help_text': _('Name of the principal to use in the kinit, e.g.: kinit -k -t /home/joe/joe.keytab joe@PROD.EDH.'),
+      'type': ''
+  }
+
+
 class SubWorkflowAction(Action):
   TYPE = 'subworkflow'
   FIELDS = {
@@ -2712,6 +2768,7 @@ NODES = {
   'java-widget': JavaAction,
   'hive-widget': HiveAction,
   'hive2-widget': HiveServer2Action,
+  'impala-widget': ImpalaAction,
   'sqoop-widget': SqoopAction,
   'mapreduce-widget': MapReduceAction,
   'subworkflow-widget': SubWorkflowAction,

BIN=BIN
apps/oozie/src/oozie/static/oozie/art/icon_impala_24.png


BIN=BIN
apps/oozie/src/oozie/static/oozie/art/icon_impala_48.png


+ 1 - 0
apps/oozie/src/oozie/static/oozie/js/workflow-editor.ko.js

@@ -1256,6 +1256,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
 
   self.draggableHiveAction = ko.observable(bareWidgetBuilder("Hive Script", "hive-widget"));
   self.draggableHive2Action = ko.observable(bareWidgetBuilder("HiveServer2 Script", "hive2-widget"));
+  self.draggableImpalaAction = ko.observable(bareWidgetBuilder("Impala Script", "impala-widget"));
   self.draggablePigAction = ko.observable(bareWidgetBuilder("Pig Script", "pig-widget"));
   self.draggableJavaAction = ko.observable(bareWidgetBuilder("Java program", "java-widget"));
   self.draggableMapReduceAction = ko.observable(bareWidgetBuilder("MapReduce job", "mapreduce-widget"));

+ 34 - 9
apps/oozie/src/oozie/templates/editor2/common_workflow.mako

@@ -176,6 +176,10 @@
       <img src="${ static('oozie/art/icon_beeswax_48.png') }" class="widget-icon"><sup style="color: #338bb8; margin-left: -4px">2</sup>
       <!-- /ko -->
 
+      <!-- ko if: widgetType() == 'impala-widget' || widgetType() == 'impala-document-widget' -->
+      <img src="${ static('oozie/art/icon_impala_48.png') }" class="widget-icon"><sup style="color: #338bb8; margin-left: -4px">2</sup>
+      <!-- /ko -->
+
       <!-- ko if: widgetType() == 'pig-widget' || widgetType() == 'pig-document-widget'  -->
       <img src="${ static('oozie/art/icon_pig_48.png') }" class="widget-icon">
       <!-- /ko -->
@@ -587,16 +591,23 @@
 
 
 <script type="text/html" id="common-action-credentials">
-  <em data-bind="visible: $root.credentials() == null || $root.credentials().length == 0">${ _('No available credentials.') }</em>
-  <ul data-bind="visible: $root.credentials() != null && $root.credentials().length > 0, foreach: $root.credentials" class="unstyled">
-    <li>
-      <label class="checkbox"><input type="checkbox" data-bind="checkedValue: $data, checked: $parent.properties.credentials" /> <span data-bind="text: $data"></span></label>
-    </li>
-  </ul>
+  <!-- ko if: $parent.widgetType() != 'impala-widget' -->
+    <em data-bind="visible: $root.credentials() == null || $root.credentials().length == 0">${ _('No available credentials.') }</em>
+    <ul data-bind="visible: $root.credentials() != null && $root.credentials().length > 0, foreach: $root.credentials" class="unstyled">
+      <li>
+        <label class="checkbox"><input type="checkbox" data-bind="checkedValue: $data, checked: $parent.properties.credentials" /> <span data-bind="text: $data"></span></label>
+      </li>
+    </ul>
 
-  <em data-bind="visible: properties.credentials && properties.credentials.indexOf('hbase') != -1">
-    ${ _('Requires hbase-site.xml in job path') }
-  </em>
+    <em data-bind="visible: properties.credentials && properties.credentials.indexOf('hbase') != -1">
+      ${ _('Requires hbase-site.xml in job path') }
+    </em>
+  <!-- /ko -->
+
+  <!-- ko if: $parent.widgetType() == 'impala-widget' -->
+    <input type="text" class="filechooser-input seventy" data-bind="filechooser: properties.key_tab_path, filechooserOptions: globalFilechooserOptions, hdfsAutocomplete: properties.key_tab_path, attr: { placeholder:  $root.workflow_properties.key_tab_path.help_text }"/>
+    <input type="text" data-bind="value: properties.user_principal, attr: { placeholder: $root.workflow_properties.user_principal.help_text }" />
+  <!-- /ko -->
 </script>
 
 
@@ -768,12 +779,21 @@
       </ul>
       <div class="tab-content">
         <div class="tab-pane active" data-bind="attr: { id: 'properties-' + id() }">
+          <!-- ko if: $root.workflow_properties.jdbc_url -->
           <span data-bind="text: $root.workflow_properties.jdbc_url.label"></span>
           <input type="text" data-bind="value: properties.jdbc_url, attr: { placeholder: $root.workflow_properties.jdbc_url.help_text }" />
           <br/>
+          <!-- /ko -->
+          <!-- ko if: $root.workflow_properties.password -->
           <span data-bind="text: $root.workflow_properties.password.label"></span>
           <input type="text" data-bind="value: properties.password, attr: { placeholder: $root.workflow_properties.password.help_text }" />
           <br/>
+          <!-- /ko -->
+          <!-- ko if: $root.workflow_properties.impalad_host -->
+          <span data-bind="text: $root.workflow_properties.impalad_host.label"></span>
+          <input type="text" data-bind="value: properties.impalad_host, attr: { placeholder: $root.workflow_properties.impalad_host.help_text }" />
+          <br/>
+          <!-- /ko -->
           <a class="pointer" data-bind="click: function(){ properties.arguments.push(ko.mapping.fromJS({'value': ''})); $(document).trigger('drawArrows') }">
             ${ _('Arguments') } <i class="fa fa-plus"></i>
           </a>
@@ -816,6 +836,11 @@
 </script>
 
 
+<script type="text/html" id="impala-widget">
+  <span data-bind="template: { name: 'hive2-widget' }"></span>
+</script>
+
+
 <script type="text/html" id="pig-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">

+ 11 - 2
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -16,10 +16,10 @@
 <%!
 from django.utils.translation import ugettext as _
 
-from desktop.views import commonheader, commonfooter, commonshare, _ko
 from desktop import conf
+from desktop.views import commonheader, commonfooter, commonshare, _ko
 
-from oozie.conf import ENABLE_DOCUMENT_ACTION
+from oozie.conf import ENABLE_DOCUMENT_ACTION, ENABLE_IMPALA_ACTION
 %>
 
 <%namespace name="dashboard" file="/common_dashboard.mako" />
@@ -223,6 +223,15 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons) }
          <a class="draggable-icon"><img src="${ static('oozie/art/icon_beeswax_48.png') }" class="app-icon"><sup style="color: #338bb8; margin-left: -4px; top: -14px; font-size: 12px">2</sup></a>
     </div>
 
+    % if ENABLE_IMPALA_ACTION.get():
+    <div data-bind="css: { 'draggable-widget': true },
+                    draggable: {data: draggableImpalaAction(), isEnabled: true,
+                    options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggableImpalaAction());}}}"
+         title="${_('Impala Script')}" rel="tooltip" data-placement="top">
+         <a class="draggable-icon"><img src="${ static('oozie/art/icon_impala_48.png') }" class="app-icon"></a>
+    </div>
+    % endif
+
     <div data-bind="css: { 'draggable-widget': true},
                     draggable: {data: draggablePigAction(), isEnabled: true,
                     options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggablePigAction());}}}"

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1094,6 +1094,9 @@
   ## Flag to enable Oozie backend filtering instead of doing it at the page level in Javascript. Requires Oozie 4.3+.
   # enable_oozie_backend_filtering=true
 
+  ## Flag to enable the Impala action.
+  # enable_impala_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1096,6 +1096,9 @@
   ## Flag to enable Oozie backend filtering instead of doing it at the page level in Javascript. Requires Oozie 4.3+.
   # enable_oozie_backend_filtering=true
 
+  ## Flag to enable the Impala action.
+  # enable_impala_action=false
+
 
 ###########################################################################
 # Settings to configure the Filebrowser app

+ 28 - 0
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -202,6 +202,34 @@ class Submission(object):
           self.job.override_subworkflow_id(action, workflow.id) # For displaying the correct graph
           self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace
 
+        elif action.data['type'] == 'impala':
+          from oozie.models2 import _get_impala_url
+          from impala.impala_flags import get_ssl_server_certificate
+
+          if self.api.security_enabled:
+            kinit = 'kinit -k -t *.keytab %(user_principal)s' % {
+              'user_principal': action.data['properties'].get('user_principal')
+            }
+          else:
+            kinit = ''
+
+          shell_script = """#!/bin/bash
+
+# Needed to launch impala shell in oozie
+export PYTHON_EGG_CACHE=./myeggs
+
+%(kinit)s
+
+impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % {
+  'impalad_host': action.data['properties'].get('impalad_host') or _get_impala_url(),
+  'kerberos_option': '' if self.api.security_enabled else '-k',
+  'ssl_option': '--ssl' if get_ssl_server_certificate() else '',
+  'query_file': action.data['properties'].get('script_path'),
+  'kinit': kinit
+  }
+
+          self._create_file(deployment_dir, 'impala.sh', shell_script)
+
         elif action.data['type'] == 'hive-document':
           from notebook.models import Notebook
           if action.data['properties'].get('uuid'):