Преглед изворни кода

HUE-1389 [oozie] Skeleton to drag & Drop a saved Hive query in a workflow

Romain Rigaux пре 9 година
родитељ
комит
f70983a

+ 96 - 1
apps/oozie/src/oozie/models2.py

@@ -1835,6 +1835,100 @@ class ForkNode(Action):
     return []
 
 
+class HiveDocumentAction(Action):
+  TYPE = 'hive-document'
+  FIELDS = {
+     'uuid': {
+          'name': 'uuid',
+          'label': _('UUID of a Hive query'),
+          'value': '',
+          'help_text': _('Select a saved Hive query you want to schedule.'),
+          'type': 'hive'
+     },
+     'parameters': {
+          'name': 'parameters',
+          'label': _('Parameters'),
+          'value': [],
+          'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}')  % {'type': TYPE.title()},
+          'type': ''
+     },
+     # Common
+     'jdbc_url': {
+          'name': 'jdbc_url',
+          'label': _('HiveServer2 URL'),
+          'value': "",
+          'help_text': _('e.g. jdbc:hive2://localhost:10000/default. JDBC URL for the Hive Server 2.'),
+          'type': ''
+     },
+     'password': {
+          'name': 'password',
+          'label': _('Password'),
+          'value': '',
+          'help_text': _('The password element must contain the password of the current user. However, the password is only used if Hive Server 2 is backed by '
+                         'something requiring a password (e.g. LDAP); non-secured Hive Server 2 or Kerberized Hive Server 2 don\'t require a password.'),
+          'type': ''
+     },
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Files put in the running directory.'),
+          'type': ''
+     },
+     'archives': {
+          'name': 'archives',
+          'label': _('Archives'),
+          'value': [],
+          'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
+          'type': ''
+     },
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production'),
+          'type': ''
+     },
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.'),
+          'type': ''
+     },
+     'job_xml': {
+          'name': 'job_xml',
+          'label': _('Job XML'),
+          'value': '',
+          'help_text': _('Refer to a Hadoop JobConf job.xml'),
+          'type': ''
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
+
+
+
+def _generate_hive_script():
+  self._create_file(deployment_dir, self.job.XML_FILE_NAME, oozie_xml)
+
+
 class DecisionNode(Action):
   TYPE = 'decision'
   FIELDS = {}
@@ -1865,7 +1959,8 @@ NODES = {
   'fork-widget': ForkNode,
   'decision-widget': DecisionNode,
   'spark-widget': SparkAction,
-  'generic-widget': GenericAction
+  'generic-widget': GenericAction,
+  'hive-document-widget': HiveDocumentAction
 }
 
 

+ 2 - 0
apps/oozie/src/oozie/static/oozie/js/workflow-editor.ko.js

@@ -516,6 +516,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
 
 
   self.subworkflows = ko.observableArray(getOtherSubworkflows(self, subworkflows_json));
+  self.hiveQueries = ko.observableArray([{"uuid": "c73171ec-acff-4c30-b350-3df0c31689a9", "name": "show tables"}]);
   self.history = ko.mapping.fromJS(history_json);
 
   self.getSubWorkflow = function (uuid) {
@@ -1227,6 +1228,7 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
   self.draggableDistCpAction = ko.observable(bareWidgetBuilder("Distcp", "distcp-widget"));
   self.draggableSparkAction = ko.observable(bareWidgetBuilder("Spark", "spark-widget"));
   self.draggableGenericAction = ko.observable(bareWidgetBuilder("Generic", "generic-widget"));
+  self.draggableHiveDocumentAction = ko.observable(bareWidgetBuilder("Hive", "hive-document-widget"));
 
   self.draggableKillNode = ko.observable(bareWidgetBuilder("Kill", "kill-widget"));
 };

+ 60 - 1
apps/oozie/src/oozie/templates/editor2/common_workflow.mako

@@ -157,7 +157,7 @@
       <img src="${ static('oozie/art/icon_beeswax_48.png') }" class="widget-icon">
       <!-- /ko -->
 
-      <!-- ko if: widgetType() == 'hive2-widget' -->
+      <!-- ko if: widgetType() == 'hive2-widget' || widgetType() == 'hive-document-widget' -->
       <img src="${ static('oozie/art/icon_beeswax_48.png') }" class="widget-icon"><sup style="color: #338bb8; margin-left: -4px">2</sup>
       <!-- /ko -->
 
@@ -931,6 +931,65 @@
 </script>
 
 
+<script type="text/html" id="hive-document-widget">
+  <!-- ko if: $root.workflow.getNodeById(id()) -->
+  <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">
+
+    <div data-bind="visible: ! $root.isEditing()">
+      <span data-bind="template: { name: 'logs-icon' }"></span>
+      <span data-bind='text: properties.uuid'></span>
+    </div>
+
+    <div data-bind="visible: $root.isEditing">
+      <div data-bind="visible: ! $parent.ooziePropertiesExpanded()" class="nowrap">
+        <span data-bind='text: properties.uuid'></span>
+
+        <div class="row-fluid">
+          <div class="span6" data-bind="template: { name: 'common-properties-parameters' }"></div>
+          <div class="span6" data-bind="template: { name: 'common-properties-files' }"></div>
+        </div>
+      </div>
+    </div>
+
+    <div data-bind="visible: $parent.ooziePropertiesExpanded">
+      <ul class="nav nav-tabs">
+        <li class="active"><a data-bind="attr: { href: '#properties-' + id()}" data-toggle="tab">${ _('Properties') }</a></li>
+        <li><a data-bind="attr: { href: '#sla-' + id()}" href="#sla" data-toggle="tab">${ _('SLA') }</a></li>
+        <li><a data-bind="attr: { href: '#credentials-' + id()}" data-toggle="tab">${ _('Credentials') }</a></li>
+        <li><a data-bind="attr: { href: '#transitions-' + id()}" data-toggle="tab">${ _('Transitions') }</a></li>
+      </ul>
+      <div class="tab-content">
+        <div class="tab-pane active" data-bind="attr: { id: 'properties-' + id() }">
+          <span data-bind="text: $root.workflow_properties.jdbc_url.label"></span>
+          <input type="text" data-bind="value: properties.jdbc_url, attr: { placeholder: $root.workflow_properties.jdbc_url.help_text }" />
+          <br/>
+          <span data-bind="text: $root.workflow_properties.password.label"></span>
+          <input type="text" data-bind="value: properties.password, attr: { placeholder: $root.workflow_properties.password.help_text }" />
+          <br/>
+          <span data-bind="template: { name: 'common-action-properties' }"></span>
+          <br/>
+          <br/>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'sla-' + id() }">
+          <span data-bind="template: { name: 'common-action-sla' }"></span>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'credentials-' + id() }">
+          <span data-bind="template: { name: 'common-action-credentials' }"></span>
+        </div>
+
+        <div class="tab-pane" data-bind="attr: { id: 'transitions-' + id() }">
+          <span data-bind="template: { name: 'common-action-transition' }"></span>
+        </div>
+      </div>
+    </div>
+  </div>
+  </div>
+  <!-- /ko -->
+</script>
+
+
 <script type="text/html" id="java-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">

+ 46 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-hive-document.xml.mako

@@ -0,0 +1,46 @@
+## -*- coding: utf-8 -*-
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%namespace name="common" file="workflow-common.xml.mako" />
+
+    <action name="${ node['name'] }"${ common.credentials(node['properties']['credentials']) }${ common.retry_max(node['properties']['retry_max']) }${ common.retry_interval(node['properties']['retry_interval']) }>
+        <hive2 xmlns="uri:oozie:hive2-action:0.1">
+            <job-tracker>${'${'}jobTracker}</job-tracker>
+            <name-node>${'${'}nameNode}</name-node>
+
+            ${ common.prepares(node['properties']['prepares']) }
+            % if node['properties']['job_xml']:
+              <job-xml>${ node['properties']['job_xml'] }</job-xml>
+            % endif
+            ${ common.configuration(node['properties']['job_properties']) }
+            
+            <jdbc-url>${ node['properties']['jdbc_url'] }</jdbc-url>
+            % if node['properties']['password']:
+            <password>${ node['properties']['password'] }</password>
+            % endif
+            <script>${'${'}oozie.wf.application.path}/${ node['name'] }.sql</script>
+
+            % for param in node['properties']['parameters']:
+              <param>${ param['value'] }</param>
+            % endfor
+
+            ${ common.distributed_cache(node['properties']['files'], node['properties']['archives']) }
+        </hive2>
+        <ok to="${ node_mapping[node['children'][0]['to']].name }"/>
+        <error to="${ node_mapping[node['children'][1]['error']].name }"/>
+        ${ common.sla(node) }
+    </action>

+ 12 - 0
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -218,6 +218,15 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons) }
          title="${_('Kill')}" rel="tooltip" data-placement="top">
          <a class="draggable-icon"><i class="fa fa-stop"></i></a>
     </div>
+
+  <div class="toolbar-label">${ _('DOCUMENTS') }</div>
+
+  <div data-bind="css: { 'draggable-widget': true },
+                  draggable: {data: draggableHiveDocumentAction(), isEnabled: true,
+                  options: {'refreshPositions': true, 'stop': function(){ $root.isDragging(false); }, 'start': function(event, ui){ $root.isDragging(true); $root.currentlyDraggedWidget(draggableHiveDocumentAction());}}}"
+       title="${_('Saved Hive query')}" rel="tooltip" data-placement="top">
+       <a class="draggable-icon"><img src="${ static('oozie/art/icon_beeswax_48.png') }" class="app-icon"><sup style="color: #338bb8; margin-left: -4px; top: -14px; font-size: 12px">2</sup></a>
+  </div>
 </%def>
 </%dashboard:layout_toolbar>
 
@@ -259,6 +268,9 @@ ${ workflow.render() }
           <!-- ko if: type() == 'workflow' -->
           <select data-bind="options: $root.subworkflows, optionsText: 'name', optionsValue: 'value', value: value"></select>
           <!-- /ko -->
+          <!-- ko if: type() == 'hive' -->
+          <select data-bind="options: $root.hiveQueries, optionsText: 'name', optionsValue: 'uuid', value: value"></select>
+          <!-- /ko -->
 
           <!-- ko if: type() == 'distcp' -->
           <ul class="unstyled">

+ 12 - 6
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -184,6 +184,12 @@ class Submission(object):
 
           self.job.override_subworkflow_id(action, workflow.id) # For displaying the correct graph
           self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace
+        elif action.data['type'] == 'hive-document':
+          from notebook.models import Notebook
+          notebook = Notebook(document=Document2.objects.get_by_uuid(uuid=action.data['properties']['uuid']))
+          
+          self._create_file(deployment_dir, action.data['name'] + '.sql', notebook.get_data()['snippets'][0]['statement'])
+          #self.data['properties']['script_path'] = _generate_hive_script(self.data['uuid']) #'workspace_%s' % workflow.uui
 
     oozie_xml = self.job.to_xml(self.properties)
     self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties)
@@ -348,12 +354,12 @@ class Submission(object):
     return Coordinator.PROPERTY_APP_PATH in self.properties
 
   def _create_file(self, deployment_dir, file_name, data, do_as=False):
-   file_path = self.fs.join(deployment_dir, file_name)
-   if do_as:
-     self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0644, data=smart_str(data))
-   else:
-     self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data))
-   LOG.debug("Created/Updated %s" % (file_path,))
+    file_path = self.fs.join(deployment_dir, file_name)
+    if do_as:
+      self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0644, data=smart_str(data))
+    else:
+      self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data))
+    LOG.debug("Created/Updated %s" % (file_path,))
 
 def create_directories(fs, directory_list=[]):
   # If needed, create the remote home, deployment and data directories