Эх сурвалжийг харах

[oozie] Pull Pig parameters from Pig action script

Romain Rigaux 11 жил өмнө
parent
commit
63325c001a

+ 2 - 0
apps/oozie/src/oozie/models2.py

@@ -1087,6 +1087,8 @@ def find_json_parameters(fields):
 
   return params
 
+def find_dollar_variables(text):
+  return re.findall('[^\n\\\\]\$([^\{ \'\"\-;\(\)]+)', text, re.MULTILINE)  
 
 
 def import_workflows_from_hue_3_7():

+ 50 - 14
apps/oozie/src/oozie/templates/editor/workflow_editor.mako

@@ -505,13 +505,21 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
         </a>
       </li>
     </ul>
-    <a class="pointer" data-bind="click: function(){ properties.prepares.push({'type': 'mkdir', 'value': ''});$(document).trigger('drawArrows') }">${ _('Directory') } <i class="fa fa-plus"></i></a>
-    <a class="pointer" data-bind="click: function(){ properties.prepares.push({'type': 'delete', 'value': ''});$(document).trigger('drawArrows') }">${ _('Delete') } <i class="fa fa-plus"></i></a>
+    <a class="pointer" data-bind="click: function(){ properties.prepares.push({'type': 'mkdir', 'value': ''});$(document).trigger('drawArrows') }">
+      ${ _('Directory') } <i class="fa fa-plus"></i>
+    </a>
+    <a class="pointer" data-bind="click: function(){ properties.prepares.push({'type': 'delete', 'value': ''});$(document).trigger('drawArrows') }">
+      ${ _('Delete') } <i class="fa fa-plus"></i>
+    </a>
 
     <h6>${ _('Job XML') }</h6>
     <input type="text" class="input-xlarge filechooser-input" data-bind="filechooser: properties.job_xml"/>
 
-    <h6><a class="pointer" data-bind="click: function(){ properties.job_properties.push({'name': '', 'value': ''});$(document).trigger('drawArrows') }">${ _('Properties') } <i class="fa fa-plus"></i></a></h6>
+    <h6>
+      <a class="pointer" data-bind="click: function(){ properties.job_properties.push({'name': '', 'value': ''});$(document).trigger('drawArrows') }">
+        ${ _('Properties') } <i class="fa fa-plus"></i>
+      </a>
+    </h6>
     <ul data-bind="visible: properties.job_properties().length > 0, foreach: properties.job_properties" class="unstyled">
       <li>
         <input data-bind="value: name"/>
@@ -523,7 +531,11 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
     </ul>
     <em data-bind="visible: properties.job_properties().length == 0">${ _('No properties defined.') }</em>
 
-    <h6><a class="pointer" data-bind="click: function(){ properties.archives.push({'name': ''});$(document).trigger('drawArrows') }">${ _('Archives') } <i class="fa fa-plus"></i></a></h6>
+    <h6>
+      <a class="pointer" data-bind="click: function(){ properties.archives.push({'name': ''});$(document).trigger('drawArrows') }">
+        ${ _('Archives') } <i class="fa fa-plus"></i>
+      </a>
+    </h6>
     <ul data-bind="visible: properties.archives().length > 0, foreach: properties.archives" class="unstyled">
       <li>
         <input data-bind="value: name"/>
@@ -538,11 +550,15 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
 
 
 <script type="text/html" id="common-properties-arguments">
-  <h6><a class="pointer" data-bind="click: function(){ properties.arguments.push({'value': ''});$(document).trigger('drawArrows') }">${ _('Arguments') } <i class="fa fa-plus"></i></a></h6>
+  <h6>
+    <a class="pointer" data-bind="click: function(){ properties.arguments.push({'value': ''}); $(document).trigger('drawArrows') }">
+      ${ _('Arguments') } <i class="fa fa-plus"></i>
+    </a>
+  </h6>
   <ul class="unstyled" data-bind="visible: properties.arguments().length > 0, foreach: properties.arguments">
     <li>
       <input type="text" class="span11" data-bind="value: value"/>
-      <a href="#" data-bind="click: function(){ $parent.properties.arguments.remove(this);$(document).trigger('drawArrows') }">
+      <a href="#" data-bind="click: function(){ $parent.properties.arguments.remove(this); $(document).trigger('drawArrows') }">
         <i class="fa fa-minus"></i>
       </a>
     </li>
@@ -552,11 +568,15 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
 
 
 <script type="text/html" id="common-properties-files">
-  <h6><a class="pointer" data-bind="click: function(){ properties.files.push({'value': ''});$(document).trigger('drawArrows') }">${ _('Files') } <i class="fa fa-plus"></i></a></h6>
+  <h6>
+    <a class="pointer" data-bind="click: function(){ properties.files.push({'value': ''}); $(document).trigger('drawArrows') }">
+      ${ _('Files') } <i class="fa fa-plus"></i>
+    </a>
+  </h6>
   <ul class="unstyled" data-bind="foreach: properties.files">
     <li style="margin-bottom: 3px">
       <input type="text" class="span9 filechooser-input" data-bind="filechooser: value"/>
-      <a href="#" data-bind="click: function(){ $parent.properties.files.remove(this);$(document).trigger('drawArrows') }">
+      <a href="#" data-bind="click: function(){ $parent.properties.files.remove(this); $(document).trigger('drawArrows') }">
         <i class="fa fa-minus"></i>
       </a>
     </li>
@@ -565,11 +585,15 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
 
 
 <script type="text/html" id="common-properties-parameters">
-  <h6><a class="pointer" data-bind="click: function(){ properties.parameters.push({'value': ''}); $(document).trigger('drawArrows')}">${ _('Parameters') } <i class="fa fa-plus"></i></a></h6>
+  <h6>
+    <a class="pointer" data-bind="click: function(){ properties.parameters.push({'value': ''}); $(document).trigger('drawArrows')}">
+      ${ _('Parameters') } <i class="fa fa-plus"></i>
+    </a>
+  </h6>
   <ul class="unstyled" data-bind="foreach: properties.parameters">
     <li style="margin-bottom: 3px">
-      <input type="text" class="span11" data-bind="value: value"/>
-      <a href="#" data-bind="click: function(){ $parent.properties.parameters.remove(this);$(document).trigger('drawArrows') }">
+      <input type="text" class="span11" data-bind="value: value, attr: { placeholder: $parent.actionParametersUI }"/>
+      <a href="#" data-bind="click: function(){ $parent.properties.parameters.remove(this); $(document).trigger('drawArrows') }">
         <i class="fa fa-minus"></i>
       </a>
     </li>
@@ -614,6 +638,13 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
 <script type="text/html" id="hive-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">
+  
+    <div data-bind="visible: ! $root.isEditing()">
+      <span data-bind="text: $root.workflow_properties.script_path.label"></span>
+      <a data-bind="attr: {href: '/filebrowser/view' + properties.script_path() }" target="_blank" title="${ _('Open script') }">
+        <strong data-bind="text: properties.script_path"></strong>
+      </a>
+    </div>
     <div data-bind="visible: $root.isEditing" style="margin-bottom: 20px">
       <div data-bind="visible: ! $parent.ooziePropertiesExpanded()">
         <span data-bind="text: $root.workflow_properties.script_path.label"></span>
@@ -624,7 +655,6 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
       </div>
     </div>
 
-
     <div data-bind="visible: $parent.ooziePropertiesExpanded">
       <ul class="nav nav-tabs">
         <li class="active"><a data-bind="attr: { href: '#properties-' + id()}" data-toggle="tab">${ _('Properties') }</a></li>
@@ -660,6 +690,13 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
 <script type="text/html" id="hive2-widget">
   <!-- ko if: $root.workflow.getNodeById(id()) -->
   <div class="row-fluid" data-bind="with: $root.workflow.getNodeById(id())" style="padding: 10px">
+  
+    <div data-bind="visible: ! $root.isEditing()">
+      <span data-bind="text: $root.workflow_properties.script_path.label"></span>
+      <a data-bind="attr: {href: '/filebrowser/view' + properties.script_path() }" target="_blank" title="${ _('Open script') }">
+        <strong data-bind="text: properties.script_path"></strong>
+      </a>
+    </div>
     <div data-bind="visible: $root.isEditing" style="margin-bottom: 20px">
       <div data-bind="visible: ! $parent.ooziePropertiesExpanded()">
         <span data-bind="text: $root.workflow_properties.script_path.label"></span>
@@ -719,11 +756,10 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user, "40px") | n,unicode }
         <strong data-bind="text: properties.script_path"></strong>
       </a>
     </div>
-
     <div data-bind="visible: $root.isEditing">
       <div data-bind="visible: ! $parent.ooziePropertiesExpanded()">
         <span data-bind="text: $root.workflow_properties.script_path.label"></span>
-        <input type="text" data-bind="value: properties.script_path" />
+        <input type="text" class="filechooser-input" data-bind="filechooser: properties.script_path" />
 
         <div class="row-fluid">
           <div class="span6" data-bind="template: { name: 'common-properties-parameters' }"></div>

+ 17 - 1
apps/oozie/src/oozie/tests2.py

@@ -27,7 +27,7 @@ from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_permission, add_to_group, reformat_json, reformat_xml
 
 
-from oozie.models2 import Workflow
+from oozie.models2 import Workflow, find_dollar_variables
 
 
 LOG = logging.getLogger(__name__)
@@ -38,6 +38,22 @@ class TestEditor():
   def setUp(self):
     self.wf = Workflow()
 
+  def test_parsing(self):
+    assert_equal(['input', 'LIMIT', 'out'], find_dollar_variables("""
+data = '$input';
+$out = LIMIT data $LIMIT; -- ${nah}
+$output = STORE "$out";   
+    """))
+
+    assert_equal(['max_salary', 'limit'], find_dollar_variables("""
+SELECT sample_07.description, sample_07.salary
+FROM
+  sample_07
+WHERE
+( sample_07.salary > $max_salary)
+ORDER BY sample_07.salary DESC
+LIMIT $limit"""))
+
   def test_workflow_gen_xml(self):
     assert_equal(
         '<workflow-app name="Test" xmlns="0">\n'

+ 2 - 1
apps/oozie/src/oozie/urls.py

@@ -77,7 +77,8 @@ urlpatterns += patterns(
   url(r'^editor/workflow/submit/(?P<doc_id>\d+)$', 'submit_workflow', name='editor_submit_workflow'),
   url(r'^editor/workflow/new_node/$', 'new_node', name='new_node'),
   url(r'^editor/workflow/add_node/$', 'add_node', name='add_node'),
-  url(r'^editor/workflow/parameters/$', 'workflow_parameters', name='workflow_parameters'),  
+  url(r'^editor/workflow/parameters/$', 'workflow_parameters', name='workflow_parameters'),
+  url(r'^editor/workflow/action/parameters/$', 'action_parameters', name='action_parameters'),
   url(r'^editor/workflow/gen_xml/$', 'gen_xml_workflow', name='gen_xml_workflow'), # Temporary
   url(r'^editor/workflow/import_workflows/$', 'import_hue_3_7_workflows', name='import_hue_3_7_workflows'), # Temporary
   

+ 25 - 1
apps/oozie/src/oozie/views/editor2.py

@@ -35,7 +35,7 @@ from liboozie.oozie_api import get_oozie
 from liboozie.submission2 import Submission
 
 from oozie.forms import ParameterForm
-from oozie.models2 import Workflow, Coordinator, Bundle, NODES, WORKFLOW_NODE_PROPERTIES, import_workflows_from_hue_3_7
+from oozie.models2 import Node, Workflow, Coordinator, Bundle, NODES, WORKFLOW_NODE_PROPERTIES, import_workflows_from_hue_3_7, find_dollar_variables
 
 
 LOG = logging.getLogger(__name__)
@@ -162,6 +162,30 @@ def add_node(request):
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
+def action_parameters(request):
+  response = {'status': -1}
+  parameters = set()
+
+  try:
+    node_data = json.loads(request.POST.get('node', '{}'))
+    
+    script_path = node_data.get('properties', {}).get('script_path', {})
+    if script_path:
+      script_path = script_path.replace('hdfs://', '')
+      if request.fs.do_as_user(request.user, request.fs.exists, script_path):
+        data =  request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024**2)
+        parameters = parameters.union(set(find_dollar_variables(data)))
+       
+    parameters = parameters.union(set(Node(node_data).find_parameters()))
+                    
+    response['status'] = 0
+    response['parameters'] = list(parameters)
+  except Exception, e:
+    response['message'] = str(e)
+    
+  return HttpResponse(json.dumps(response), mimetype="application/json")
+
+
 def workflow_parameters(request):
   response = {'status': -1}
 

+ 0 - 4
apps/oozie/static/js/coordinator-editor.ko.js

@@ -72,10 +72,6 @@ var Coordinator = function (vm, coordinator) {
 
 	self.variables.push(ko.mapping.fromJS(_var));	  
   };
-  
-  self.init = function() {
-    // load
-  };
 }
 
 

+ 27 - 0
apps/oozie/static/js/workflow-editor.ko.js

@@ -83,6 +83,12 @@ var Node = function (node) {
   self.properties = ko.mapping.fromJS(typeof node.properties != "undefined" && node.properties != null ? node.properties : {});
   self.children = ko.mapping.fromJS(typeof node.children != "undefined" && node.children != null ? node.children : []);
 
+  self.actionParameters = ko.observableArray([]);
+  self.actionParametersUI = ko.computed(function() { // TODO: remove truncation when autocomplete
+    return $.map(self.actionParameters().slice(0, 3), function(param) {return param + '=...'}).join();
+  });
+  self.actionParametersFetched = ko.observable(false);
+  
   self.get_link = function (name) {
     var _link = null;
     $.each(self.children(), function (index, link) {
@@ -118,6 +124,27 @@ var Node = function (node) {
       self.children.remove(_link);
     }
   }
+  
+  if (typeof self.properties.parameters != "undefined") { // Fetch once the possible variable when they exist 
+    self.properties.parameters.subscribe(function(newVal) { // TODO: only fetch when first time focus on one of the parameters and showing the autocomplete.
+      if (newVal && ! self.actionParametersFetched()) {
+	    $.post("/oozie/editor/workflow/action/parameters/", {
+		  "node": ko.mapping.toJSON(self),
+	     }, function (data) {
+	       self.actionParametersFetched(true);
+  	       self.actionParameters(data.parameters);
+	    }).fail(function (xhr, textStatus, errorThrown) {
+	      $(document).trigger("error", xhr.responseText);
+	    });
+	  }
+    });  
+  }
+  
+  if (typeof self.properties.script_path != "undefined") {
+    self.properties.script_path.subscribe(function() {
+      self.actionParametersFetched(false);
+    });
+  }
 }