Sfoglia il codice sorgente

HUE-1129 [pig] Parameters support

Part 1: support $var names in the script itself
Part 2: support parameter properties
Merge both at submission and show a popup

Add mocking in Beeswax tests
Romain Rigaux 12 anni fa
parent
commit
c134c56

+ 48 - 27
apps/beeswax/src/beeswax/tests.py

@@ -1411,38 +1411,59 @@ def test_beeswax_get_kerberos_security():
     finish()
 
 
-def test_save_design_properties():
-  client = make_logged_in_client()
-
-  resp = client.get('/beeswax/save_design_properties')
-  content = json.loads(resp.content)
-  assert_equal(-1, content['status'])
+class MockDbms:
+  
+  def __init__(self, client, server_type):
+    pass
 
-  response = _make_query(client, 'SELECT', submission_type='Save', name='My Name', desc='My Description')
-  design = response.context['design']
+  def get_databases(self):
+    return ['default', 'test']
 
-  try:
-    resp = client.post('/beeswax/save_design_properties', {'name': 'name', 'value': 'New Name', 'pk': design.id})
-    design = SavedQuery.objects.get(id=design.id)
-    content = json.loads(resp.content)
-    assert_equal(0, content['status'])
-    assert_equal('New Name', design.name)
-    assert_equal('My Description', design.desc)
-  finally:
-    design.delete()
 
-  response = _make_query(client, 'SELECT', submission_type='Save', name='My Name', desc='My Description')
-  design = response.context['design']
+class MockServer(object):
 
-  try:
-    resp = client.post('/beeswax/save_design_properties', {'name': 'description', 'value': 'New Description', 'pk': design.id})
-    design = SavedQuery.objects.get(id=design.id)
+  def setUp(self):
+    # Beware: Monkey patch Beeswax/Hive server with Mock API
+    if not hasattr(dbms, 'OriginalBeeswaxApi'):
+      dbms.OriginalBeeswaxApi = dbms.Dbms
+    dbms.Dbms = MockDbms
+
+    self.client = make_logged_in_client(is_superuser=False)
+    grant_access("test", "test", "beeswax")
+
+  def tearDown(self):
+    dbms.Dbms = dbms.OriginalBeeswaxApi
+  
+  def test_save_design_properties(self):  
+    resp = self.client.get('/beeswax/save_design_properties')
     content = json.loads(resp.content)
-    assert_equal(0, content['status'])
-    assert_equal('My Name', design.name)
-    assert_equal('New Description', design.desc)
-  finally:
-    design.delete()
+    assert_equal(-1, content['status'])
+  
+    response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name', desc='My Description')
+    design = response.context['design']
+  
+    try:
+      resp = self.client.post('/beeswax/save_design_properties', {'name': 'name', 'value': 'New Name', 'pk': design.id})
+      design = SavedQuery.objects.get(id=design.id)
+      content = json.loads(resp.content)
+      assert_equal(0, content['status'])
+      assert_equal('New Name', design.name)
+      assert_equal('My Description', design.desc)
+    finally:
+      design.delete()
+  
+    response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name', desc='My Description')
+    design = response.context['design']
+  
+    try:
+      resp = self.client.post('/beeswax/save_design_properties', {'name': 'description', 'value': 'New Description', 'pk': design.id})
+      design = SavedQuery.objects.get(id=design.id)
+      content = json.loads(resp.content)
+      assert_equal(0, content['status'])
+      assert_equal('My Name', design.name)
+      assert_equal('New Description', design.desc)
+    finally:
+      design.delete()
 
 
 def search_log_line(component, expected_log, all_logs):

+ 13 - 5
apps/pig/src/pig/api.py

@@ -15,6 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+try:
+  import json
+except ImportError:
+  import simplejson as json
 import logging
 import re
 import time
@@ -49,11 +53,10 @@ class OozieApi:
     self.fs = fs
     self.user = user
 
-  def submit(self, pig_script, mapping):
-    # TODO: will come from script properties later
-    mapping.update({
+  def submit(self, pig_script, params):
+    mapping = {
       'oozie.use.system.libpath':  'true',
-    })
+    }
 
     workflow = Workflow.objects.new_workflow(self.user)
     workflow.name = OozieApi.WORKFLOW_NAME
@@ -64,7 +67,12 @@ class OozieApi:
     script_path = workflow.deployment_dir + '/script.pig'
     self.fs.create(script_path, data=pig_script.dict['script'])
 
-    action = Pig.objects.create(name='pig', script_path=script_path, workflow=workflow, node_type='pig')
+    pig_params = []
+    for param in json.loads(params):
+      pig_params.append({"type":"argument","value":"-param"})
+      pig_params.append({"type":"argument","value":"%(name)s=%(value)s" % param})
+
+    action = Pig.objects.create(name='pig', script_path=script_path, workflow=workflow, node_type='pig', params=json.dumps(pig_params))
     action.add_node(workflow.end)
 
     start_link = workflow.start.get_link()

+ 8 - 12
apps/pig/src/pig/models.py

@@ -47,21 +47,16 @@ class Document(models.Model):
 
 
 class PigScript(Document):
-  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id']
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters']
 
-  data = models.TextField(default=json.dumps({'script': '', 'name': '', 'properties': [], 'job_id': None}))
+  data = models.TextField(default=json.dumps({'script': '', 'name': '', 'properties': [], 'job_id': None, 'parameters': []}))
 
   def update_from_dict(self, attrs):
     data_dict = self.dict
 
-    if attrs.get('script'):
-      data_dict['script'] = attrs['script']
-
-    if attrs.get('name'):
-      data_dict['name'] = attrs['name']
-
-    if attrs.get('job_id'):
-      data_dict['job_id'] = attrs['job_id']
+    for attr in PigScript._ATTRIBUTES:
+      if attrs.get(attr):
+        data_dict[attr] = attrs[attr]
 
     self.data = json.dumps(data_dict)
 
@@ -75,7 +70,7 @@ class Submission(models.Model):
   workflow = models.ForeignKey(Workflow)
 
 
-def create_or_update_script(id, name, script, user, is_design=True):
+def create_or_update_script(id, name, script, user, parameters, is_design=True):
   """Take care of security"""
   try:
     pig_script = PigScript.objects.get(id=id)
@@ -83,7 +78,7 @@ def create_or_update_script(id, name, script, user, is_design=True):
   except:
     pig_script = PigScript.objects.create(owner=user, is_design=is_design)
 
-  pig_script.update_from_dict({'name': name, 'script': script})
+  pig_script.update_from_dict({'name': name, 'script': script, 'parameters': parameters})
   pig_script.save()
 
   return pig_script
@@ -98,6 +93,7 @@ def get_scripts(user, max_count=200):
       'id': script.id,
       'name': data['name'],
       'script': data['script'],
+      'parameters': data['parameters'],
       'isDesign': script.is_design,
     }
     scripts.append(massaged_script)

+ 64 - 6
apps/pig/src/pig/templates/app.mako

@@ -124,7 +124,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
                 <i class="icon-save"></i> ${ _('Save') }
               </a>
             </li>
-            <li data-bind="click: runScript, visible: !currentScript().isRunning()">
+            <li data-bind="click: showSubmissionModal, visible: !currentScript().isRunning()">
               <a href="#" title="${ _('Run the script') }" rel="tooltip" data-placement="right">
                 <i class="icon-play"></i> ${ _('Run') }
               </a>
@@ -156,18 +156,53 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
     <div class="span10">
       <div id="edit" class="section">
         <div class="alert alert-info"><h3>${ _('Edit') } '<span data-bind="text: currentScript().name"></span>'</h3></div>
-
         <form id="queryForm">
           <textarea id="scriptEditor" data-bind="text:currentScript().script"></textarea>
         </form>
       </div>
+
       <div id="properties" class="section hide">
         <div class="alert alert-info"><h3>${ _('Edit properties for') } '<span data-bind="text: currentScript().name"></span>'</h3></div>
-        <form class="form-inline" style="padding-left: 10px">
-          <label>${ _('Script name') } &nbsp; <input type="text" id="scriptName" class="input-xlarge" data-bind="value: currentScript().name" />
+         <form class="form-inline" style="padding-left: 10px">
+          <label>
+            ${ _('Script name') } &nbsp;
+            <input type="text" id="scriptName" class="input-xlarge" data-bind="value: currentScript().name" />
+          </label>
+          <br/>
+          <br/>
+          <label>${ _('Parameters') } &nbsp;
+            <button class="btn" data-bind="click: currentScript().addParameter, visible: currentScript().parameters().length == 0" style="margin-left: 4px">
+              <i class="icon-plus"></i> ${ _('Add') }
+            </button>
           </label>
+          <div>
+            <table data-bind="css: {'parameterTable': currentScript().parameters().length > 0}">
+              <thead data-bind="visible: currentScript().parameters().length > 0">
+                <tr>
+                  <th>${ _('Name') }</th>
+                  <th>${ _('Value') }</th>
+                  <th>&nbsp;</th>
+                </tr>
+              </thead>
+              <tbody data-bind="foreach: currentScript().parameters">
+                <tr>
+                  <td><input type="text" data-bind="value: name" class="input-large" /></td>
+                  <td><input type="text" data-bind="value: value" class="input-large" /></td>
+                  <td><button data-bind="click: viewModel.currentScript().removeParameter" class="btn"><i class="icon-trash"></i> ${ _('Remove') }</button></td>
+                </tr>
+              </tbody>
+              <tfoot data-bind="visible: currentScript().parameters().length > 0">
+                <tr>
+                  <td colspan="3">
+                    <button class="btn" data-bind="click: currentScript().addParameter"><i class="icon-plus"></i> ${ _('Add') }</button>
+                  </td>
+                </tr>
+              </tfoot>
+            </table>
+          </div>
         </form>
       </div>
+
       <div id="logs" class="section hide">
         <div class="alert alert-info"><h3>${ _('Logs for') } '<span data-bind="text: currentScript().name"></span>'</h3></div>
         <div data-bind="visible: currentScript().actions().length == 0">
@@ -241,7 +276,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
   </div>
 </div>
 
-<!-- delete modal -->
+
 <div id="deleteModal" class="modal hide fade">
   <div class="modal-header">
     <a href="#" class="close" data-dismiss="modal">&times;</a>
@@ -257,6 +292,28 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
   </div>
 </div>
 
+
+<div id="submitModal" class="modal hide fade">
+  <div class="modal-header">
+    <a href="#" class="close" data-dismiss="modal">&times;</a>
+    <h3>${_('Submit Script')} '<span data-bind="text: currentScript().name"></span>' ${_('?')}</h3>
+  </div>
+  <div class="modal-body" data-bind="visible: submissionVariables().length > 0">
+    <legend style="color:#666">${_('Script variables')}</legend>
+    <div data-bind="foreach: submissionVariables" style="margin-bottom: 20px">
+      <div class="row-fluid">
+        <span data-bind="text: name" class="span3"></span>
+        <input type="text" data-bind="value: value" class="span9" />
+      </div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <a class="btn" data-dismiss="modal">${_('No')}</a>
+    <a class="btn btn-danger" data-bind="click: runScript">${_('Yes')}</a>
+  </div>
+</div>
+
+
 <div class="bottomAlert alert"></div>
 
 <script src="/static/ext/js/knockout-2.1.0.js" type="text/javascript" charset="utf-8"></script>
@@ -280,7 +337,8 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
     TOOLTIP_STOP: "${ _('Stop the execution') }",
     SAVED: "${ _('Saved') }",
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
-    NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';"
+    NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';",
+    NEW_SCRIPT_PARAMETERS: []
   };
 
   var scripts = ${ scripts | n,unicode };

+ 25 - 0
apps/pig/src/pig/tests.py

@@ -14,3 +14,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from django.contrib.auth.models import User
+
+from nose.tools import assert_true, assert_equal
+
+from desktop.lib.django_test_util import make_logged_in_client
+from desktop.lib.test_utils import grant_access
+from pig.models import create_or_update_script
+
+
+class TestPigBase(object):
+  def setUp(self):
+    self.c = make_logged_in_client(is_superuser=False)
+    grant_access("test", "test", "pig")
+    self.user = User.objects.get(username='test')
+
+  def create_script(self):
+    return create_or_update_script(10000, 'Test', 'A = LOAD "$data"; STOPE A INTO "$output";', self.user)
+
+
+class TestMock(TestPigBase):
+
+  def test_create_script(self):
+    pig_script = self.create_script()
+    assert_equal('Test', pig_script.dict['name'])

+ 18 - 3
apps/pig/src/pig/views.py

@@ -65,7 +65,14 @@ def save(request):
   if request.method != 'POST':
     raise PopupException(_('POST request required.'))
 
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
+  attrs = {
+    'id': request.POST.get('id'),
+    'name': request.POST.get('name'),
+    'script': request.POST.get('script'),
+    'user': request.user,
+    'parameters': json.loads(request.POST.get('parameters')),
+  }
+  pig_script = create_or_update_script(**attrs)
   pig_script.is_design = True
   pig_script.save()
 
@@ -79,9 +86,17 @@ def save(request):
 
 @show_oozie_error
 def run(request):
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_design=False)
+  attrs = {
+    'id': request.POST.get('id'),
+    'name': request.POST.get('name'),
+    'script': request.POST.get('script'),
+    'user': request.user,
+    'parameters': json.loads(request.POST.get('parameters')),
+    'is_design': False
+  }
+  pig_script = create_or_update_script(**attrs)
+  params = request.POST.get('parameters')
 
-  params = {}
   oozie_id = api.get(request.fs, request.user).submit(pig_script, params)
 
   pig_script.update_from_dict({'job_id': oozie_id})

+ 9 - 1
apps/pig/static/css/pig.css

@@ -46,4 +46,12 @@
   -moz-border-radius-topright: 0;
   -webkit-border-top-right-radius: 0;
   border-top-right-radius: 0;
-}
+}
+
+.parameterTable {
+  background-color: #eeeeee;
+}
+
+.parameterTable td {
+  padding: 7px;
+}

+ 61 - 20
apps/pig/static/js/pig.ko.js

@@ -16,24 +16,47 @@
 
 
 var PigScript = function (pigScript) {
-  return {
-    id: ko.observable(pigScript.id),
-    isDesign: ko.observable(pigScript.isDesign),
-    name: ko.observable(pigScript.name),
-    script: ko.observable(pigScript.script),
-    scriptSumup: ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100)),
-    isRunning: ko.observable(false),
-    selected: ko.observable(false),
-    watchUrl: ko.observable(""),
-    actions: ko.observableArray([]),
-    handleSelect: function (row, e) {
-      this.selected(!this.selected());
-    },
-    hovered: ko.observable(false),
-    toggleHover: function (row, e) {
-      this.hovered(!this.hovered());
+  var self = this;
+
+  self.id = ko.observable(pigScript.id);
+  self.isDesign = ko.observable(pigScript.isDesign);
+  self.name = ko.observable(pigScript.name);
+  self.script = ko.observable(pigScript.script);
+  self.scriptSumup = ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100));
+  self.isRunning = ko.observable(false);
+  self.selected = ko.observable(false);
+  self.watchUrl = ko.observable("");
+  self.actions = ko.observableArray([]);
+  self.handleSelect = function (row, e) {
+    this.selected(!this.selected());
+  };
+  self.hovered = ko.observable(false);
+  self.toggleHover = function (row, e) {
+    this.hovered(!this.hovered());
+  };
+  self.parameters = ko.observableArray(pigScript.parameters);
+  self.addParameter = function () {
+    self.parameters.push({name: '', value: ''});
+  };
+  self.removeParameter = function () {
+    self.parameters.remove(this);
+  };
+  self.getParameters = function () {
+    var params = {};
+    var variables = this.script().match(/\$(\w)+/g);
+    if (variables) {
+      $.each(variables, function(index, param) {
+        var p = param.substring(1);
+        params[p] = '';
+        $.each(self.parameters(), function(index, param) {
+          if (param['name'] == p) {
+            params[p] = param['value'];
+          }
+        });
+      });
     }
-  }
+    return params;
+  };
 }
 
 var Workflow = function (wf) {
@@ -72,6 +95,7 @@ var PigViewModel = function (scripts, props) {
 
   self.isLoading = ko.observable(false);
   self.allSelected = ko.observable(false);
+  self.submissionVariables = ko.observableArray([]);
 
   self.scripts = ko.observableArray(ko.utils.arrayMap(scripts, function (pigScript) {
     return new PigScript(pigScript);
@@ -85,7 +109,8 @@ var PigViewModel = function (scripts, props) {
   var _defaultScript = {
     id: -1,
     name: self.LABELS.NEW_SCRIPT_NAME,
-    script: self.LABELS.NEW_SCRIPT_CONTENT
+    script: self.LABELS.NEW_SCRIPT_CONTENT,
+    parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
   };
 
   self.currentScript = ko.observable(new PigScript(_defaultScript));
@@ -224,6 +249,19 @@ var PigViewModel = function (scripts, props) {
     }));
   }
 
+  self.showSubmissionModal = function showSubmissionModal() {
+    var script = self.currentScript();
+    self.submissionVariables.removeAll();
+    $.each(script.getParameters(), function (key, value) {
+      self.submissionVariables.push({'name': key, 'value': value});
+    });
+
+    $("#submitModal").modal({
+      keyboard: true,
+      show: true
+    });
+  }
+
   function showDeleteModal() {
     $(".deleteMsg").addClass("hide");
     if (self.currentDeleteType() == "single") {
@@ -249,7 +287,8 @@ var PigViewModel = function (scripts, props) {
         {
           id: script.id(),
           name: script.name(),
-          script: script.script()
+          script: script.script(),
+          parameters: ko.utils.stringifyJson(script.parameters())
         },
         function (data) {
           self.currentScript().id(data.id);
@@ -264,7 +303,8 @@ var PigViewModel = function (scripts, props) {
         {
           id: script.id(),
           name: script.name(),
-          script: script.script()
+          script: script.script(),
+            parameters: ko.utils.stringifyJson(self.submissionVariables())
         },
         function (data) {
           if (data.id && self.currentScript().id() != data.id){
@@ -277,6 +317,7 @@ var PigViewModel = function (scripts, props) {
           $(document).trigger("refreshDashboard");
           $(document).trigger("showLogs");
           self.updateScripts();
+          $("#submitModal").modal("hide");
         }, "json");
   }
 

+ 6 - 6
desktop/core/static/ext/js/codemirror-pig.js

@@ -157,15 +157,15 @@ CodeMirror.defineMode("pig", function(_config, parserConfig) {
 	+ "JOIN CROSS UNION SPLIT INTO IF OTHERWISE ALL AS BY USING INNER OUTER ONSCHEMA PARALLEL "
 	+ "PARTITION GROUP AND OR NOT GENERATE FLATTEN ASC DESC IS STREAM THROUGH STORE MAPREDUCE "
 	+ "SHIP CACHE INPUT OUTPUT STDERROR STDIN STDOUT LIMIT SAMPLE LEFT RIGHT FULL EQ GT LT GTE LTE " 
-	+ "NEQ MATCHES TRUE FALSE "; 
+	+ "NEQ MATCHES TRUE FALSE REGISTER DUMP"; 
 	
 	// data types
 	var pTypes = "BOOLEAN INT LONG FLOAT DOUBLE CHARARRAY BYTEARRAY BAG TUPLE MAP ";
 	
 	CodeMirror.defineMIME("text/x-pig", {
-	 name: "pig",
-	 builtins: keywords(pBuiltins),
-	 keywords: keywords(pKeywords),
-	 types: keywords(pTypes)
-	 });
+	  name: "pig",
+	  builtins: keywords(pBuiltins),
+	  keywords: keywords(pKeywords),
+	  types: keywords(pTypes)
+	});
 }());