瀏覽代碼

HUE-1135 [pig] Options support

Might need to refactor the API a bit at some point, it is difficult to
add new fields.
Some follow-up on HUE-1368.
Romain Rigaux 12 年之前
父節點
當前提交
85e63ff

+ 0 - 1
apps/oozie/src/oozie/models.py

@@ -46,7 +46,6 @@ from liboozie.submittion import Submission
 from liboozie.submittion import create_directories
 
 from oozie.conf import REMOTE_SAMPLE_DIR, SHARE_JOBS
-from oozie.management.commands import oozie_setup
 from timezones import TIMEZONES
 
 

+ 65 - 39
apps/pig/src/pig/api.py

@@ -60,52 +60,78 @@ class OozieApi:
       'oozie.use.system.libpath':  'true',
     }
 
-    workflow = Workflow.objects.new_workflow(self.user)
+    workflow = None
 
     try:
-      workflow.name = OozieApi.WORKFLOW_NAME
-      workflow.is_history = True
-      workflow.save()
-      Workflow.objects.initialize(workflow, self.fs)
+      workflow = self._create_workflow(pig_script, params)
+      oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
+    finally:
+      if workflow:
+        workflow.delete()
 
-      script_path = workflow.deployment_dir + '/script.pig'
-      self.fs.create(script_path, data=pig_script.dict['script'])
+    return oozie_wf
 
-      pig_params = []
-      for param in json.loads(params):
+  def _create_workflow(self, pig_script, params):
+    workflow = Workflow.objects.new_workflow(self.user)
+    workflow.name = OozieApi.WORKFLOW_NAME
+    workflow.is_history = True
+    workflow.save()
+    Workflow.objects.initialize(workflow, self.fs)
+
+    script_path = workflow.deployment_dir + '/script.pig'
+    self.fs.create(script_path, data=pig_script.dict['script'])
+
+    files = []
+    archives = []
+
+    popup_params = json.loads(params)
+    popup_params_names = [param['name'] for param in popup_params]
+    pig_params = self._build_parameters(popup_params)
+    script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names]
+
+    pig_params += self._build_parameters(script_params)
+
+    job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']]
+
+    for resource in pig_script.dict['resources']:
+      if resource['type'] == 'file':
+        files.append(resource['value'])
+      if resource['type'] == 'archive':
+        archives.append({"dummy": "", "name": resource['value']})
+
+    action = Pig.objects.create(
+        name='pig',
+        script_path=script_path,
+        workflow=workflow,
+        node_type='pig',
+        params=json.dumps(pig_params),
+        files=json.dumps(files),
+        archives=json.dumps(archives),
+        job_properties=json.dumps(job_properties),
+    )
+
+    action.add_node(workflow.end)
+
+    start_link = workflow.start.get_link()
+    start_link.child = action
+    start_link.save()
+
+    return workflow
+
+  def _build_parameters(self, params):
+    pig_params = []
+
+    for param in params:
+      if param['name'].startswith('-'):
+        pig_params.append({"type": "argument", "value": "%(name)s" % param})
+        if param['value']:
+          pig_params.append({"type": "argument", "value": "%(value)s" % param})
+      else:
+        # Simpler way and backward compatibility for parameters
         pig_params.append({"type": "argument", "value": "-param"})
         pig_params.append({"type": "argument", "value": "%(name)s=%(value)s" % param})
 
-      files = []
-      archives = []
-
-      for resource in pig_script.dict['resources']:
-        if resource['type'] == 'file':
-          files.append(resource['value'])
-        if resource['type'] == 'archive':
-          archives.append({"dummy": "", "name": resource['value']})
-
-      action = Pig.objects.create(
-          name='pig',
-          script_path=script_path,
-          workflow=workflow,
-          node_type='pig',
-          params=json.dumps(pig_params),
-          files=json.dumps(files),
-          archives=json.dumps(archives),
-      )
-
-      action.add_node(workflow.end)
-
-      start_link = workflow.start.get_link()
-      start_link.child = action
-      start_link.save()
-
-      oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
-    finally:
-      workflow.delete()
-
-    return oozie_wf
+    return pig_params
 
   def stop(self, job_id):
     return get_oozie().job_control(job_id, 'kill')

+ 1 - 1
apps/pig/src/pig/fixtures/initial_pig_examples.json

@@ -11,7 +11,7 @@
     "pk": 1100713,
     "model": "pig.pigscript",
     "fields": {
-      "data": "{\"job_id\": null, \"parameters\": [], \"script\": \"REGISTER piggybank.jar;\\n\\n\\ndata = LOAD '/user/hue/pig/examples/data/midsummer.txt' as (text:CHARARRAY);\\n\\nupper_case =  FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);\\n\\n\\nSTORE upper_case INTO '$output' ;\\n\", \"properties\": [], \"resources\": [{\"type\": \"file\", \"value\": \"/user/hue/pig/examples/lib/piggybank.jar\"}], \"name\": \"UpperText (example)\"}"
+      "data": "{\"job_id\": null, \"parameters\": [], \"script\": \"REGISTER piggybank.jar;\\n\\n\\ndata = LOAD '/user/hue/pig/examples/data/midsummer.txt' as (text:CHARARRAY);\\n\\nupper_case =  FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);\\n\\n\\nSTORE upper_case INTO '$output' ;\\n\", \"properties\": [], \"hadoopProperties\": [], \"resources\": [{\"type\": \"file\", \"value\": \"/user/hue/pig/examples/lib/piggybank.jar\"}], \"name\": \"UpperText (example)\"}"
     }
   }
 ]

+ 7 - 4
apps/pig/src/pig/models.py

@@ -45,7 +45,7 @@ class Document(models.Model):
 
 
 class PigScript(Document):
-  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters', 'resources']
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters', 'resources', 'hadoopProperties']
 
   data = models.TextField(default=json.dumps({
       'script': '',
@@ -53,7 +53,8 @@ class PigScript(Document):
       'properties': [],
       'job_id': None,
       'parameters': [],
-      'resources': []
+      'resources': [],
+      'hadoopProperties': []
   }))
 
   def update_from_dict(self, attrs):
@@ -70,7 +71,7 @@ class PigScript(Document):
     return json.loads(self.data)
 
 
-def create_or_update_script(id, name, script, user, parameters, resources, is_design=True):
+def create_or_update_script(id, name, script, user, parameters, resources, hadoopProperties, is_design=True):
   """This take care of security"""
   try:
     pig_script = PigScript.objects.get(id=id)
@@ -82,7 +83,8 @@ def create_or_update_script(id, name, script, user, parameters, resources, is_de
       'name': name,
       'script': script,
       'parameters': parameters,
-      'resources': resources
+      'resources': resources,
+      'hadoopProperties': hadoopProperties
   })
 
   return pig_script
@@ -103,6 +105,7 @@ def get_scripts(user, max_count=200, is_design=None):
       'script': data['script'],
       'parameters': data['parameters'],
       'resources': data['resources'],
+      'hadoopProperties': data.get('hadoopProperties', []),
       'isDesign': script.is_design,
     }
     scripts.append(massaged_script)

+ 108 - 24
apps/pig/src/pig/templates/app.mako

@@ -103,21 +103,16 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
           <ul class="nav nav-list">
             <li class="nav-header">${_('Editor')}</li>
             <li data-bind="click: editScript" class="active" data-section="edit">
-              <a href="#"><i class="icon-edit"></i> ${ _('Edit script') }</a>
+              <a href="#"><i class="icon-edit"></i> ${ _('Script') }</a>
             </li>
-            <li data-bind="click: confirmNewScript">
-              <a href="#" title="${ _('New script') }" rel="tooltip" data-placement="right">
-                <i class="icon-plus-sign"></i> ${ _('New script') }
-              </a>
-            </li>
-            <li class="nav-header">${_('Properties')}</li>
             <li data-bind="click: editScriptProperties" data-section="properties">
-              <a href="#"><i class="icon-reorder"></i> ${ _('Edit properties') }</a>
+              <a href="#"><i class="icon-reorder"></i> ${ _('Properties') }</a>
+            </li>
+            <li data-bind="click: saveScript">
+              <a href="#" title="${ _('Save the script') }" rel="tooltip" data-placement="right">
+                <i class="icon-save"></i> ${ _('Save') }
+              </a>
             </li>
-            ##<li class="nav-header">${_('Python UDF')}</li>
-            ##<li><a href="#createDataset">${ _('New') }</a></li>
-            ##<li><a href="#createDataset">${ _('List') }</a></li>
-            <li class="nav-header">${_('Actions')}</li>
             <li data-bind="click: runOrShowSubmissionModal, visible: !currentScript().isRunning()">
               <a href="#" title="${ _('Run the script') }" rel="tooltip" data-placement="right">
                 <i class="icon-play"></i> ${ _('Run') }
@@ -128,9 +123,13 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
                 <i class="icon-ban-circle"></i> ${ _('Stop') }
               </a>
             </li>
-            <li data-bind="click: saveScript">
-              <a href="#" title="${ _('Save the script') }" rel="tooltip" data-placement="right">
-                <i class="icon-save"></i> ${ _('Save') }
+            ##<li class="nav-header">${_('UDF')}</li>
+            ##<li><a href="#createDataset">${ _('Python') }</a></li>
+            ##<li><a href="#createDataset">${ _('Ruby') }</a></li>
+            <li class="nav-header">${_('Actions')}</li>
+            <li data-bind="click: confirmNewScript">
+              <a href="#" title="${ _('New script') }" rel="tooltip" data-placement="right">
+                <i class="icon-plus-sign"></i> ${ _('New script') }
               </a>
             </li>
             <li data-bind="visible: currentScript().id() != -1, click: copyScript">
@@ -143,9 +142,11 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
                 <i class="icon-trash"></i> ${ _('Delete') }
               </a>
             </li>
-            <li class="nav-header" data-bind="visible: currentScript().isRunning()">${_('Logs')}</li>
-            <li data-bind="visible: currentScript().isRunning(), click: showScriptLogs" data-section="logs">
-              <a href="#" title="${ _('Show Logs') }" rel="tooltip" data-placement="right">${ _('Current Logs') }</a>
+            <li class="nav-header">${_('Logs')}</li>
+            <li data-bind="click: showScriptLogs" data-section="logs">
+              <a href="#" title="${ _('Show Logs') }" rel="tooltip" data-placement="right">
+                <i class="icon-tasks"></i> ${ _('Current Logs') }
+              </a>
             </li>
             <li>
             <br/>
@@ -181,9 +182,19 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
             ${ _('Script name') } &nbsp;
             <input type="text" id="scriptName" class="input-xlarge" data-bind="value: currentScript().name" />
           </label>
+
           <br/>
           <br/>
-          <label>${ _('Parameters') } &nbsp;
+
+          <label>${ _('Parameters') } &nbsp; <i id="parameters-dyk" class="icon-question-sign"></i>
+            <div id="parameters-dyk-content" class="hide">
+              <ul style="text-align: left;">
+                <li>input /user/data</li>
+                <li>-param input=/user/data</li>
+                <li>-optimizer_off SplitFilter</li>
+                <li>-verbose</li>
+              </ul>
+            </div>
             <button class="btn" data-bind="click: currentScript().addParameter, visible: currentScript().parameters().length == 0" style="margin-left: 4px">
               <i class="icon-plus"></i> ${ _('Add') }
             </button>
@@ -199,8 +210,11 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
               </thead>
               <tbody data-bind="foreach: currentScript().parameters">
                 <tr>
-                  <td><input type="text" data-bind="value: name" class="input-large" /></td>
-                  <td><input type="text" data-bind="value: value" class="input-large" /></td>
+                  <td><input type="text" data-bind="value: name" class="input-xlarge" /></td>
+                  <td>
+                    <input type="text" data-bind="value: value" class="input-xxlarge" />
+                    <button class="btn fileChooserBtn" data-bind="click: $root.showFileChooser">..</button>
+                  </td>
                   <td><button data-bind="click: viewModel.currentScript().removeParameter" class="btn"><i class="icon-trash"></i> ${ _('Remove') }</button></td>
                 </tr>
               </tbody>
@@ -213,8 +227,57 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
               </tfoot>
             </table>
           </div>
+
           <br/>
-          <label>${ _('Resources') } &nbsp;
+
+          <label>${ _('Hadoop properties') } &nbsp; <i id="properties-dyk" class="icon-question-sign"></i>
+            <div id="properties-dyk-content" class="hide">
+              <ul style="text-align: left; word-wrap:break-word">
+                <li>mapred.job.queue.name production</li>
+                <li>mapred.map.tasks.speculative.execution false</li>
+              </ul>
+            </div>
+            <button class="btn" data-bind="click: currentScript().addHadoopProperties, visible: currentScript().hadoopProperties().length == 0" style="margin-left: 4px">
+              <i class="icon-plus"></i> ${ _('Add') }
+            </button>
+          </label>
+          <div>
+            <table data-bind="css: {'parameterTable': currentScript().hadoopProperties().length > 0}">
+              <thead data-bind="visible: currentScript().hadoopProperties().length > 0">
+                <tr>
+                  <th>${ _('Name') }</th>
+                  <th>${ _('Value') }</th>
+                  <th>&nbsp;</th>
+                </tr>
+              </thead>
+              <tbody data-bind="foreach: currentScript().hadoopProperties">
+                <tr>
+                  <td><input type="text" data-bind="value: name" class="input-xlarge" /></td>
+                  <td>
+                    <input type="text" data-bind="value: value" class="input-xxlarge" />
+                    <button class="btn fileChooserBtn" data-bind="click: $root.showFileChooser">..</button>
+                  </td>
+                  <td><button data-bind="click: viewModel.currentScript().removeHadoopProperties" class="btn"><i class="icon-trash"></i> ${ _('Remove') }</button></td>
+                </tr>
+              </tbody>
+              <tfoot data-bind="visible: currentScript().hadoopProperties().length > 0">
+                <tr>
+                  <td colspan="3">
+                    <button class="btn" data-bind="click: currentScript().addHadoopProperties"><i class="icon-plus"></i> ${ _('Add') }</button>
+                  </td>
+                </tr>
+              </tfoot>
+            </table>
+          </div>
+
+          <br/>
+
+          <label>${ _('Resources') } &nbsp; <i id="resources-dyk" class="icon-question-sign"></i>
+            <div id="resources-dyk-content" class="hide">
+              <ul style="text-align: left;">
+                <li>${ _("Path to a HDFS file or zip file to add to the workspace of the running script") }</li>
+              </ul>
+            </div>
             <button class="btn" data-bind="click: currentScript().addResource, visible: currentScript().resources().length == 0" style="margin-left: 4px">
               <i class="icon-plus"></i> ${ _('Add') }
             </button>
@@ -231,7 +294,7 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
               <tbody data-bind="foreach: currentScript().resources">
                 <tr>
                   <td>
-                    <select type="text" data-bind="value: type" class="input-large">
+                    <select type="text" data-bind="value: type" class="input-xlarge">
                       ##<option value="udf">${ _('UDF') }</option>
                       <option value="file">${ _('File') }</option>
                       <option value="archive">${ _('Archive') }</option>
@@ -506,7 +569,8 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
     NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';",
     NEW_SCRIPT_PARAMETERS: [],
-    NEW_SCRIPT_RESOURCES: []
+    NEW_SCRIPT_RESOURCES: [],
+    NEW_SCRIPT_HADOOP_PROPERTIES: []
   };
 
   var appProperties = {
@@ -960,6 +1024,26 @@ ${ commonheader(None, "pig", user, "100px") | n,unicode }
       'html': true
     });
 
+    $("#parameters-dyk").popover({
+      'title': "${_('Names and values of Pig parameters and options, e.g.')}",
+      'content': $("#parameters-dyk-content").html(),
+      'trigger': 'hover',
+      'html': true
+    });
+
+    $("#properties-dyk").popover({
+      'title': "${_('Names and values of Hadoop properties, e.g.')}",
+      'content': $("#properties-dyk-content").html(),
+      'trigger': 'hover',
+      'html': true
+    });
+
+    $("#resources-dyk").popover({
+      'title': "${_('Include files or compressed files')}",
+      'content': $("#resources-dyk-content").html(),
+      'trigger': 'hover',
+      'html': true
+    });
   });
 
   window.onbeforeunload = function (e) {

+ 63 - 5
apps/pig/src/pig/tests.py

@@ -28,6 +28,7 @@ from nose.tools import assert_true, assert_equal
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
+from hadoop import pseudo_hdfs4
 from liboozie.oozie_api_test import OozieServerProvider
 from oozie.tests import OozieBase
 
@@ -42,6 +43,7 @@ class TestPigBase(object):
       'script': 'A = LOAD "$data"; STORE A INTO "$output";',
       'parameters': [],
       'resources': [],
+      'hadoopProperties': []
   }
 
   def setUp(self):
@@ -50,9 +52,15 @@ class TestPigBase(object):
     self.user = User.objects.get(username='test')
 
   def create_script(self):
-    attrs = {'user': self.user,}
-    attrs.update(TestPigBase.SCRIPT_ATTRS)
-    return create_or_update_script(**attrs)
+    return create_script(self.user)
+
+
+def create_script(user, xattrs=None):
+  attrs = {'user': user}
+  attrs.update(TestPigBase.SCRIPT_ATTRS)
+  if xattrs is not None:
+    attrs.update(xattrs)
+  return create_or_update_script(**attrs)
 
   def make_log_links(self):
     # FileBrowser
@@ -109,8 +117,9 @@ class TestMock(TestPigBase):
   def test_save(self):
     attrs = {'user': self.user,}
     attrs.update(TestPigBase.SCRIPT_ATTRS)
-    attrs['parameters'] = json.dumps([])
-    attrs['resources'] = json.dumps([])
+    attrs['parameters'] = json.dumps(TestPigBase.SCRIPT_ATTRS['parameters'])
+    attrs['resources'] = json.dumps(TestPigBase.SCRIPT_ATTRS['resources'])
+    attrs['hadoopProperties'] = json.dumps(TestPigBase.SCRIPT_ATTRS['hadoopProperties'])
 
     # Save
     self.c.post(reverse('pig:save'), data=attrs, follow=True)
@@ -126,6 +135,53 @@ class TestWithHadoop(OozieBase):
     grant_access("test", "test", "pig")
     self.c.post(reverse('pig:install_examples'))
 
+  def test_create_workflow(self):
+    cluster = pseudo_hdfs4.shared_cluster()
+    api = OozieApi(cluster.fs, self.user)
+
+    xattrs = {
+      'parameters': [
+        {'name': 'output', 'value': '/tmp'},
+        {'name': '-param', 'value': 'input=/data'}, # Alternative way for params
+        {'name': '-optimizer_off', 'value': 'SplitFilter'},
+        {'name': '-v', 'value': ''},
+       ],
+      'resources': [
+        {'type': 'file', 'value': '/tmp/file'},
+        {'type': 'archive', 'value': '/tmp/file.zip'},
+      ],
+      'hadoopProperties': [
+        {'name': 'mapred.map.tasks.speculative.execution', 'value': 'false'},
+        {'name': 'mapred.job.queue', 'value': 'fast'},
+      ]
+    }
+
+    pig_script = create_script(self.user, xattrs)
+    params = json.dumps([
+      {'name': 'output', 'value': '/tmp2'},
+    ])
+
+    workflow = api._create_workflow(pig_script, params)
+    pig_action = workflow.start.get_child('to').get_full_node()
+
+    assert_equal([
+        {u'type': u'argument', u'value': u'-param'}, {u'type': u'argument', u'value': u'output=/tmp2'},
+        {u'type': u'argument', u'value': u'-param'}, {u'type': u'argument', u'value': u'input=/data'},
+        {u'type': u'argument', u'value': u'-optimizer_off'}, {u'type': u'argument', u'value': u'SplitFilter'},
+        {u'type': u'argument', u'value': u'-v'},
+    ], pig_action.get_params())
+
+    assert_equal([
+        {u'name': u'mapred.map.tasks.speculative.execution', u'value': u'false'},
+        {u'name': u'mapred.job.queue', u'value': u'fast'},
+    ], pig_action.get_properties())
+
+    assert_equal(['/tmp/file'], pig_action.get_files())
+
+    assert_equal([
+        {u'dummy': u'', u'name': u'/tmp/file.zip'},
+    ], pig_action.get_archives())
+
   def wait_until_completion(self, pig_script_id, timeout=300.0, step=5, expected_status='SUCCEEDED'):
     script = PigScript.objects.get(id=pig_script_id)
     job_id = script.dict['job_id']
@@ -159,6 +215,7 @@ class TestWithHadoop(OozieBase):
       'user': script.owner,
       'parameters': json.dumps(script_dict['parameters']),
       'resources': json.dumps(script_dict['resources']),
+      'hadoopProperties': json.dumps(script_dict['hadoopProperties']),
       'submissionVariables': json.dumps([{"name": "output", "value": '/tmp/test_pig'}]),
     }
 
@@ -178,6 +235,7 @@ class TestWithHadoop(OozieBase):
       'user': script.owner,
       'parameters': json.dumps(script_dict['parameters']),
       'resources': json.dumps(script_dict['resources']),
+      'hadoopProperties': json.dumps(script_dict['hadoopProperties']),
       'submissionVariables': json.dumps([{"name": "output", "value": '/tmp/test_pig'}]),
     }
 

+ 12 - 2
apps/pig/src/pig/views.py

@@ -72,6 +72,7 @@ def save(request):
     'user': request.user,
     'parameters': json.loads(request.POST.get('parameters')),
     'resources': json.loads(request.POST.get('resources')),
+    'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
   }
   pig_script = create_or_update_script(**attrs)
   pig_script.is_design = True
@@ -115,6 +116,7 @@ def run(request):
     'user': request.user,
     'parameters': json.loads(request.POST.get('parameters')),
     'resources': json.loads(request.POST.get('resources')),
+    'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
     'is_design': False
   }
 
@@ -146,9 +148,16 @@ def copy(request):
   script = existing_script_data["script"]
   parameters = existing_script_data["parameters"]
   resources = existing_script_data["resources"]
+  hadoopProperties = existing_script_data["hadoopProperties"]
 
   pig_script = PigScript.objects.create(owner=request.user)
-  pig_script.update_from_dict({'name': name, 'script': script, 'parameters': parameters, 'resources': resources})
+  pig_script.update_from_dict({
+      'name': name,
+      'script': script,
+      'parameters': parameters,
+      'resources': resources,
+      'hadoopProperties': hadoopProperties
+  })
   pig_script.save()
 
   response = {
@@ -156,7 +165,8 @@ def copy(request):
     'name': name,
     'script': script,
     'parameters': parameters,
-    'resources': resources
+    'resources': resources,
+    'hadoopProperties': hadoopProperties
   }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")

+ 39 - 12
apps/pig/static/js/pig.ko.js

@@ -21,6 +21,16 @@ var Resource = function (resource) {
   self.value = ko.observable(resource.value);
 };
 
+var HadoopProperty = function (property) {
+  var self = this;
+
+  self.name = ko.observable(property.name);
+  self.value = ko.observable(property.value);
+};
+
+var PigParameter = HadoopProperty;
+
+
 var PigScript = function (pigScript) {
   var self = this;
 
@@ -41,9 +51,13 @@ var PigScript = function (pigScript) {
   self.toggleHover = function (row, e) {
     this.hovered(!this.hovered());
   };
-  self.parameters = ko.observableArray(pigScript.parameters);
+
+  self.parameters = ko.observableArray([]);
+  ko.utils.arrayForEach(pigScript.parameters, function (parameter) {
+    self.parameters.push(new PigParameter({name: parameter.name, value: parameter.value}));
+  });
   self.addParameter = function () {
-    self.parameters.push({name: '', value: ''});
+    self.parameters.push(new PigParameter({name: '', value: ''}));
   };
   self.removeParameter = function () {
     self.parameters.remove(this);
@@ -55,15 +69,25 @@ var PigScript = function (pigScript) {
       $.each(variables, function(index, param) {
         var p = param.substring(1);
         params[p] = '';
-        $.each(self.parameters(), function(index, param) {
-          if (param['name'] == p) {
-            params[p] = param['value'];
-          }
-        });
       });
     }
+    $.each(self.parameters(), function(index, param) {
+        params[param.name()] = param.value();
+    });
     return params;
   };
+
+  self.hadoopProperties = ko.observableArray([]);
+  ko.utils.arrayForEach(pigScript.hadoopProperties, function (property) {
+    self.hadoopProperties.push(new HadoopProperty({name: property.name, value: property.value}));
+  });
+  self.addHadoopProperties = function () {
+    self.hadoopProperties.push(new HadoopProperty({name: '', value: ''}));
+  };
+  self.removeHadoopProperties = function () {
+    self.hadoopProperties.remove(this);
+  };
+
   self.resources = ko.observableArray([]);
   ko.utils.arrayForEach(pigScript.resources, function (resource) {
     self.resources.push(new Resource({type: resource.type, value: resource.value}));
@@ -133,7 +157,8 @@ var PigViewModel = function (props) {
     name: self.LABELS.NEW_SCRIPT_NAME,
     script: self.LABELS.NEW_SCRIPT_CONTENT,
     parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
-    resources: self.LABELS.NEW_SCRIPT_RESOURCES
+    resources: self.LABELS.NEW_SCRIPT_RESOURCES,
+    hadoopProperties: self.LABELS.NEW_SCRIPT_HADOOP_PROPERTIES
   };
 
   self.currentScript = ko.observable(new PigScript(_defaultScript));
@@ -420,8 +445,9 @@ var PigViewModel = function (props) {
           id: script.id(),
           name: script.name(),
           script: script.script(),
-          parameters: ko.utils.stringifyJson(script.parameters()),
-          resources: ko.toJSON(script.resources())
+          parameters: ko.toJSON(script.parameters()),
+          resources: ko.toJSON(script.resources()),
+          hadoopProperties: ko.toJSON(script.hadoopProperties()),
         },
         function (data) {
           self.currentScript().id(data.id);
@@ -438,9 +464,10 @@ var PigViewModel = function (props) {
           id: script.id(),
           name: script.name(),
           script: script.script(),
-          parameters: ko.utils.stringifyJson(script.parameters()),
+          parameters: ko.toJSON(script.parameters()),
           submissionVariables: ko.utils.stringifyJson(self.submissionVariables()),
-          resources: ko.toJSON(script.resources())
+          resources: ko.toJSON(script.resources()),
+          hadoopProperties: ko.toJSON(script.hadoopProperties())
         },
         function (data) {
           if (data.id && self.currentScript().id() != data.id){