Browse Source

[notebook] Support of Pig properties

For Pig parameters, Hadoop properties and resources like files and
archives.

Note: basic but works, we can improve the UX a bit.

The ${var} conflicts in the UI with the native Pig parameters.
Romain Rigaux 10 years ago
parent
commit
a460409

+ 21 - 8
apps/pig/src/pig/api.py

@@ -86,17 +86,30 @@ class OozieApi(object):
     popup_params = json.loads(params)
     popup_params_names = [param['name'] for param in popup_params]
     pig_params = self._build_parameters(popup_params)
-    script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names]
 
-    pig_params += self._build_parameters(script_params)
+    if pig_script.isV2:
+      pig_params += [{"type": "argument", "value": param} for param in pig_script.dict['parameters']]
 
-    job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']]
+      job_properties = [{"name": prop.split('=', 1)[0], "value": prop.split('=', 1)[1]} for prop in pig_script.dict['hadoopProperties']]
 
-    for resource in pig_script.dict['resources']:
-      if resource['type'] == 'file':
-        files.append(resource['value'])
-      if resource['type'] == 'archive':
-        archives.append({"dummy": "", "name": resource['value']})
+      for resource in pig_script.dict['resources']:
+        if resource.endswith('.zip') or resource.endswith('.tgz') or resource.endswith('.tar') or resource.endswith('.gz'):
+          archives.append({"dummy": "", "name": resource})
+        else:
+          files.append(resource)
+
+    else:
+      script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names]
+
+      pig_params += self._build_parameters(script_params)
+
+      job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']]
+
+      for resource in pig_script.dict['resources']:
+        if resource['type'] == 'file':
+          files.append(resource['value'])
+        if resource['type'] == 'archive':
+          archives.append({"dummy": "", "name": resource['value']})
 
     action = Pig.objects.create(
         name='pig',

+ 3 - 0
apps/pig/src/pig/models.py

@@ -59,6 +59,8 @@ class PigScript(Document):
 
   doc = generic.GenericRelation(Doc, related_name='pig_doc')
 
+  isV2 = False
+
   def update_from_dict(self, attrs):
     data_dict = self.dict
 
@@ -91,6 +93,7 @@ class PigScript(Document):
 
 
 class PigScript2(object):
+  isV2 = True     # V2 is for the Notebook app
 
   def __init__(self, attrs=None):
     self.data = json.dumps({

+ 4 - 0
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -442,6 +442,10 @@ from desktop.views import _ko
         <!-- ko template: { if: typeof properties().archives != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _('Archives') }', value: properties().archives, title: '${ _('The YARN queue to submit to (Default: default)') }', placeholder: '${ _('e.g. archive.dat') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().files != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _('Files') }', value: properties().files, title: '${ _('The YARN queue to submit to (Default: default)') }', placeholder: '${ _('e.g. file.dat') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().settings != 'undefined', name: 'property', data: { type: 'csv', label: '${ _('Settings') }', value: properties().settings, title: '${ _('The YARN queue to submit to (Default: default)') }', placeholder: '${ _('e.g. foo') }'}} --><!-- /ko -->
+
+        <!-- ko template: { if: typeof properties().parameters != 'undefined', name: 'property', data: { type: 'csv', label: '${ _('Parameters') }', value: properties().parameters, title: '${ _('Names and values of Pig parameters and options') }', placeholder: '${ _('e.g. input /user/data, -param input=/user/data, -optimizer_off SplitFilter, -verbose') }'}} --><!-- /ko -->
+        <!-- ko template: { if: typeof properties().hadoopProperties != 'undefined', name: 'property', data: { type: 'csv', label: '${ _('Hadoop properties') }', value: properties().hadoopProperties, title: '${ _('Name and values of Hadoop properties') }', placeholder: '${ _('e.g. mapred.job.queue.name=production, mapred.map.tasks.speculative.execution=false') }'}} --><!-- /ko -->
+        <!-- ko template: { if: typeof properties().resources != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _('Resources') }', value: properties().resources, title: '${ _('HDFS Files or compressed files') }', placeholder: '${ _('e.g. /tmp/file, /tmp.file.zip') }'}} --><!-- /ko -->
       </form>
     </div>
     <a class="pointer demi-modal-chevron" data-bind="click: function() { settingsVisible(! settingsVisible()) }"><i class="fa fa-chevron-up"></i></a>