Эх сурвалжийг харах

HUE-1130 [pig] UDF support

Add jars as files, then 'register my.jar' in Pig.
Resources: files (aka jar or UDF), archives.
Romain Rigaux 12 жил өмнө
parent
commit
b20347e

+ 21 - 2
apps/pig/src/pig/api.py

@@ -72,7 +72,25 @@ class OozieApi:
       pig_params.append({"type":"argument","value":"-param"})
       pig_params.append({"type":"argument","value":"%(name)s=%(value)s" % param})
 
-    action = Pig.objects.create(name='pig', script_path=script_path, workflow=workflow, node_type='pig', params=json.dumps(pig_params))
+    files = []
+    archives = []
+
+    for resource in pig_script.dict['resources']:
+      if resource['type'] == 'file':
+        files.append(resource['value'])
+      if resource['type'] == 'archive':
+        archives.append({"dummy": "", "name": resource['value']})
+
+    action = Pig.objects.create(
+        name='pig',
+        script_path=script_path,
+        workflow=workflow,
+        node_type='pig',
+        params=json.dumps(pig_params),
+        files=json.dumps(files),
+        archives=json.dumps(archives),
+    )
+
     action.add_node(workflow.end)
 
     start_link = workflow.start.get_link()
@@ -109,7 +127,8 @@ class OozieApi:
         'status': action.status,
         'logs': logs.get(action.name, ''),
         'progress': oozie_workflow.get_progress(),
-        'progressPercent': '%d%%' % oozie_workflow.get_progress()
+        'progressPercent': '%d%%' % oozie_workflow.get_progress(),
+        'absoluteUrl': oozie_workflow.get_absolute_url(),
       }
       workflow_actions.append(appendable)
 

+ 20 - 8
apps/pig/src/pig/models.py

@@ -47,15 +47,22 @@ class Document(models.Model):
 
 
 class PigScript(Document):
-  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters']
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters', 'resources']
 
-  data = models.TextField(default=json.dumps({'script': '', 'name': '', 'properties': [], 'job_id': None, 'parameters': []}))
+  data = models.TextField(default=json.dumps({
+      'script': '',
+      'name': '',
+      'properties': [],
+      'job_id': None,
+      'parameters': [],
+      'resources': []
+  }))
 
   def update_from_dict(self, attrs):
     data_dict = self.dict
 
     for attr in PigScript._ATTRIBUTES:
-      if attrs.get(attr):
+      if attrs.get(attr) is not None:
         data_dict[attr] = attrs[attr]
 
     self.data = json.dumps(data_dict)
@@ -70,16 +77,20 @@ class Submission(models.Model):
   workflow = models.ForeignKey(Workflow)
 
 
-def create_or_update_script(id, name, script, user, parameters, is_design=True):
-  """Take care of security"""
+def create_or_update_script(id, name, script, user, parameters, resources, is_design=True):
+  """This take care of security"""
   try:
     pig_script = PigScript.objects.get(id=id)
     pig_script.can_edit_or_exception(user)
   except:
     pig_script = PigScript.objects.create(owner=user, is_design=is_design)
 
-  pig_script.update_from_dict({'name': name, 'script': script, 'parameters': parameters})
-  pig_script.save()
+  pig_script.update_from_dict({
+      'name': name,
+      'script': script,
+      'parameters': parameters,
+      'resources': resources
+  })
 
   return pig_script
 
@@ -88,12 +99,13 @@ def get_scripts(user, max_count=200):
   scripts = []
 
   for script in PigScript.objects.filter(owner=user).order_by('-id')[:max_count]:
-    data = json.loads(script.data)
+    data = script.dict
     massaged_script = {
       'id': script.id,
       'name': data['name'],
       'script': data['script'],
       'parameters': data['parameters'],
+      'resources': data['resources'],
       'isDesign': script.is_design,
     }
     scripts.append(massaged_script)

+ 75 - 13
apps/pig/src/pig/templates/app.mako

@@ -28,7 +28,6 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       <li class="active"><a href="#editor">${ _('Editor') }</a></li>
       <li><a href="#scripts">${ _('Scripts') }</a></li>
       <li><a href="#dashboard">${ _('Dashboard') }</a></li>
-      ##<li class="${utils.is_selected(section, 'udfs')}"><a href="${ url('pig:udfs') }">${ _('UDF') }</a></li>
       </ul>
   </div>
 </div>
@@ -115,9 +114,9 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
             <li data-bind="click: editScriptProperties" data-section="properties">
               <a href="#"><i class="icon-reorder"></i> ${ _('Edit properties') }</a>
             </li>
-            ##<li class="nav-header">${_('UDF')}</li>
+            ##<li class="nav-header">${_('Python UDF')}</li>
             ##<li><a href="#createDataset">${ _('New') }</a></li>
-            ##<li><a href="#createDataset">${ _('Add') }</a></li>
+            ##<li><a href="#createDataset">${ _('List') }</a></li>
             <li class="nav-header">${_('Actions')}</li>
             <li data-bind="click: saveScript">
               <a href="#" title="${ _('Save the script') }" rel="tooltip" data-placement="right">
@@ -200,6 +199,51 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
               </tfoot>
             </table>
           </div>
+          <br/>
+          <label>${ _('Resources') } &nbsp;
+            <button class="btn" data-bind="click: currentScript().addResource, visible: currentScript().resources().length == 0" style="margin-left: 4px">
+              <i class="icon-plus"></i> ${ _('Add') }
+            </button>
+          </label>
+          <div>
+            <table data-bind="css: {'parameterTable': currentScript().resources().length > 0}">
+              <thead data-bind="visible: currentScript().resources().length > 0">
+                <tr>
+                  <th>${ _('Type') }</th>
+                  <th>${ _('Value') }</th>
+                  <th>&nbsp;</th>
+                </tr>
+              </thead>
+              <tbody data-bind="foreach: currentScript().resources">
+                <tr>
+                  <td>
+                    <select type="text" data-bind="value: type" class="input-large">
+                      ##<option value="udf">${ _('UDF') }</option>
+                      <option value="file">${ _('File') }</option>
+                      <option value="archive">${ _('Archive') }</option>
+                    </select>
+                  </td>
+                  <td>
+                    <div class="input-append">
+                      <input type="text" data-bind="value: value" class="input-xxlarge" />
+                      <button class="btn fileChooserBtn" data-bind="click: $root.showFileChooser">..</button>
+                    </div>
+                  </td>
+                  <td>
+                    <button data-bind="click: viewModel.currentScript().removeResource" class="btn">
+                    <i class="icon-trash"></i> ${ _('Remove') }</button>
+                  </td>
+                </tr>
+              </tbody>
+              <tfoot data-bind="visible: currentScript().resources().length > 0">
+                <tr>
+                  <td colspan="3">
+                    <button class="btn" data-bind="click: currentScript().addResource"><i class="icon-plus"></i> ${ _('Add') }</button>
+                  </td>
+                </tr>
+              </tfoot>
+            </table>
+          </div>
         </form>
       </div>
 
@@ -211,11 +255,13 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         <div data-bind="template: {name: 'logTemplate', foreach: currentScript().actions}"></div>
         <script id="logTemplate" type="text/html">
           <div data-bind="css:{'alert-modified': name != '', 'alert': name != '', 'alert-success': status == 'SUCCEEDED' || status == 'OK', 'alert-error': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}">
-            <div class="pull-right" data-bind="text: status"></div>
-              <h4>${ _('Progress:') } <span data-bind="text: progress"></span>${ _('%') }</h4>
-              <div data-bind="css: {'progress': name != '', 'progress-striped': name != '', 'active': status == 'RUNNING'}" style="margin-top:10px">
-                <div data-bind="css: {'bar': name != '', 'bar-success': status == 'SUCCEEDED' || status == 'OK', 'bar-warning': status == 'RUNNING' || status == 'PREP', 'bar-danger': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}, attr: {'style': 'width:' + progressPercent}"></div>
-              </div>
+            <div class="pull-right">
+              <i class="icon-share-alt"></i> <a data-bind="text: status, visible: absoluteUrl != '', attr: {'href': absoluteUrl}" target="_blank"/>
+            </div>
+            <h4>${ _('Progress:') } <span data-bind="text: progress"></span>${ _('%') }</h4>
+            <div data-bind="css: {'progress': name != '', 'progress-striped': name != '', 'active': status == 'RUNNING'}" style="margin-top:10px">
+              <div data-bind="css: {'bar': name != '', 'bar-success': status == 'SUCCEEDED' || status == 'OK', 'bar-warning': status == 'RUNNING' || status == 'PREP', 'bar-danger': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}, attr: {'style': 'width:' + progressPercent}"></div>
+            </div>
           </div>
           <pre data-bind="visible: logs == ''">${ _('No available logs.') }</pre>
           <pre data-bind="visible: logs != '', text: logs"></pre>
@@ -226,7 +272,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
   </div>
 
   <div id="dashboard" class="row-fluid mainSection hide">
-    <h3>Running</h3>
+    <h3>${ _('Running') }</h3>
     <div class="alert alert-info" data-bind="visible: runningScripts().length == 0">
       ${_('There are currently no running scripts.')}
     </div>
@@ -243,7 +289,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       </tbody>
     </table>
 
-    <h3>Completed</h3>
+    <h3>${ _('Completed') }</h3>
     <div class="alert alert-info" data-bind="visible: completedScripts().length == 0">
       ${_('There are currently no completed scripts.')}
     </div>
@@ -313,6 +359,18 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
   </div>
 </div>
 
+<div id="chooseFile" class="modal hide fade">
+    <div class="modal-header">
+        <a href="#" class="close" data-dismiss="modal">&times;</a>
+        <h3>${_('Choose a file')}</h3>
+    </div>
+    <div class="modal-body">
+        <div id="filechooser">
+        </div>
+    </div>
+    <div class="modal-footer">
+    </div>
+</div>
 
 <div class="bottomAlert alert"></div>
 
@@ -328,6 +386,11 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 <script src="/static/js/Source/jHue/codemirror-pig-hint.js"></script>
 <link rel="stylesheet" href="/static/ext/css/codemirror-show-hint.css">
 
+<style>
+  .fileChooserBtn {
+    border-radius: 0 3px 3px 0;
+  }
+</style>
 
 <script type="text/javascript" charset="utf-8">
 
@@ -338,7 +401,8 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
     SAVED: "${ _('Saved') }",
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
     NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';",
-    NEW_SCRIPT_PARAMETERS: []
+    NEW_SCRIPT_PARAMETERS: [],
+    NEW_SCRIPT_RESOURCES: []
   };
 
   var scripts = ${ scripts | n,unicode };
@@ -356,7 +420,6 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
   ko.applyBindings(viewModel);
 
   $(document).ready(function () {
-
     var scriptEditor = $("#scriptEditor")[0];
 
     CodeMirror.commands.autocomplete = function(cm) {
@@ -588,7 +651,6 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         showSection("editor", "logs");
       }
     });
-
   });
 
   var _bottomAlertFade = -1;

+ 9 - 1
apps/pig/src/pig/tests.py

@@ -31,7 +31,15 @@ class TestPigBase(object):
     self.user = User.objects.get(username='test')
 
   def create_script(self):
-    return create_or_update_script(10000, 'Test', 'A = LOAD "$data"; STOPE A INTO "$output";', self.user)
+    attrs = {
+      'id': 1000,
+      'name': 'Test',
+      'script': 'A = LOAD "$data"; STORE A INTO "$output";',
+      'user': self.user,
+      'parameters': [],
+      'resources': [],
+    }
+    return create_or_update_script(**attrs)
 
 
 class TestMock(TestPigBase):

+ 7 - 1
apps/pig/src/pig/views.py

@@ -71,6 +71,7 @@ def save(request):
     'script': request.POST.get('script'),
     'user': request.user,
     'parameters': json.loads(request.POST.get('parameters')),
+    'resources': json.loads(request.POST.get('resources')),
   }
   pig_script = create_or_update_script(**attrs)
   pig_script.is_design = True
@@ -86,17 +87,22 @@ def save(request):
 
 @show_oozie_error
 def run(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
   attrs = {
     'id': request.POST.get('id'),
     'name': request.POST.get('name'),
     'script': request.POST.get('script'),
     'user': request.user,
     'parameters': json.loads(request.POST.get('parameters')),
+    'resources': json.loads(request.POST.get('resources')),
     'is_design': False
   }
+
   pig_script = create_or_update_script(**attrs)
-  params = request.POST.get('parameters')
 
+  params = request.POST.get('parameters')
   oozie_id = api.get(request.fs, request.user).submit(pig_script, params)
 
   pig_script.update_from_dict({'job_id': oozie_id})

+ 36 - 3
apps/pig/static/js/pig.ko.js

@@ -14,6 +14,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+var Resource = function (resource) {
+  var self = this;
+
+  self.type = ko.observable(resource.type);
+  self.value = ko.observable(resource.value);
+};
 
 var PigScript = function (pigScript) {
   var self = this;
@@ -57,6 +63,16 @@ var PigScript = function (pigScript) {
     }
     return params;
   };
+  self.resources = ko.observableArray([]);
+  ko.utils.arrayForEach(pigScript.resources, function (resource) {
+    self.resources.push(new Resource({type: resource.type, value: resource.value}));
+  });
+  self.addResource = function () {
+    self.resources.push(new Resource({type: 'file', value: ''}));
+  };
+  self.removeResource = function () {
+    self.resources.remove(this);
+  };
 }
 
 var Workflow = function (wf) {
@@ -111,6 +127,7 @@ var PigViewModel = function (scripts, props) {
     name: self.LABELS.NEW_SCRIPT_NAME,
     script: self.LABELS.NEW_SCRIPT_CONTENT,
     parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
+    resources: self.LABELS.NEW_SCRIPT_RESOURCES
   };
 
   self.currentScript = ko.observable(new PigScript(_defaultScript));
@@ -260,7 +277,21 @@ var PigViewModel = function (scripts, props) {
       keyboard: true,
       show: true
     });
-  }
+  };
+
+  self.showFileChooser = function showFileChooser() {
+    var inputPath = this;
+    var path = inputPath.value().substr(0, inputPath.value().lastIndexOf("/"));
+    $("#filechooser").jHueFileChooser({
+      initialPath: path,
+      onFileChoose: function (filePath) {
+        inputPath.value(filePath);
+        $("#chooseFile").modal("hide");
+      },
+      createFolder: false
+    });
+    $("#chooseFile").modal("show");
+  };
 
   function showDeleteModal() {
     $(".deleteMsg").addClass("hide");
@@ -288,7 +319,8 @@ var PigViewModel = function (scripts, props) {
           id: script.id(),
           name: script.name(),
           script: script.script(),
-          parameters: ko.utils.stringifyJson(script.parameters())
+          parameters: ko.utils.stringifyJson(script.parameters()),
+          resources: ko.toJSON(script.resources())
         },
         function (data) {
           self.currentScript().id(data.id);
@@ -304,7 +336,8 @@ var PigViewModel = function (scripts, props) {
           id: script.id(),
           name: script.name(),
           script: script.script(),
-            parameters: ko.utils.stringifyJson(self.submissionVariables())
+          parameters: ko.utils.stringifyJson(self.submissionVariables()),
+          resources: ko.toJSON(script.resources())
         },
         function (data) {
           if (data.id && self.currentScript().id() != data.id){