浏览代码

[pig] Various UX improvements

Link Pig script to running job on the Dashboard
Show snippet of the script
Fix log retrieval from Oozie
Move the tabs
Romain Rigaux 12 年之前
父节点
当前提交
296c421

+ 31 - 22
apps/pig/src/pig/api.py

@@ -30,48 +30,48 @@ LOG = logging.getLogger(__name__)
 
 class Api:
   WORKFLOW_NAME = 'pig-app-hue-script'
-  RE_LOGS = re.compile('>>> Invoking Pig command line now >>>'
-                       '\n\n\nRun pig script using PigRunner.run\(\) for Pig version .+?\n'
-                       '(?P<pig>.*?)'
-                       '(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.DOTALL | re.M)
-  
+  RE_LOG_END = re.compile('(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed <<<)')
+  RE_LOG_START_RUNNING = re.compile('>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?\n(.+?)(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.M | re.DOTALL)
+  RE_LOG_START_FINISHED = re.compile('(>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?)\n', re.M | re.DOTALL)
+
+
   def __init__(self, fs, user):
     self.fs = fs
     self.user = user
-    
-  def submit(self, pig_script, mapping):    
+
+  def submit(self, pig_script, mapping):
     workflow = Workflow.objects.new_workflow(self.user)
     workflow.name = Api.WORKFLOW_NAME
     workflow.is_history = True
     workflow.save()
     Workflow.objects.initialize(workflow, self.fs)
-    
+
     script_path = workflow.deployment_dir + '/script.pig'
     self.fs.create(script_path, data=pig_script.dict['script'])
-    
+
     action = Pig.objects.create(name='pig', script_path=script_path, workflow=workflow, node_type='pig')
     action.add_node(workflow.end)
-    
+
     start_link = workflow.start.get_link()
     start_link.child = action
-    start_link.save()    
-    
+    start_link.save()
+
     return _submit_workflow(self.user, self.fs, workflow, mapping)
-  
-  def get_log(self, request, oozie_workflow):    
+
+  def get_log(self, request, oozie_workflow):
     logs = {}
-  
+
     for action in oozie_workflow.get_working_actions():
       try:
         if action.externalId:
-          log = job_single_logs(request, **{'job': action.externalId})  
+          log = job_single_logs(request, **{'job': action.externalId})
           if log:
-            logs[action.name] = Api.RE_LOGS.search(log['logs'][1]).group(1)
+            logs[action.name] = self.match_logs(log['logs'][1])
       except Exception, e:
         LOG.error('An error happen while watching the demo running: %(error)s' % {'error': e})
-  
+
     workflow_actions = []
-    
+
     if oozie_workflow.get_working_actions():
       for action in oozie_workflow.get_working_actions():
         appendable = {
@@ -79,6 +79,15 @@ class Api:
           'status': action.status,
           'logs': logs.get(action.name, '')
         }
-        workflow_actions.append(appendable)   
-      
-    return logs, workflow_actions 
+        workflow_actions.append(appendable)
+
+    return logs, workflow_actions
+
+  def match_logs(self, logs):
+    """Difficult to match multi lines of text"""
+    if Api.RE_LOG_END.search(logs):
+      return re.search(Api.RE_LOG_START_RUNNING, logs).group(1)
+    else:
+      group = re.search(Api.RE_LOG_START_FINISHED, logs)
+      i = logs.index(group.group(1)) + len(group.group(1))
+      return logs[i:]

+ 15 - 12
apps/pig/src/pig/models.py

@@ -30,44 +30,47 @@ from django.utils.translation import ugettext as _, ugettext_lazy as _t
 from oozie.models import Workflow
 
 
-# To move somewhere else when official commit
 class Document(models.Model):
-  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Person who can modify the job.'))
-  is_history = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is managed'),
+  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can modify the job.'))
+  is_history = models.BooleanField(default=True, db_index=True, verbose_name=_t('Is a submitted job'),
                                   help_text=_t('If the job should show up in the history'))
 
 
 class PigScript(Document):
-  _ATTRIBUTES = ['script', 'name', 'properties']
-  
-  data = models.TextField(default=json.dumps({'script': '', 'name': ''}))
-    
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id']
+
+  data = models.TextField(default=json.dumps({'script': '', 'name': '', 'properties': [], 'job_id': None}))
+
   def update_from_dict(self, attrs):
     data_dict = self.dict
-    
+
     if attrs.get('script'):
       data_dict['script'] = attrs['script']
 
     if attrs.get('name'):
       data_dict['name'] = attrs['name']
 
+    if attrs.get('job_id'):
+      data_dict['job_id'] = attrs['job_id']
+
     self.data = json.dumps(data_dict)
-    
+
   @property
   def dict(self):
     return json.loads(self.data)
-  
-  
+
+
 class Submission(models.Model):
   script = models.ForeignKey(PigScript)
   workflow = models.ForeignKey(Workflow)
-  
+
 
 class Udf:
   pass
 
 
 def get_workflow_output(oozie_workflow, fs):
+  # TODO: guess from the STORE
   output = None
 
   if 'workflowRoot' in oozie_workflow.conf_dict:

+ 11 - 11
apps/pig/src/pig/templates/app.mako

@@ -25,8 +25,8 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 <div class="subnav subnav-fixed">
   <div class="container-fluid">
     <ul class="nav nav nav-pills">
-      <li class="active"><a href="#scripts">${ _('Scripts') }</a></li>
-      <li><a href="#editor">${ _('Editor') }</a></li>
+      <li class="active"><a href="#editor">${ _('Editor') }</a></li>
+      <li><a href="#scripts">${ _('Scripts') }</a></li>
       <li><a href="#dashboard">${ _('Dashboard') }</a></li>
       ##<li class="${utils.is_selected(section, 'udfs')}"><a href="${ url('pig:udfs') }">${ _('UDF') }</a></li>
       </ul>
@@ -91,7 +91,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
           <strong><a href="#" data-bind="click: $root.viewScript, text: name"></a></strong>
         </td>
         <td data-bind="click: $root.viewScript">
-          <span data-bind="text: script"></span>
+          <span data-bind="text: scriptSumup"></span>
         </td>
       </tr>
     </script>
@@ -167,7 +167,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         <div data-bind="template: {name: 'logTemplate', foreach: currentScript().actions}"></div>
         <script id="logTemplate" type="text/html">
           <div data-bind="css:{'alert-modified': name != '', 'alert': name != '', 'alert-success': status == 'SUCCEEDED' || status == 'OK', 'alert-error': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}">
-            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Action') } '<span data-bind="text: name"></span>'</h4></div>
+            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Progress: 100%') } '<span data-bind="text: name"></span>'</h4></div>
           <pre data-bind="visible: logs == ''">${ _('No available logs.') }</pre>
           <pre data-bind="visible: logs != '', text: logs"></pre>
         </script>
@@ -202,7 +202,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       <thead>
       <tr>
         <th width="20%">${_('Name')}</th>
-        <th width="10%">${_('Status')}</th>
+        <th width="20%">${_('Status')}</th>
         <th width="">${_('Created on')}</th>
       </tr>
       </thead>
@@ -213,14 +213,14 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 
     <script id="dashboardTemplate" type="text/html">
       <tr style="cursor: pointer">
-        <td>
-          <strong><a data-bind="text: appName, attr: {'href': absoluteUrl}" target="_blank"></a></strong>
+        <td data-bind="click: $root.viewSubmittedScript" title="${_('Click to edit')}">
+          <strong><a data-bind="text: appName"></a></strong>
         </td>
         <td>
           <span data-bind="attr: {'class': statusClass}, text: status"></span>
         </td>
         <td>
-          <span data-bind="text: created"></span>
+          <strong><a data-bind="text: created, attr: {'href': absoluteUrl}" target="_blank"></a></strong>
         </td>
       </tr>
     </script>
@@ -266,7 +266,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
     TOOLTIP_STOP: "${ _('Stop the execution') }",
     SAVED: "${ _('Saved') }",
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
-    NEW_SCRIPT_CONTENT: "ie. a = LOAD '/user/${ user }/data';"
+    NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';"
   };
 
   var scripts = ${ scripts | n,unicode };
@@ -310,7 +310,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       viewModel.currentScript().script(codeMirror.getValue());
     });
 
-    showMainSection("scripts");
+    showMainSection("editor");
 
     $(document).on("loadEditor", function () {
       codeMirror.setValue(viewModel.currentScript().script());
@@ -424,7 +424,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
           else {
             viewModel.currentScript().isRunning(false);
             $(document).trigger("stopLogsRefresh");
-            $(document).trigger("showEditor");
+            //$(document).trigger("showEditor");
           }
         });
       }

+ 4 - 4
apps/pig/src/pig/templates/udfs.mako

@@ -26,17 +26,17 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 
 <div class="container-fluid">
    ${ navigation.menubar(section='udfs') }
-  
+
     <div class="tab-content">
       <div class="tab-pane active">
 
 		<div class="container-fluid">
-		    <div class="row-fluid">         
-		      No UDFS         
+		    <div class="row-fluid">
+		      No UDFS
 		    </div>
 		</div>
 
-   
+
       </div>
 
     </div>

+ 37 - 25
apps/pig/src/pig/views.py

@@ -23,12 +23,11 @@ import logging
 import time
 
 from django.core.urlresolvers import reverse
+from django.http import HttpResponse
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_http_methods
-from django.http import HttpResponse
 
-from desktop.lib.django_util import render, encode_json_for_js
-from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.django_util import render
 from desktop.lib.view_util import format_duration_in_millis
 from liboozie.oozie_api import get_oozie
 from oozie.views.dashboard import show_oozie_error, check_job_access_permission
@@ -39,20 +38,22 @@ from pig.models import get_workflow_output, hdfs_link, PigScript
 
 LOG = logging.getLogger(__name__)
 
+
 def app(request):
   return render('app.mako', request, {
-    'scripts': json.dumps(get_scripts())
+    'scripts': json.dumps(get_scripts(is_history=True))
     }
   )
 
+
 def scripts(request):
   return HttpResponse(json.dumps(get_scripts()), mimetype="application/json")
 
 
-def get_scripts():
+def get_scripts(is_history=True):
   scripts = []
 
-  for script in PigScript.objects.filter(is_history=False):
+  for script in PigScript.objects.filter(is_history=is_history):
     data = json.loads(script.data)
     massaged_script = {
       'id': script.id,
@@ -63,6 +64,7 @@ def get_scripts():
 
   return scripts
 
+
 @show_oozie_error
 def dashboard(request):
   kwargs = {'cnt': 100,}
@@ -70,7 +72,9 @@ def dashboard(request):
   kwargs['name'] = Api.WORKFLOW_NAME
 
   jobs = get_oozie().get_workflows(**kwargs).jobs
-  return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(jobs, request.user)), mimetype="application/json")
+  hue_jobs = PigScript.objects.filter(owner=request.user)
+
+  return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(jobs, hue_jobs, request.user)), mimetype="application/json")
 
 
 def udfs(request):
@@ -80,8 +84,8 @@ def udfs(request):
 @require_http_methods(["POST"])
 def save(request):
   # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
-  
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=True)
+
   response = {
     'id': pig_script.id,
   }
@@ -93,15 +97,18 @@ def save(request):
 @show_oozie_error
 def run(request):
   # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=False)
 
-  # Todo, will come from script properties later
+  # TODO: will come from script properties later
   mapping = {
     'oozie.use.system.libpath':  'true',
-  }  
+  }
 
   oozie_id = Api(request.fs, request.user).submit(pig_script, mapping)
 
+  pig_script.update_from_dict({'job_id': oozie_id})
+  pig_script.save()
+
   response = {
     'id': pig_script.id,
     'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python'
@@ -109,6 +116,7 @@ def run(request):
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
+
 @require_http_methods(["POST"])
 def copy(request):
   # TODO security
@@ -124,10 +132,11 @@ def copy(request):
     'id': pig_script.id,
     'name': name,
     'script': script
-    }
+  }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
+
 @require_http_methods(["POST"])
 def delete(request):
   # TODO security
@@ -147,20 +156,21 @@ def delete(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-def create_or_update_script(id, name, script, user):
+def create_or_update_script(id, name, script, user, is_history=True):
   try:
     pig_script = PigScript.objects.get(id=id)
   except:
-    pig_script = PigScript.objects.create(owner=user)
+    pig_script = PigScript.objects.create(owner=user, is_history=is_history)
 
   pig_script.update_from_dict({'name': name, 'script': script})
   pig_script.save()
 
   return pig_script
 
+
 @show_oozie_error
 def watch(request, job_id):
-  oozie_workflow = check_job_access_permission(request, job_id) 
+  oozie_workflow = check_job_access_permission(request, job_id)
   logs, workflow_actions = Api(request, job_id).get_log(request, oozie_workflow)
   output = get_workflow_output(oozie_workflow, request.fs)
 
@@ -179,7 +189,7 @@ def watch(request, job_id):
     'logs': logs,
     'output': hdfs_link(output)
   }
-  
+
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
@@ -189,23 +199,24 @@ def format_time(st_time):
   else:
     return time.strftime("%a, %d %b %Y %H:%M:%S", st_time)
 
+
 def has_job_edition_permission(oozie_job, user):
   return user.is_superuser or oozie_job.user == user.username
 
+
 def has_dashboard_jobs_access(user):
   return user.is_superuser or user.has_hue_permission(action="dashboard_jobs_access", app=DJANGO_APPS[0])
 
-def massaged_oozie_jobs_for_json(oozie_jobs, user):
+
+def massaged_oozie_jobs_for_json(oozie_jobs, hue_jobs, user):
   jobs = []
+  hue_jobs = dict([(script.dict.get('job_id'), script) for script in hue_jobs if script.dict.get('job_id')])
 
   for job in oozie_jobs:
     if job.is_running():
-      if job.type == 'Workflow':
-        job = get_oozie().get_job(job.id)
-      elif job.type == 'Coordinator':
-        job = get_oozie().get_coordinator(job.id)
-      else:
-        job = get_oozie().get_bundle(job.id)
+      job = get_oozie().get_job(job.id)
+
+    script = hue_jobs.get(job.id) and hue_jobs.get(job.id) or None
 
     massaged_job = {
       'id': job.id,
@@ -216,7 +227,8 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user):
       'status': job.status,
       'isRunning': job.is_running(),
       'duration': job.endTime and job.startTime and format_duration_in_millis(( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000) or None,
-      'appName': job.appName,
+      'appName': script and script.dict['name'] or _('Unsaved script'),
+      'scriptId': script and script.id or -1,
       'progress': job.get_progress(),
       'user': job.user,
       'absoluteUrl': job.get_absolute_url(),

+ 8 - 0
apps/pig/static/js/pig.ko.js

@@ -20,6 +20,7 @@ var PigScript = function (pigScript) {
     id: ko.observable(pigScript.id),
     name: ko.observable(pigScript.name),
     script: ko.observable(pigScript.script),
+    scriptSumup: ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100)),
     isRunning: ko.observable(false),
     selected: ko.observable(false),
     watchUrl: ko.observable(""),
@@ -37,6 +38,7 @@ var PigScript = function (pigScript) {
 var Workflow = function (wf) {
   return {
     id: wf.id,
+    scriptId: wf.scriptId,
     lastModTime: wf.lastModTime,
     endTime: wf.endTime,
     status: wf.status,
@@ -302,4 +304,10 @@ var PigViewModel = function (scripts, props) {
           $("#deleteModal").modal("hide");
         }, "json");
   }
+
+  self.viewSubmittedScript = function (workflow) {
+    self.loadScript(workflow.scriptId);
+    $(document).trigger("loadEditor");
+    $(document).trigger("showEditor");
+  };
 };