소스 검색

[pig] Various UX improvements

Link Pig script to running job on the Dashboard
Show snippet of the script
Fix log retrieval from Oozie
Move the tabs
Romain Rigaux 12 년 전
부모
커밋
296c421ffa
6개의 변경된 파일106개의 추가작업 그리고 74개의 파일을 삭제
  1. 31 22
      apps/pig/src/pig/api.py
  2. 15 12
      apps/pig/src/pig/models.py
  3. 11 11
      apps/pig/src/pig/templates/app.mako
  4. 4 4
      apps/pig/src/pig/templates/udfs.mako
  5. 37 25
      apps/pig/src/pig/views.py
  6. 8 0
      apps/pig/static/js/pig.ko.js

+ 31 - 22
apps/pig/src/pig/api.py

@@ -30,48 +30,48 @@ LOG = logging.getLogger(__name__)
 
 class Api:
   WORKFLOW_NAME = 'pig-app-hue-script'
-  RE_LOGS = re.compile('>>> Invoking Pig command line now >>>'
-                       '\n\n\nRun pig script using PigRunner.run\(\) for Pig version .+?\n'
-                       '(?P<pig>.*?)'
-                       '(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.DOTALL | re.M)
-  
+  RE_LOG_END = re.compile('(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed <<<)')
+  RE_LOG_START_RUNNING = re.compile('>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?\n(.+?)(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.M | re.DOTALL)
+  RE_LOG_START_FINISHED = re.compile('(>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?)\n', re.M | re.DOTALL)
+
+
   def __init__(self, fs, user):
     self.fs = fs
     self.user = user
-    
-  def submit(self, pig_script, mapping):    
+
+  def submit(self, pig_script, mapping):
     workflow = Workflow.objects.new_workflow(self.user)
     workflow.name = Api.WORKFLOW_NAME
     workflow.is_history = True
     workflow.save()
     Workflow.objects.initialize(workflow, self.fs)
-    
+
     script_path = workflow.deployment_dir + '/script.pig'
     self.fs.create(script_path, data=pig_script.dict['script'])
-    
+
     action = Pig.objects.create(name='pig', script_path=script_path, workflow=workflow, node_type='pig')
     action.add_node(workflow.end)
-    
+
     start_link = workflow.start.get_link()
     start_link.child = action
-    start_link.save()    
-    
+    start_link.save()
+
     return _submit_workflow(self.user, self.fs, workflow, mapping)
-  
-  def get_log(self, request, oozie_workflow):    
+
+  def get_log(self, request, oozie_workflow):
     logs = {}
-  
+
     for action in oozie_workflow.get_working_actions():
       try:
         if action.externalId:
-          log = job_single_logs(request, **{'job': action.externalId})  
+          log = job_single_logs(request, **{'job': action.externalId})
           if log:
-            logs[action.name] = Api.RE_LOGS.search(log['logs'][1]).group(1)
+            logs[action.name] = self.match_logs(log['logs'][1])
       except Exception, e:
         LOG.error('An error happen while watching the demo running: %(error)s' % {'error': e})
-  
+
     workflow_actions = []
-    
+
     if oozie_workflow.get_working_actions():
       for action in oozie_workflow.get_working_actions():
         appendable = {
@@ -79,6 +79,15 @@ class Api:
           'status': action.status,
           'logs': logs.get(action.name, '')
         }
-        workflow_actions.append(appendable)   
-      
-    return logs, workflow_actions 
+        workflow_actions.append(appendable)
+
+    return logs, workflow_actions
+
+  def match_logs(self, logs):
+    """Difficult to match multi lines of text"""
+    if Api.RE_LOG_END.search(logs):
+      return re.search(Api.RE_LOG_START_RUNNING, logs).group(1)
+    else:
+      group = re.search(Api.RE_LOG_START_FINISHED, logs)
+      i = logs.index(group.group(1)) + len(group.group(1))
+      return logs[i:]

+ 15 - 12
apps/pig/src/pig/models.py

@@ -30,44 +30,47 @@ from django.utils.translation import ugettext as _, ugettext_lazy as _t
 from oozie.models import Workflow
 
 
-# To move somewhere else when official commit
 class Document(models.Model):
-  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Person who can modify the job.'))
-  is_history = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is managed'),
+  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can modify the job.'))
+  is_history = models.BooleanField(default=True, db_index=True, verbose_name=_t('Is a submitted job'),
                                   help_text=_t('If the job should show up in the history'))
 
 
 class PigScript(Document):
-  _ATTRIBUTES = ['script', 'name', 'properties']
-  
-  data = models.TextField(default=json.dumps({'script': '', 'name': ''}))
-    
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id']
+
+  data = models.TextField(default=json.dumps({'script': '', 'name': '', 'properties': [], 'job_id': None}))
+
   def update_from_dict(self, attrs):
     data_dict = self.dict
-    
+
     if attrs.get('script'):
       data_dict['script'] = attrs['script']
 
     if attrs.get('name'):
       data_dict['name'] = attrs['name']
 
+    if attrs.get('job_id'):
+      data_dict['job_id'] = attrs['job_id']
+
     self.data = json.dumps(data_dict)
-    
+
   @property
   def dict(self):
     return json.loads(self.data)
-  
-  
+
+
 class Submission(models.Model):
   script = models.ForeignKey(PigScript)
   workflow = models.ForeignKey(Workflow)
-  
+
 
 class Udf:
   pass
 
 
 def get_workflow_output(oozie_workflow, fs):
+  # TODO: guess from the STORE
   output = None
 
   if 'workflowRoot' in oozie_workflow.conf_dict:

+ 11 - 11
apps/pig/src/pig/templates/app.mako

@@ -25,8 +25,8 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 <div class="subnav subnav-fixed">
   <div class="container-fluid">
     <ul class="nav nav nav-pills">
-      <li class="active"><a href="#scripts">${ _('Scripts') }</a></li>
-      <li><a href="#editor">${ _('Editor') }</a></li>
+      <li class="active"><a href="#editor">${ _('Editor') }</a></li>
+      <li><a href="#scripts">${ _('Scripts') }</a></li>
       <li><a href="#dashboard">${ _('Dashboard') }</a></li>
       ##<li class="${utils.is_selected(section, 'udfs')}"><a href="${ url('pig:udfs') }">${ _('UDF') }</a></li>
       </ul>
@@ -91,7 +91,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
           <strong><a href="#" data-bind="click: $root.viewScript, text: name"></a></strong>
         </td>
         <td data-bind="click: $root.viewScript">
-          <span data-bind="text: script"></span>
+          <span data-bind="text: scriptSumup"></span>
         </td>
       </tr>
     </script>
@@ -167,7 +167,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         <div data-bind="template: {name: 'logTemplate', foreach: currentScript().actions}"></div>
         <script id="logTemplate" type="text/html">
           <div data-bind="css:{'alert-modified': name != '', 'alert': name != '', 'alert-success': status == 'SUCCEEDED' || status == 'OK', 'alert-error': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}">
-            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Action') } '<span data-bind="text: name"></span>'</h4></div>
+            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Progress: 100%') } '<span data-bind="text: name"></span>'</h4></div>
           <pre data-bind="visible: logs == ''">${ _('No available logs.') }</pre>
           <pre data-bind="visible: logs != '', text: logs"></pre>
         </script>
@@ -202,7 +202,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       <thead>
       <tr>
         <th width="20%">${_('Name')}</th>
-        <th width="10%">${_('Status')}</th>
+        <th width="20%">${_('Status')}</th>
         <th width="">${_('Created on')}</th>
       </tr>
       </thead>
@@ -213,14 +213,14 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 
     <script id="dashboardTemplate" type="text/html">
       <tr style="cursor: pointer">
-        <td>
-          <strong><a data-bind="text: appName, attr: {'href': absoluteUrl}" target="_blank"></a></strong>
+        <td data-bind="click: $root.viewSubmittedScript" title="${_('Click to edit')}">
+          <strong><a data-bind="text: appName"></a></strong>
         </td>
         <td>
           <span data-bind="attr: {'class': statusClass}, text: status"></span>
         </td>
         <td>
-          <span data-bind="text: created"></span>
+          <strong><a data-bind="text: created, attr: {'href': absoluteUrl}" target="_blank"></a></strong>
         </td>
       </tr>
     </script>
@@ -266,7 +266,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
     TOOLTIP_STOP: "${ _('Stop the execution') }",
     SAVED: "${ _('Saved') }",
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
-    NEW_SCRIPT_CONTENT: "ie. a = LOAD '/user/${ user }/data';"
+    NEW_SCRIPT_CONTENT: "ie. A = LOAD '/user/${ user }/data';"
   };
 
   var scripts = ${ scripts | n,unicode };
@@ -310,7 +310,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       viewModel.currentScript().script(codeMirror.getValue());
     });
 
-    showMainSection("scripts");
+    showMainSection("editor");
 
     $(document).on("loadEditor", function () {
       codeMirror.setValue(viewModel.currentScript().script());
@@ -424,7 +424,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
           else {
             viewModel.currentScript().isRunning(false);
             $(document).trigger("stopLogsRefresh");
-            $(document).trigger("showEditor");
+            //$(document).trigger("showEditor");
           }
         });
       }

+ 4 - 4
apps/pig/src/pig/templates/udfs.mako

@@ -26,17 +26,17 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
 
 <div class="container-fluid">
    ${ navigation.menubar(section='udfs') }
-  
+
     <div class="tab-content">
       <div class="tab-pane active">
 
 		<div class="container-fluid">
-		    <div class="row-fluid">         
-		      No UDFS         
+		    <div class="row-fluid">
+		      No UDFS
 		    </div>
 		</div>
 
-   
+
       </div>
 
     </div>

+ 37 - 25
apps/pig/src/pig/views.py

@@ -23,12 +23,11 @@ import logging
 import time
 
 from django.core.urlresolvers import reverse
+from django.http import HttpResponse
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_http_methods
-from django.http import HttpResponse
 
-from desktop.lib.django_util import render, encode_json_for_js
-from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.django_util import render
 from desktop.lib.view_util import format_duration_in_millis
 from liboozie.oozie_api import get_oozie
 from oozie.views.dashboard import show_oozie_error, check_job_access_permission
@@ -39,20 +38,22 @@ from pig.models import get_workflow_output, hdfs_link, PigScript
 
 LOG = logging.getLogger(__name__)
 
+
 def app(request):
   return render('app.mako', request, {
-    'scripts': json.dumps(get_scripts())
+    'scripts': json.dumps(get_scripts(is_history=True))
     }
   )
 
+
 def scripts(request):
   return HttpResponse(json.dumps(get_scripts()), mimetype="application/json")
 
 
-def get_scripts():
+def get_scripts(is_history=True):
   scripts = []
 
-  for script in PigScript.objects.filter(is_history=False):
+  for script in PigScript.objects.filter(is_history=is_history):
     data = json.loads(script.data)
     massaged_script = {
       'id': script.id,
@@ -63,6 +64,7 @@ def get_scripts():
 
   return scripts
 
+
 @show_oozie_error
 def dashboard(request):
   kwargs = {'cnt': 100,}
@@ -70,7 +72,9 @@ def dashboard(request):
   kwargs['name'] = Api.WORKFLOW_NAME
 
   jobs = get_oozie().get_workflows(**kwargs).jobs
-  return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(jobs, request.user)), mimetype="application/json")
+  hue_jobs = PigScript.objects.filter(owner=request.user)
+
+  return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(jobs, hue_jobs, request.user)), mimetype="application/json")
 
 
 def udfs(request):
@@ -80,8 +84,8 @@ def udfs(request):
 @require_http_methods(["POST"])
 def save(request):
   # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
-  
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=True)
+
   response = {
     'id': pig_script.id,
   }
@@ -93,15 +97,18 @@ def save(request):
 @show_oozie_error
 def run(request):
   # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=False)
 
-  # Todo, will come from script properties later
+  # TODO: will come from script properties later
   mapping = {
     'oozie.use.system.libpath':  'true',
-  }  
+  }
 
   oozie_id = Api(request.fs, request.user).submit(pig_script, mapping)
 
+  pig_script.update_from_dict({'job_id': oozie_id})
+  pig_script.save()
+
   response = {
     'id': pig_script.id,
     'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python'
@@ -109,6 +116,7 @@ def run(request):
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
+
 @require_http_methods(["POST"])
 def copy(request):
   # TODO security
@@ -124,10 +132,11 @@ def copy(request):
     'id': pig_script.id,
     'name': name,
     'script': script
-    }
+  }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
+
 @require_http_methods(["POST"])
 def delete(request):
   # TODO security
@@ -147,20 +156,21 @@ def delete(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-def create_or_update_script(id, name, script, user):
+def create_or_update_script(id, name, script, user, is_history=True):
   try:
     pig_script = PigScript.objects.get(id=id)
   except:
-    pig_script = PigScript.objects.create(owner=user)
+    pig_script = PigScript.objects.create(owner=user, is_history=is_history)
 
   pig_script.update_from_dict({'name': name, 'script': script})
   pig_script.save()
 
   return pig_script
 
+
 @show_oozie_error
 def watch(request, job_id):
-  oozie_workflow = check_job_access_permission(request, job_id) 
+  oozie_workflow = check_job_access_permission(request, job_id)
   logs, workflow_actions = Api(request, job_id).get_log(request, oozie_workflow)
   output = get_workflow_output(oozie_workflow, request.fs)
 
@@ -179,7 +189,7 @@ def watch(request, job_id):
     'logs': logs,
     'output': hdfs_link(output)
   }
-  
+
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
@@ -189,23 +199,24 @@ def format_time(st_time):
   else:
     return time.strftime("%a, %d %b %Y %H:%M:%S", st_time)
 
+
 def has_job_edition_permission(oozie_job, user):
   return user.is_superuser or oozie_job.user == user.username
 
+
 def has_dashboard_jobs_access(user):
   return user.is_superuser or user.has_hue_permission(action="dashboard_jobs_access", app=DJANGO_APPS[0])
 
-def massaged_oozie_jobs_for_json(oozie_jobs, user):
+
+def massaged_oozie_jobs_for_json(oozie_jobs, hue_jobs, user):
   jobs = []
+  hue_jobs = dict([(script.dict.get('job_id'), script) for script in hue_jobs if script.dict.get('job_id')])
 
   for job in oozie_jobs:
     if job.is_running():
-      if job.type == 'Workflow':
-        job = get_oozie().get_job(job.id)
-      elif job.type == 'Coordinator':
-        job = get_oozie().get_coordinator(job.id)
-      else:
-        job = get_oozie().get_bundle(job.id)
+      job = get_oozie().get_job(job.id)
+
+    script = hue_jobs.get(job.id) and hue_jobs.get(job.id) or None
 
     massaged_job = {
       'id': job.id,
@@ -216,7 +227,8 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user):
       'status': job.status,
       'isRunning': job.is_running(),
       'duration': job.endTime and job.startTime and format_duration_in_millis(( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000) or None,
-      'appName': job.appName,
+      'appName': script and script.dict['name'] or _('Unsaved script'),
+      'scriptId': script and script.id or -1,
       'progress': job.get_progress(),
       'user': job.user,
       'absoluteUrl': job.get_absolute_url(),

+ 8 - 0
apps/pig/static/js/pig.ko.js

@@ -20,6 +20,7 @@ var PigScript = function (pigScript) {
     id: ko.observable(pigScript.id),
     name: ko.observable(pigScript.name),
     script: ko.observable(pigScript.script),
+    scriptSumup: ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100)),
     isRunning: ko.observable(false),
     selected: ko.observable(false),
     watchUrl: ko.observable(""),
@@ -37,6 +38,7 @@ var PigScript = function (pigScript) {
 var Workflow = function (wf) {
   return {
     id: wf.id,
+    scriptId: wf.scriptId,
     lastModTime: wf.lastModTime,
     endTime: wf.endTime,
     status: wf.status,
@@ -302,4 +304,10 @@ var PigViewModel = function (scripts, props) {
           $("#deleteModal").modal("hide");
         }, "json");
   }
+
+  self.viewSubmittedScript = function (workflow) {
+    self.loadScript(workflow.scriptId);
+    $(document).trigger("loadEditor");
+    $(document).trigger("showEditor");
+  };
 };