فهرست منبع

[pig] Decouple view from API

Load script from history
Save unsaved script into history ala Beeswax
Add icons
Romain Rigaux 12 سال پیش
والد
کامیت
13847d8

+ 78 - 9
apps/pig/src/pig/api.py

@@ -17,31 +17,46 @@
 
 import logging
 import re
+import time
 
+from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 
 from jobbrowser.views import job_single_logs
+from desktop.lib.view_util import format_duration_in_millis
 from oozie.models import Workflow, Pig
 from oozie.views.editor import _submit_workflow
-
+from liboozie.oozie_api import get_oozie
 
 LOG = logging.getLogger(__name__)
 
 
-class Api:
+def get(fs, user):
+  return OozieApi(fs, user)
+
+
+class OozieApi:
+  """
+  Oozie submission.
+  """
   WORKFLOW_NAME = 'pig-app-hue-script'
   RE_LOG_END = re.compile('(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed <<<)')
   RE_LOG_START_RUNNING = re.compile('>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?\n(.+?)(<<< Invocation of Pig command completed <<<|<<< Invocation of Main class completed)', re.M | re.DOTALL)
   RE_LOG_START_FINISHED = re.compile('(>>> Invoking Pig command line now >>>\n\n\nRun pig script using PigRunner.run\(\) for Pig version [^\n]+?)\n', re.M | re.DOTALL)
-
+  MAX_DASHBOARD_JOBS = 100
 
   def __init__(self, fs, user):
     self.fs = fs
     self.user = user
 
   def submit(self, pig_script, mapping):
+    # TODO: will come from script properties later
+    mapping.update({
+      'oozie.use.system.libpath':  'true',
+    })
+
     workflow = Workflow.objects.new_workflow(self.user)
-    workflow.name = Api.WORKFLOW_NAME
+    workflow.name = OozieApi.WORKFLOW_NAME
     workflow.is_history = True
     workflow.save()
     Workflow.objects.initialize(workflow, self.fs)
@@ -58,6 +73,13 @@ class Api:
 
     return _submit_workflow(self.user, self.fs, workflow, mapping)
 
+  def get_jobs(self):
+    kwargs = {'cnt': OozieApi.MAX_DASHBOARD_JOBS,}
+    kwargs['user'] = self.user.username
+    kwargs['name'] = OozieApi.WORKFLOW_NAME
+
+    return get_oozie().get_workflows(**kwargs).jobs
+
   def get_log(self, request, oozie_workflow):
     logs = {}
 
@@ -66,7 +88,7 @@ class Api:
         if action.externalId:
           log = job_single_logs(request, **{'job': action.externalId})
           if log:
-            logs[action.name] = self.match_logs(log['logs'][1])
+            logs[action.name] = self._match_logs(log['logs'][1])
       except Exception, e:
         LOG.error('An error happen while watching the demo running: %(error)s' % {'error': e})
 
@@ -83,11 +105,58 @@ class Api:
 
     return logs, workflow_actions
 
-  def match_logs(self, logs):
+  def _match_logs(self, logs):
     """Difficult to match multi lines of text"""
-    if Api.RE_LOG_END.search(logs):
-      return re.search(Api.RE_LOG_START_RUNNING, logs).group(1)
+    if OozieApi.RE_LOG_END.search(logs):
+      return re.search(OozieApi.RE_LOG_START_RUNNING, logs).group(1)
     else:
-      group = re.search(Api.RE_LOG_START_FINISHED, logs)
+      group = re.search(OozieApi.RE_LOG_START_FINISHED, logs)
       i = logs.index(group.group(1)) + len(group.group(1))
       return logs[i:]
+
+  def massaged_jobs_for_json(self, oozie_jobs, hue_jobs):
+    jobs = []
+    hue_jobs = dict([(script.dict.get('job_id'), script) for script in hue_jobs if script.dict.get('job_id')])
+
+    for job in oozie_jobs:
+      if job.is_running():
+        job = get_oozie().get_job(job.id)
+
+      hue_pig = hue_jobs.get(job.id) and hue_jobs.get(job.id) or None
+
+      massaged_job = {
+        'id': job.id,
+        'lastModTime': hasattr(job, 'lastModTime') and job.lastModTime and format_time(job.lastModTime) or None,
+        'kickoffTime': hasattr(job, 'kickoffTime') and job.kickoffTime or None,
+        'timeOut': hasattr(job, 'timeOut') and job.timeOut or None,
+        'endTime': job.endTime and format_time(job.endTime) or None,
+        'status': job.status,
+        'isRunning': job.is_running(),
+        'duration': job.endTime and job.startTime and format_duration_in_millis(( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000) or None,
+        'appName': hue_pig and hue_pig.dict['name'] or _('Unsaved script'),
+        'scriptId': hue_pig and hue_pig.id or -1,
+        'progress': job.get_progress(),
+        'user': job.user,
+        'absoluteUrl': job.get_absolute_url(),
+        'canEdit': has_job_edition_permission(job, self.user),
+        'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'kill'}),
+        'created': hasattr(job, 'createdTime') and job.createdTime and job.createdTime and ((job.type == 'Bundle' and job.createdTime) or format_time(job.createdTime)),
+        'startTime': hasattr(job, 'startTime') and format_time(job.startTime) or None,
+        'run': hasattr(job, 'run') and job.run or 0,
+        'frequency': hasattr(job, 'frequency') and job.frequency or None,
+        'timeUnit': hasattr(job, 'timeUnit') and job.timeUnit or None,
+        }
+      jobs.append(massaged_job)
+
+    return jobs
+
+def format_time(st_time):
+  if st_time is None:
+    return '-'
+  else:
+    return time.strftime("%a, %d %b %Y %H:%M:%S", st_time)
+
+
+def has_job_edition_permission(oozie_job, user):
+  return user.is_superuser or oozie_job.user == user.username
+

+ 0 - 4
apps/pig/src/pig/forms.py

@@ -14,7 +14,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
-from django import forms
-

+ 43 - 7
apps/pig/src/pig/models.py

@@ -15,25 +15,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 try:
   import json
 except ImportError:
   import simplejson as json
 import posixpath
 
-from hadoop.fs.hadoopfs import Hdfs
 from django.db import models
 from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
+from desktop.lib.exceptions_renderable import PopupException
+from hadoop.fs.hadoopfs import Hdfs
+
 from oozie.models import Workflow
 
 
 class Document(models.Model):
   owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can modify the job.'))
-  is_history = models.BooleanField(default=True, db_index=True, verbose_name=_t('Is a submitted job'),
-                                  help_text=_t('If the job should show up in the history'))
+  is_design = models.BooleanField(default=True, db_index=True, verbose_name=_t('Is a user document, not a document submission.'),
+                                     help_text=_t('If the document is not a submitted job but a real query, script, workflow.'))
+
+  def is_editable(self, user):
+    return user.is_superuser or self.owner == user
+
+  def can_edit_or_exception(self, user, exception_class=PopupException):
+    if self.is_editable(user):
+      return True
+    else:
+      raise exception_class(_('Only superusers and %s are allowed to modify this document.') % user)
 
 
 class PigScript(Document):
@@ -65,12 +75,38 @@ class Submission(models.Model):
   workflow = models.ForeignKey(Workflow)
 
 
-class Udf:
-  pass
+def create_or_update_script(id, name, script, user, is_design=True):
+  """Take care of security"""
+  try:
+    pig_script = PigScript.objects.get(id=id)
+    pig_script.can_edit_or_exception(user)
+  except:
+    pig_script = PigScript.objects.create(owner=user, is_design=is_design)
+
+  pig_script.update_from_dict({'name': name, 'script': script})
+  pig_script.save()
+
+  return pig_script
+
+
+def get_scripts(user, max_count=200):
+  scripts = []
+
+  for script in PigScript.objects.filter(owner=user).order_by('-id')[:max_count]:
+    data = json.loads(script.data)
+    massaged_script = {
+      'id': script.id,
+      'name': data['name'],
+      'script': data['script'],
+      'isDesign': script.is_design,
+    }
+    scripts.append(massaged_script)
+
+  return scripts
 
 
 def get_workflow_output(oozie_workflow, fs):
-  # TODO: guess from the STORE
+  # TODO: guess from the STORE or parameters
   output = None
 
   if 'workflowRoot' in oozie_workflow.conf_dict:

+ 14 - 3
apps/pig/src/pig/templates/app.mako

@@ -103,9 +103,18 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         <form id="advancedSettingsForm" method="POST" class="form form-horizontal noPadding">
           <ul class="nav nav-list">
             <li class="nav-header">${_('Editor')}</li>
-            <li data-bind="click: editScript" class="active" data-section="edit"><a href="#">${ _('Edit script') }</a></li>
+            <li data-bind="click: editScript" class="active" data-section="edit">
+              <a href="#"><i class="icon-edit"></i> ${ _('Edit script') }</a>
+            </li>
+            <li data-bind="click: newScript">
+              <a href="#" title="${ _('New script') }" rel="tooltip" data-placement="right">
+                <i class="icon-plus-sign"></i> ${ _('New script') }
+              </a>
+            </li>            
             <li class="nav-header">${_('Properties')}</li>
-            <li data-bind="click: editScriptProperties" data-section="properties"><a href="#">${ _('Edit properties') }</a></li>
+            <li data-bind="click: editScriptProperties" data-section="properties">
+              <a href="#"><i class="icon-reorder"></i> ${ _('Edit properties') }</a>
+            </li>
             ##<li class="nav-header">${_('UDF')}</li>
             ##<li><a href="#createDataset">${ _('New') }</a></li>
             ##<li><a href="#createDataset">${ _('Add') }</a></li>
@@ -167,7 +176,7 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
         <div data-bind="template: {name: 'logTemplate', foreach: currentScript().actions}"></div>
         <script id="logTemplate" type="text/html">
           <div data-bind="css:{'alert-modified': name != '', 'alert': name != '', 'alert-success': status == 'SUCCEEDED' || status == 'OK', 'alert-error': status != 'RUNNING' && status != 'SUCCEEDED' && status != 'OK' && status != 'PREP'}">
-            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Progress: 100%') } '<span data-bind="text: name"></span>'</h4></div>
+            <div class="pull-right" data-bind="text: status"></div><h4>${ _('Progress: 50%') } '<span data-bind="text: name"></span>'</h4></div>
           <pre data-bind="visible: logs == ''">${ _('No available logs.') }</pre>
           <pre data-bind="visible: logs != '', text: logs"></pre>
         </script>
@@ -403,6 +412,8 @@ ${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
       }, 350);
     });
 
+    viewModel.filterScripts('');
+
     refreshDashboard();
 
     var dashboardRefreshInterval = window.setInterval(function () {

+ 0 - 46
apps/pig/src/pig/templates/udfs.mako

@@ -1,46 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-<%!
-from desktop.views import commonheader, commonfooter
-from django.utils.translation import ugettext as _
-%>
-
-<%namespace name="navigation" file="navigation-bar.inc.mako" />
-<%namespace name="utils" file="utils.inc.mako" />
-
-${ commonheader(_('Pig'), "pig", user, "100px") | n,unicode }
-
-
-<div class="container-fluid">
-   ${ navigation.menubar(section='udfs') }
-
-    <div class="tab-content">
-      <div class="tab-pane active">
-
-		<div class="container-fluid">
-		    <div class="row-fluid">
-		      No UDFS
-		    </div>
-		</div>
-
-
-      </div>
-
-    </div>
-  </div>
-</div>
-
-${ commonfooter(messages) | n,unicode }

+ 0 - 1
apps/pig/src/pig/urls.py

@@ -21,7 +21,6 @@ urlpatterns = patterns('pig.views',
   url(r'^$', 'app', name='index'),
 
   url(r'^app/$', 'app', name='app'),
-  url(r'^udfs/$', 'udfs', name='udfs'),
 
   # Ajax
   url(r'^scripts/$', 'scripts', name='scripts'),

+ 33 - 113
apps/pig/src/pig/views.py

@@ -20,7 +20,7 @@ try:
 except ImportError:
   import simplejson as json
 import logging
-import time
+
 
 from django.core.urlresolvers import reverse
 from django.http import HttpResponse
@@ -28,12 +28,12 @@ from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_http_methods
 
 from desktop.lib.django_util import render
-from desktop.lib.view_util import format_duration_in_millis
-from liboozie.oozie_api import get_oozie
+from desktop.lib.exceptions_renderable import PopupException
 from oozie.views.dashboard import show_oozie_error, check_job_access_permission
 
-from pig.api import Api
-from pig.models import get_workflow_output, hdfs_link, PigScript
+from pig import api
+from pig.models import get_workflow_output, hdfs_link, PigScript,\
+  create_or_update_script, get_scripts
 
 
 LOG = logging.getLogger(__name__)
@@ -41,50 +41,33 @@ LOG = logging.getLogger(__name__)
 
 def app(request):
   return render('app.mako', request, {
-    'scripts': json.dumps(get_scripts(is_history=True))
+    'scripts': json.dumps(get_scripts(request.user))
     }
   )
 
 
 def scripts(request):
-  return HttpResponse(json.dumps(get_scripts()), mimetype="application/json")
-
-
-def get_scripts(is_history=True):
-  scripts = []
-
-  for script in PigScript.objects.filter(is_history=is_history):
-    data = json.loads(script.data)
-    massaged_script = {
-      'id': script.id,
-      'name': data["name"],
-      'script': data["script"]
-    }
-    scripts.append(massaged_script)
-
-  return scripts
+  return HttpResponse(json.dumps(get_scripts(request.user)), mimetype="application/json")
 
 
 @show_oozie_error
 def dashboard(request):
-  kwargs = {'cnt': 100,}
-  kwargs['user'] = request.user.username
-  kwargs['name'] = Api.WORKFLOW_NAME
+  pig_api = api.get(request.fs, request.user)
 
-  jobs = get_oozie().get_workflows(**kwargs).jobs
+  jobs = pig_api.get_jobs()
   hue_jobs = PigScript.objects.filter(owner=request.user)
+  massaged_jobs = pig_api.massaged_jobs_for_json(jobs, hue_jobs)
 
-  return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(jobs, hue_jobs, request.user)), mimetype="application/json")
+  return HttpResponse(json.dumps(massaged_jobs), mimetype="application/json")
 
 
-def udfs(request):
-  return render('udfs.mako', request, {})
-
-
-@require_http_methods(["POST"])
 def save(request):
-  # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=True)
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
+  pig_script.is_design = True
+  pig_script.save()
 
   response = {
     'id': pig_script.id,
@@ -93,18 +76,13 @@ def save(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-@require_http_methods(["POST"])
+
 @show_oozie_error
 def run(request):
-  # TODO security
-  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_history=False)
-
-  # TODO: will come from script properties later
-  mapping = {
-    'oozie.use.system.libpath':  'true',
-  }
+  pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_design=False)
 
-  oozie_id = Api(request.fs, request.user).submit(pig_script, mapping)
+  params = {}
+  oozie_id = api.get(request.fs, request.user).submit(pig_script, params)
 
   pig_script.update_from_dict({'job_id': oozie_id})
   pig_script.save()
@@ -117,10 +95,14 @@ def run(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-@require_http_methods(["POST"])
 def copy(request):
-  # TODO security
-  existing_script_data = json.loads((PigScript.objects.get(id=request.POST.get('id'))).data)
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  pig_script = PigScript.objects.get(id=request.POST.get('id'))
+  pig_script.can_edit_or_exception(request.user)
+
+  existing_script_data = pig_script.dict
   name = existing_script_data["name"] + _(' (Copy)')
   script = existing_script_data["script"]
 
@@ -137,14 +119,16 @@ def copy(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-@require_http_methods(["POST"])
 def delete(request):
-  # TODO security
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
   ids = request.POST.get('ids').split(",")
 
   for script_id in ids:
     try:
       pig_script = PigScript.objects.get(id=script_id)
+      pig_script.can_edit_or_exception(request.user)
       pig_script.delete()
     except:
       None
@@ -156,22 +140,10 @@ def delete(request):
   return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-def create_or_update_script(id, name, script, user, is_history=True):
-  try:
-    pig_script = PigScript.objects.get(id=id)
-  except:
-    pig_script = PigScript.objects.create(owner=user, is_history=is_history)
-
-  pig_script.update_from_dict({'name': name, 'script': script})
-  pig_script.save()
-
-  return pig_script
-
-
 @show_oozie_error
 def watch(request, job_id):
   oozie_workflow = check_job_access_permission(request, job_id)
-  logs, workflow_actions = Api(request, job_id).get_log(request, oozie_workflow)
+  logs, workflow_actions = api.get(request, job_id).get_log(request, oozie_workflow)
   output = get_workflow_output(oozie_workflow, request.fs)
 
   workflow = {
@@ -191,55 +163,3 @@ def watch(request, job_id):
   }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
-
-
-def format_time(st_time):
-  if st_time is None:
-    return '-'
-  else:
-    return time.strftime("%a, %d %b %Y %H:%M:%S", st_time)
-
-
-def has_job_edition_permission(oozie_job, user):
-  return user.is_superuser or oozie_job.user == user.username
-
-
-def has_dashboard_jobs_access(user):
-  return user.is_superuser or user.has_hue_permission(action="dashboard_jobs_access", app=DJANGO_APPS[0])
-
-
-def massaged_oozie_jobs_for_json(oozie_jobs, hue_jobs, user):
-  jobs = []
-  hue_jobs = dict([(script.dict.get('job_id'), script) for script in hue_jobs if script.dict.get('job_id')])
-
-  for job in oozie_jobs:
-    if job.is_running():
-      job = get_oozie().get_job(job.id)
-
-    script = hue_jobs.get(job.id) and hue_jobs.get(job.id) or None
-
-    massaged_job = {
-      'id': job.id,
-      'lastModTime': hasattr(job, 'lastModTime') and job.lastModTime and format_time(job.lastModTime) or None,
-      'kickoffTime': hasattr(job, 'kickoffTime') and job.kickoffTime or None,
-      'timeOut': hasattr(job, 'timeOut') and job.timeOut or None,
-      'endTime': job.endTime and format_time(job.endTime) or None,
-      'status': job.status,
-      'isRunning': job.is_running(),
-      'duration': job.endTime and job.startTime and format_duration_in_millis(( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000) or None,
-      'appName': script and script.dict['name'] or _('Unsaved script'),
-      'scriptId': script and script.id or -1,
-      'progress': job.get_progress(),
-      'user': job.user,
-      'absoluteUrl': job.get_absolute_url(),
-      'canEdit': has_job_edition_permission(job, user),
-      'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'kill'}),
-      'created': hasattr(job, 'createdTime') and job.createdTime and job.createdTime and ((job.type == 'Bundle' and job.createdTime) or format_time(job.createdTime)),
-      'startTime': hasattr(job, 'startTime') and format_time(job.startTime) or None,
-      'run': hasattr(job, 'run') and job.run or 0,
-      'frequency': hasattr(job, 'frequency') and job.frequency or None,
-      'timeUnit': hasattr(job, 'timeUnit') and job.timeUnit or None,
-      }
-    jobs.append(massaged_job)
-
-  return jobs

+ 2 - 1
apps/pig/static/js/pig.ko.js

@@ -18,6 +18,7 @@
 var PigScript = function (pigScript) {
   return {
     id: ko.observable(pigScript.id),
+    isDesign: ko.observable(pigScript.isDesign),
     name: ko.observable(pigScript.name),
     script: ko.observable(pigScript.script),
     scriptSumup: ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100)),
@@ -120,7 +121,7 @@ var PigViewModel = function (scripts, props) {
 
   self.filterScripts = function (filter) {
     self.filteredScripts(ko.utils.arrayFilter(self.scripts(), function (script) {
-      return script.name().toLowerCase().indexOf(filter.toLowerCase()) > -1
+      return script.isDesign() && script.name().toLowerCase().indexOf(filter.toLowerCase()) > -1
     }));
   };