Browse Source

HUE-1147 [pig] UpperText example

Shows script
Shows parameters
Show Java UDF
Adding a test that submits the example
Romain Rigaux 12 năm trước cách đây
mục cha
commit
d7c0815

+ 13 - 6
apps/about/src/about/templates/admin_wizard.mako

@@ -49,10 +49,10 @@ ${ commonheader(_('About Hue'), "quick_start", user, "100px") | n,unicode }
   <div class="row-fluid">
    <div id="properties" class="section">
     <ul class="nav nav-tabs">
-      <li class="active"><a href="#step1" class="step">${ _('Step 1: Check Configuration') }</a></li>
-      <li><a href="#step2" class="step">${ _('Step 2: Examples') }</a></li>
-      <li><a href="#step3" class="step">${ _('Step 3: Users') }</a></li>
-      <li><a href="${ url('desktop.views.home') }" class="step">${ _('Step 4: Use Hue') }</a></li>
+      <li class="active"><a href="#step1" class="step">${ _('Step 1:') } <i class="icon-cogs"></i> ${ _('Check Configuration') }</a></li>
+      <li><a href="#step2" class="step">${ _('Step 2:') } <i class="icon-comments-alt"></i> ${ _('Examples') }</a></li>
+      <li><a href="#step3" class="step">${ _('Step 3:') } <i class="icon-user"></i> ${ _('Users') }</a></li>
+      <li><a href="${ url('desktop.views.home') }" class="step">${ _('Step 4:') } <i class="icon-home"></i> ${_('Use Hue') }</a></li>
     </ul>
 
     <div class="steps" >
@@ -67,18 +67,25 @@ ${ commonheader(_('About Hue'), "quick_start", user, "100px") | n,unicode }
       <ul class="nav nav-tabs nav-stacked">
         % if 'beeswax' in app_names:
           <li>
-            <button type="button" class="btn" data-loading-text="Installing..." data-url="${ url('beeswax:install_examples') }">
+            <button type="button" class="btn" data-loading-text="${ _('Installing...') }" data-url="${ url('beeswax:install_examples') }">
              <i class="icon-download-alt"></i> Beeswax
             </button>
           </li>
         % endif
         % if 'oozie' in app_names or 'jobsub' in app_names:
           <li>
-            <button type="button" class="btn" data-loading-text="Installing..." data-url="${ url('oozie:setup_app') }">
+            <button type="button" class="btn" data-loading-text="${ _('Installing...') }" data-url="${ url('oozie:setup_app') }">
               <i class="icon-download-alt"></i> Oozie / Job Designer
             </button>
           </li>
         % endif
+        % if 'pig' in app_names:
+          <li>
+            <button type="button" class="btn" data-loading-text="${ _('Installing...') }" data-url="${ url('pig:install_examples') }">
+             <i class="icon-download-alt"></i> Pig
+            </button>
+          </li>
+        % endif
       </ul>
     </div>
 

+ 12 - 12
apps/jobbrowser/src/jobbrowser/templates/jobs.mako

@@ -76,18 +76,18 @@ ${ commonheader(_('Job Browser'), "jobbrowser", user) | n,unicode }
 <table class="datatables table table-striped table-condensed">
     <thead>
         <tr>
-            <th>${_('Logs')}</th>
-            <th>${_('ID')}</th>
-            <th>${_('Name')}</th>
-            <th>${_('Status')}</th>
-            <th>${_('User')}</th>
-            <th>${_('Maps')}</th>
-            <th>${_('Reduces')}</th>
-            <th>${_('Queue')}</th>
-            <th>${_('Priority')}</th>
-            <th>${_('Duration')}</th>
-            <th>${_('Date')}</th>
-            <th data-row-selector-exclude="true"></th>
+            <th width="4%">${_('Logs')}</th>
+            <th width="10%">${_('ID')}</th>
+            <th width="49%">${_('Name')}</th>
+            <th width="5%">${_('Status')}</th>
+            <th width="5%">${_('User')}</th>
+            <th width="2%">${_('Maps')}</th>
+            <th width="2%">${_('Reduces')}</th>
+            <th width="5%">${_('Queue')}</th>
+            <th width="4%">${_('Priority')}</th>
+            <th width="4%">${_('Duration')}</th>
+            <th width="12%">${_('Date')}</th>
+            <th width="3%" data-row-selector-exclude="true" ></th>
         </tr>
     </thead>
     <tbody>

+ 7 - 7
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinators.mako

@@ -58,14 +58,14 @@ ${layout.menubar(section='dashboard')}
     <table class="table table-condensed" id="running-table">
       <thead>
         <tr>
-          <th width="10%">${ _('Next submission') }</th>
-          <th width="10%">${ _('Status') }</th>
+          <th width="12%">${ _('Next submission') }</th>
+          <th width="5%">${ _('Status') }</th>
           <th width="20%">${ _('Name') }</th>
           <th width="5%">${ _('Progress') }</th>
           <th width="10%">${ _('Submitter') }</th>
-          <th width="5%">${ _('Frequency') }</th>
+          <th width="3%">${ _('Frequency') }</th>
           <th width="5%">${ _('Time unit') }</th>
-          <th width="10%">${ _('Start Time') }</th>
+          <th width="12%">${ _('Start Time') }</th>
           <th width="15%">${ _('Id') }</th>
           <th width="10%">${ _('Action') }</th>
         </tr>
@@ -81,14 +81,14 @@ ${layout.menubar(section='dashboard')}
     <table class="table table-condensed" id="completed-table" data-tablescroller-disable="true">
       <thead>
         <tr>
-          <th width="10%">${ _('Completion') }</th>
+          <th width="12%">${ _('Completion') }</th>
           <th width="5%">${ _('Status') }</th>
-          <th width="25%">${ _('Name') }</th>
+          <th width="20%">${ _('Name') }</th>
           <th width="10%">${ _('Duration') }</th>
           <th width="10%">${ _('Submitter') }</th>
           <th width="5%">${ _('Frequency') }</th>
           <th width="5%">${ _('Time unit') }</th>
-          <th width="10%">${ _('Start Time') }</th>
+          <th width="13%">${ _('Start Time') }</th>
           <th width="20%">${ _('Id') }</th>
         </tr>
       </thead>

+ 10 - 10
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflows.mako

@@ -59,14 +59,14 @@ ${ layout.menubar(section='dashboard') }
     <table class="table table-condensed" id="running-table">
       <thead>
         <tr>
-          <th width="10%">${ _('Submission') }</th>
-          <th width="10%">${ _('Status') }</th>
+          <th width="12%">${ _('Submission') }</th>
+          <th width="5%">${ _('Status') }</th>
           <th width="20%">${ _('Name') }</th>
           <th width="5%">${ _('Progress') }</th>
           <th width="5%">${ _('Submitter') }</th>
-          <th width="10%">${ _('Created') }</th>
-          <th width="10%">${ _('Last modified') }</th>
-          <th width="5%">${ _('Run') }</th>
+          <th width="13%">${ _('Created') }</th>
+          <th width="13%">${ _('Last modified') }</th>
+          <th width="2%">${ _('Run') }</th>
           <th width="15%">${ _('Id') }</th>
           <th width="10%">${ _('Action') }</th>
         </tr>
@@ -83,14 +83,14 @@ ${ layout.menubar(section='dashboard') }
     <table class="table table-condensed" id="completed-table" data-tablescroller-disable="true">
       <thead>
         <tr>
-          <th width="10%">${ _('Completion') }</th>
-          <th width="10%">${ _('Status') }</th>
+          <th width="12%">${ _('Completion') }</th>
+          <th width="5%">${ _('Status') }</th>
           <th width="25%">${ _('Name') }</th>
           <th width="5%">${ _('Duration') }</th>
           <th width="5%">${ _('Submitter') }</th>
-          <th width="10%">${ _('Created') }</th>
-          <th width="10%">${ _('Last modified') }</th>
-          <th width="5%">${ _('Run') }</th>
+          <th width="13%">${ _('Created') }</th>
+          <th width="13%">${ _('Last modified') }</th>
+          <th width="2%">${ _('Run') }</th>
           <th width="25%">${ _('Id') }</th>
         </tr>
       </thead>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-common.xml.mako

@@ -55,7 +55,7 @@
 <%def name="distributed_cache(files, archives)">
     % for f in files:
         % if f:
-            <file>${ filelink(f['name']) }</file>
+            <file>${ filelink(f) }</file>
         % endif
     % endfor
     % for a in archives:

+ 0 - 1
apps/oozie/src/oozie/templates/navigation-bar.mako

@@ -29,7 +29,6 @@
         <li class="${utils.is_selected(section, 'workflows')}"><a href="${url('oozie:list_workflows')}">${ _('Workflows') }</a></li>
         <li class="${utils.is_selected(section, 'coordinators')}"><a href="${url('oozie:list_coordinators')}">${ _('Coordinators') }</a></li>
         <li class="${utils.is_selected(section, 'bundles')}"><a href="${url('oozie:list_bundles')}">${ _('Bundles') }</a></li>
-        <li class="${utils.is_selected(section, 'history')}"><a href="${url('oozie:list_history')}">${ _('History') }</a></li>
         <li><a href="http://oozie.apache.org/docs/3.3.0/index.html" target="_blank">${ _('Documentation') }</a></li>
       </ul>
     </div>

+ 1 - 1
apps/oozie/src/oozie/urls.py

@@ -52,7 +52,7 @@ urlpatterns = patterns(
   url(r'^create_bundled_coordinator/(?P<bundle>\d+)$', 'create_bundled_coordinator', name='create_bundled_coordinator'),
   url(r'^edit_bundled_coordinator/(?P<bundle>\d+)/(?P<bundled_coordinator>\d+)$', 'edit_bundled_coordinator', name='edit_bundled_coordinator'),
 
-  url(r'^list_history$', 'list_history', name='list_history'),
+  url(r'^list_history$', 'list_history', name='list_history'), # Unused
   url(r'^list_history/(?P<record_id>[-\w]+)$', 'list_history_record', name='list_history_record'),
   url(r'^setup_app/$', 'setup_app', name='setup_app'),
   url(r'^jasmine', 'jasmine'),

+ 11 - 9
apps/oozie/src/oozie/views/dashboard.py

@@ -638,16 +638,18 @@ def split_oozie_jobs(oozie_jobs):
   jobs_completed = []
 
   for job in oozie_jobs:
-    if job.is_running():
-      if job.type == 'Workflow':
-        job = get_oozie().get_job(job.id)
-      elif job.type == 'Coordinator':
-        job = get_oozie().get_coordinator(job.id)
+    # Bulk load instead
+    if job.appName != 'pig-app-hue-script':
+      if job.is_running():
+        if job.type == 'Workflow':
+          job = get_oozie().get_job(job.id)
+        elif job.type == 'Coordinator':
+          job = get_oozie().get_coordinator(job.id)
+        else:
+          job = get_oozie().get_bundle(job.id)
+        jobs_running.append(job)
       else:
-        job = get_oozie().get_bundle(job.id)
-      jobs_running.append(job)
-    else:
-      jobs_completed.append(job)
+        jobs_completed.append(job)
 
   jobs['running_jobs'] = sorted(jobs_running, key=lambda w: w.status)
   jobs['completed_jobs'] = sorted(jobs_completed, key=lambda w: w.status)

BIN
apps/pig/examples/lib/piggybank.jar


+ 11 - 17
apps/pig/src/pig/conf.py

@@ -15,26 +15,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os.path import dirname, join
+import os
+
 from django.utils.translation import ugettext_lazy as _
 
 from desktop.lib.conf import Config
 
 
-# TODO update hue.inis
-
-REPO_PROTOCOL = Config(
-  key="repo_protocol",
-  help=_("Protocol of the repository URI (e.g. 'http://, file://)."),
-  default='http://')
-
-REPO_URI = Config(
-  key="repo_uri",
-  # TODO remove
-  default=join(dirname(__file__), "..", "..", "src/pig/test_data/test-repo"),
-  help=_("Path of the repository to fetch."))
+LOCAL_SAMPLE_DIR = Config(
+  key="local_data_dir",
+  default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"),
+  help=_("Location on local filesystem where the examples are stored."),
+  private=True)
 
-HDFS_WORKSPACE = Config(
-  key="hdfs_workspace",
-  default="/tmp",
-  help=_("Where to upload the demo and save their outputs."))
+REMOTE_SAMPLE_DIR = Config(
+  key="remote_data_dir",
+  default="/user/hue/pig/examples",
+  help=_("Location on HDFS where the Pig examples are stored."))

+ 17 - 0
apps/pig/src/pig/fixtures/initial_pig_examples.json

@@ -0,0 +1,17 @@
+[
+  {
+    "pk": 1,
+    "model": "pig.document",
+    "fields": {
+      "owner": 1100713,
+      "is_design": false
+    }
+  },
+  {
+    "pk": 1,
+    "model": "pig.pigscript",
+    "fields": {
+      "data": "{\"job_id\": null, \"parameters\": [], \"script\": \"REGISTER piggybank.jar;\\n\\n\\ndata = LOAD '/user/hue/pig/examples/data/midsummer.txt' as (text:CHARARRAY);\\n\\nupper_case =  FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);\\n\\n\\nSTORE upper_case INTO '$output' ;\\n\", \"properties\": [], \"resources\": [{\"type\": \"file\", \"value\": \"/user/hue/pig/examples/lib/piggybank.jar\"}], \"name\": \"UpperText\"}"
+    }
+  }
+]

+ 0 - 0
apps/pig/src/pig/management/__init__.py


+ 0 - 0
apps/pig/src/pig/management/commands/__init__.py


+ 63 - 0
apps/pig/src/pig/management/commands/pig_setup.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from django.contrib.auth.models import User
+from django.core import management
+from django.core.management.base import NoArgsCommand
+from django.db import transaction
+from django.utils.translation import ugettext as _
+
+from hadoop import cluster
+
+from pig.conf import LOCAL_SAMPLE_DIR, REMOTE_SAMPLE_DIR
+from liboozie.submittion import create_directories
+from desktop.lib import paths
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(NoArgsCommand):
+  def handle_noargs(self, **options):
+    fs = cluster.get_hdfs()
+    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
+    remote_dir = REMOTE_SAMPLE_DIR.get()
+
+    # Copy examples binaries
+    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
+      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
+      remote_data_dir = fs.join(remote_dir, name)
+      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
+                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
+      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
+
+    # Copy sample data
+    local_dir = paths.get_thirdparty_root("sample_data")
+    remote_data_dir = fs.join(remote_dir, 'data')
+    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
+                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
+    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
+
+    # Load jobs
+    USERNAME = 'sample'
+    try:
+      sample_user = User.objects.get(username=USERNAME)
+    except User.DoesNotExist:
+      sample_user = User.objects.create(username=USERNAME, password='!', is_active=False, is_superuser=False, id=1100713, pk=1100713)
+    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)

+ 2 - 2
apps/pig/src/pig/models.py

@@ -98,7 +98,7 @@ def create_or_update_script(id, name, script, user, parameters, resources, is_de
 def get_scripts(user, max_count=200):
   scripts = []
 
-  for script in PigScript.objects.filter(owner=user).order_by('-id')[:max_count]:
+  for script in PigScript.objects.filter(owner__pk__in=[user.pk, 1100713]).order_by('-id')[:max_count]:
     data = script.dict
     massaged_script = {
       'id': script.id,
@@ -114,7 +114,7 @@ def get_scripts(user, max_count=200):
 
 
 def get_workflow_output(oozie_workflow, fs):
-  # TODO: guess from the STORE or parameters
+  # TODO: guess from the Input(s):/Output(s)
   output = None
 
   if 'workflowRoot' in oozie_workflow.conf_dict:

+ 57 - 1
apps/pig/src/pig/tests.py

@@ -14,14 +14,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from liboozie.oozie_api_test import OozieServerProvider
+
+try:
+  import json
+except ImportError:
+  import simplejson as json
+import time
 
 from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
 
 from nose.tools import assert_true, assert_equal
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
-from pig.models import create_or_update_script
+from pig.models import create_or_update_script, PigScript
+from oozie.tests import OozieBase
 
 
 class TestPigBase(object):
@@ -47,3 +56,50 @@ class TestMock(TestPigBase):
   def test_create_script(self):
     pig_script = self.create_script()
     assert_equal('Test', pig_script.dict['name'])
+
+
+class TestWithHadoop(OozieBase):
+
+  def setUp(self):
+    super(TestWithHadoop, self).setUp()
+    grant_access("test", "test", "pig")
+    self.c.post(reverse('pig:install_examples'))
+
+  def wait_until_completion(self, pig_script_id, timeout=300.0, step=5):
+    script = PigScript.objects.get(id=pig_script_id)
+    job_id = script.dict['job_id']
+
+    response = self.c.get(reverse('pig:watch', args=[job_id]))
+    response = json.loads(response.content)
+
+    start = time.time()
+
+    while response['workflow']['status'] in ['PREP', 'RUNNING'] and time.time() - start < timeout:
+      time.sleep(step)
+      response = self.c.get(reverse('pig:watch', args=[job_id]))
+      response = json.loads(response.content)
+
+    logs = OozieServerProvider.oozie.get_job_log(job_id)
+
+    if response['workflow']['status'] != 'SUCCEEDED':
+      msg = "[%d] %s took more than %d to complete or %s: %s" % (time.time(), job_id, timeout, response['workflow']['status'], logs)
+      raise Exception(msg)
+
+    return pig_script_id
+
+  def test_submit(self):
+    script = PigScript.objects.get(id=1)
+    script_dict = script.dict
+
+    post_data = {
+      'id': script.id,
+      'name': script_dict['name'],
+      'script': script_dict['script'],
+      'user': script.owner,
+      'parameters': json.dumps(script_dict['parameters']),
+      'resources': json.dumps(script_dict['resources']),
+      'submissionVariables': json.dumps([{"name": "output", "value": '/tmp/test_pig'}]),
+    }
+
+    response = self.c.post(reverse('pig:run'), data=post_data, follow=True)
+    self.wait_until_completion(json.loads(response.content)['id'])

+ 1 - 0
apps/pig/src/pig/urls.py

@@ -30,4 +30,5 @@ urlpatterns = patterns('pig.views',
   url(r'^copy/$', 'copy', name='copy'),
   url(r'^delete/$', 'delete', name='delete'),
   url(r'^watch/(?P<job_id>[-\w]+)$', 'watch', name='watch'),
+  url(r'^install_examples$', 'install_examples', name='install_examples'),
 )

+ 18 - 1
apps/pig/src/pig/views.py

@@ -32,6 +32,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from oozie.views.dashboard import show_oozie_error, check_job_access_permission
 
 from pig import api
+from pig.management.commands import pig_setup
 from pig.models import get_workflow_output, hdfs_link, PigScript,\
   create_or_update_script, get_scripts
 
@@ -99,7 +100,7 @@ def run(request):
 
   pig_script = create_or_update_script(**attrs)
 
-  params = request.POST.get('parameters')
+  params = request.POST.get('submissionVariables')
   oozie_id = api.get(request.fs, request.user).submit(pig_script, params)
 
   pig_script.update_from_dict({'job_id': oozie_id})
@@ -181,3 +182,19 @@ def watch(request, job_id):
   }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")
+
+
+def install_examples(request):
+  result = {'status': -1, 'message': ''}
+
+  if request.method != 'POST':
+    result['message'] = _('A POST request is required.')
+  else:
+    try:
+      pig_setup.Command().handle_noargs()
+      result['status'] = 0
+    except Exception, e:
+      LOG.exception(e)
+      result['message'] = str(e)
+
+  return HttpResponse(json.dumps(result), mimetype="application/json")

+ 2 - 1
apps/pig/static/js/pig.ko.js

@@ -338,7 +338,8 @@ var PigViewModel = function (props) {
           id: script.id(),
           name: script.name(),
           script: script.script(),
-          parameters: ko.utils.stringifyJson(self.submissionVariables()),
+          parameters: ko.utils.stringifyJson(script.parameters()),
+          submissionVariables: ko.utils.stringifyJson(self.submissionVariables()),
           resources: ko.toJSON(script.resources())
         },
         function (data) {

+ 1 - 1
desktop/core/src/desktop/templates/check_config.mako

@@ -22,7 +22,7 @@ from django.utils.translation import ugettext as _
     ${_('Configuration files located in')} <code>${conf_dir}</code>
     <br/><br/>
     % if error_list:
-      <div class="alert alert-error">${_('Potential misconfiguration detected. Fix and restart Hue.')}</div>
+      <div class="alert alert-warn">${_('Potential misconfiguration detected. Fix and restart Hue.')}</div>
       <br/>
         <table class="table table-striped">
       % for confvar, error in error_list:

+ 1 - 1
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -72,7 +72,7 @@ class OozieServerProvider(object):
       LOG.info(msg)
       raise Exception(msg)
     else:
-      LOG.info('[%d] Job %s tooke %d: %s' % (time.time(), job.id, time.time() - start, logs))
+      LOG.info('[%d] Job %s took %d: %s' % (time.time(), job.id, time.time() - start, logs))
 
     return job