Bläddra i källkod

HUE-1476 [fb] Submit oozie jobs directly from HDFS with FileBrowser

Works with:
  - worfklows
  - coodinators
  - bundles
Romain Rigaux 12 år sedan
förälder
incheckning
e86f6af2f7

+ 4 - 0
apps/filebrowser/src/filebrowser/templates/listdir.mako

@@ -73,6 +73,10 @@ ${ fb_components.menubar() }
             <!-- ko ifnot: trashEnabled -->
               <button class="btn fileToolbarBtn delete-link" title="${_('Delete forever')}" data-bind="enable: selectedFiles().length > 0, click: deleteSelected"><i class="fa fa-bolt"></i> ${_('Delete forever')}</button>
             <!-- /ko -->
+            <button class="btn fileToolbarBtn" title="${_('Submit')}"
+              data-bind="visible: selectedFiles().length == 1 && $.inArray(selectedFile().name, ['workflow.xml', 'coordinator.xml', 'bundle.xml']) > -1, click: submitSelected">
+              <i class="fa fa-play"></i> ${_('Submit')}
+            </button>
           </div>
         </%def>
 

+ 11 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -147,7 +147,7 @@ from django.utils.translation import ugettext as _
             <td class="center" data-bind="click: handleSelect" style="cursor: default">
                 <div data-bind="visible: name != '.' && name != '..', css: {hueCheckbox: name != '.' && name != '..', 'fa': name != '.' && name != '..', 'fa-check': selected}"></div>
             </td>
-            <td data-bind="click: $root.viewFile" class="left"><i data-bind="css: {'fa': true, 'fa-file-o': type == 'file', 'fa-folder': type != 'file', 'fa-folder-open': type != 'file' && hovered}"></i></td>
+            <td data-bind="click: $root.viewFile" class="left"><i data-bind="css: {'fa': true, 'fa-play': $.inArray(name, ['workflow.xml', 'coordinator.xml', 'bundle.xml']) > -1, 'fa-file-o': type == 'file', 'fa-folder': type != 'file', 'fa-folder-open': type != 'file' && hovered}"></i></td>
             <td data-bind="click: $root.viewFile, attr: {'title': tooltip}" rel="tooltip">
                 <strong><a href="#" data-bind="click: $root.viewFile, text: name"></a></strong>
             </td>
@@ -486,6 +486,8 @@ from django.utils.translation import ugettext as _
         </form>
     </div>
 
+    <div id="submit-wf-modal" class="modal hide"></div>
+
     <script type="text/javascript" charset="utf-8">
     // ajax modal windows
     function openChownWindow(path, user, group, next) {
@@ -1191,6 +1193,14 @@ from django.utils.translation import ugettext as _
         deleteSelected();
       };
 
+      self.submitSelected = function() {
+        $.get("${ url('oozie:submit_external_job', application_path='/') }../" + self.selectedFile().path, function (response) {
+            $('#submit-wf-modal').html(response);
+            $('#submit-wf-modal').modal('show');
+          }
+        );
+      }
+
       self.createDirectory = function (formElement) {
         $(formElement).attr("action", "/filebrowser/mkdir?next=${url('filebrowser.views.view', path=urlencode('/'))}" + "." + self.currentPath());
         return true;

+ 3 - 3
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako

@@ -67,7 +67,7 @@ ${ layout.menubar(section='workflows', dashboard=True) }
           % if parameters and len(parameters) < 10:
               <li class="nav-header">${ _('Variables') }</li>
               % for var, value in parameters.iteritems():
-                % if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath':
+                % if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path':
                   % if utils.is_linkable(var, str(value)):
                   <li rel="tooltip" title="${ var } : ${ str(value) }">
                     <a href="${ utils.hdfs_link_js(str(value)) }"><i class="fa fa-eye"></i> <span class="variable hide">${ var }</span></a>
@@ -362,8 +362,8 @@ ${ layout.menubar(section='workflows', dashboard=True) }
     $("*[rel=tooltip]").tooltip();
 
     $(".variable").each(function () {
-      if ($(this).text().length > 15) {
-        $(this).html($(this).text().substr(0, 14) + "&hellip;");
+      if ($(this).text().length > 25) {
+        $(this).html($(this).text().substr(0, 24) + "&hellip;");
       }
       $(this).removeClass("hide");
     });

Filskillnaden har hållts tillbaka eftersom den är för stor
+ 0 - 0
apps/oozie/src/oozie/tests.py


+ 2 - 0
apps/oozie/src/oozie/urls.py

@@ -96,5 +96,7 @@ urlpatterns += patterns(
   url(r'^rerun_oozie_bundle/(?P<job_id>[-\w]+)/(?P<app_path>.+?)$', 'rerun_oozie_bundle', name='rerun_oozie_bundle'),
   url(r'^manage_oozie_jobs/(?P<job_id>[-\w]+)/(?P<action>(start|suspend|resume|kill|rerun))$', 'manage_oozie_jobs', name='manage_oozie_jobs'),
 
+  url(r'^submit_external_job/(?P<application_path>.+?)$', 'submit_external_job', name='submit_external_job'),
+
   url(r'^list_oozie_info/$', 'list_oozie_info', name='list_oozie_info'),
 )

+ 44 - 1
apps/oozie/src/oozie/views/dashboard.py

@@ -17,6 +17,7 @@
 
 import json
 import logging
+import os
 import time
 
 from django.forms.formsets import formset_factory
@@ -28,16 +29,19 @@ from django.shortcuts import redirect
 
 from desktop.lib.django_util import render, encode_json_for_js
 from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.i18n import smart_str
 from desktop.lib.rest.http_client import RestException
 from desktop.lib.view_util import format_duration_in_millis
 from desktop.log.access import access_warn
+
 from liboozie.oozie_api import get_oozie
 from liboozie.submittion import Submission
 
 from oozie.conf import OOZIE_JOBS_COUNT
 from oozie.forms import RerunForm, ParameterForm, RerunCoordForm,\
   RerunBundleForm
-from oozie.models import History, Job, Workflow, utc_datetime_format
+from oozie.models import History, Job, Workflow, utc_datetime_format, Bundle,\
+  Coordinator
 from oozie.settings import DJANGO_APPS
 
 
@@ -491,6 +495,45 @@ def _rerun_bundle(request, oozie_id, args, params, properties):
                          detail=ex._headers.get('oozie-error-message', ex))
 
 
+def submit_external_job(request, application_path):
+  ParametersFormSet = formset_factory(ParameterForm, extra=0)
+
+  if request.method == 'POST':
+    params_form = ParametersFormSet(request.POST)
+
+    if params_form.is_valid():
+      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
+      application_name = os.path.basename(application_path)
+      application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else Workflow
+      mapping[application_class.get_application_path_key()] = application_path
+
+      try:
+        submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
+        job_id = submission.run(application_path)
+      except RestException, ex:
+        detail = ex._headers.get('oozie-error-message', ex)
+        if 'Max retries exceeded with url' in str(detail):
+          detail = '%s: %s' % (_('The Oozie server is not running'), detail)
+        LOG.error(smart_str(detail))
+        raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
+
+      request.info(_('Oozie job submitted'))
+      view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
+      return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
+    else:
+      request.error(_('Invalid submission form: %s' % params_form.errors))
+  else:
+    parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
+    initial_params = ParameterForm.get_initial_params(parameters)
+    params_form = ParametersFormSet(initial=initial_params)
+
+  popup = render('editor/submit_job_popup.mako', request, {
+                   'params_form': params_form,
+                   'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path})
+                 }, force_template=True).content
+  return HttpResponse(json.dumps(popup), mimetype="application/json")
+
+
 def massaged_workflow_actions_for_json(workflow_actions, oozie_coordinator, oozie_bundle):
   actions = []
 

+ 2 - 5
apps/oozie/src/oozie/views/editor.py

@@ -14,12 +14,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from desktop.models import Document
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+import json
 import logging
 import shutil
 import time
@@ -38,6 +34,7 @@ from desktop.lib.django_util import render, extract_field_data
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
 from desktop.lib.rest.http_client import RestException
+from desktop.models import Document
 from liboozie.submittion import Submission
 
 from filebrowser.lib.archives import archive_factory

+ 1 - 1
apps/pig/src/pig/conf.py

@@ -42,7 +42,7 @@ def config_validator(user):
   res = []
 
   if not 'test' in sys.argv: # Avoid tests hanging
-    status = get_oozie_status()
+    status = get_oozie_status(user)
 
     if 'NORMAL' not in status:
       res.append((NICE_NAME, _("The app won't work without a running Oozie server")))

+ 36 - 3
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -17,6 +17,7 @@
 
 import errno
 import logging
+import os
 import time
 
 from django.utils.translation import ugettext as _
@@ -27,6 +28,7 @@ from hadoop.fs.hadoopfs import Hdfs
 
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
+from jobsub.parameterization import find_variables
 
 LOG = logging.getLogger(__name__)
 
@@ -61,7 +63,7 @@ class Submission(object):
       res += " -- " + self.oozie_id
     return res
 
-  def run(self):
+  def run(self, deployment_dir=None):
     """
     Take care of all the actions of submitting a Oozie workflow.
     Returns the oozie job id if all goes well.
@@ -71,13 +73,14 @@ class Submission(object):
 
     jobtracker = cluster.get_cluster_addr_for_job_submission()
 
-    deployment_dir = self.deploy()
+    if deployment_dir is None:
+      deployment_dir = self.deploy()
 
     self._update_properties(jobtracker, deployment_dir)
     self.oozie_id = self.api.submit_job(self.properties)
     LOG.info("Submitted: %s" % (self,))
 
-    if self.job.get_type() == 'workflow':
+    if self._is_workflow():
       self.api.job_control(self.oozie_id, 'start')
       LOG.info("Started: %s" % (self,))
 
@@ -148,6 +151,28 @@ class Submission(object):
 
     return deployment_dir
 
+  def get_external_parameters(self, application_path):
+    """From XML and job.properties HDFS files"""
+    deployment_dir = os.path.dirname(application_path)
+    xml = self.fs.do_as_user(self.user, self.fs.read, application_path, 0, 1 * 1024**2)
+
+    properties_file = deployment_dir + '/job.properties'
+    if self.fs.do_as_user(self.user, self.fs.exists, properties_file):
+      properties = self.fs.do_as_user(self.user, self.fs.read, properties_file, 0, 1 * 1024**2)
+    else:
+      properties = None
+
+    return self._get_external_parameters(xml, properties)
+
+  def _get_external_parameters(self, xml, properties=None):
+    from oozie.models import DATASET_FREQUENCY
+    parameters = dict([(var, '') for var in find_variables(xml) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
+
+    if properties:
+      parameters.update(dict([line.strip().split('=')
+                              for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
+    return parameters
+
   def _update_properties(self, jobtracker_addr, deployment_dir):
     if self.fs and self.jt:
       self.properties.update({
@@ -245,6 +270,14 @@ class Submission(object):
                "%s (owner %s). Caused by: %s",
                self.job.name, self.user, ex)
 
+  def _is_workflow(self):
+    from oozie.models import Workflow
+    return Workflow.get_application_path_key() in self.properties
+
+  def _is_coordinator(self):
+    from oozie.models import Coordinator
+    return Coordinator.get_application_path_key() in self.properties
+
 
 def create_directories(fs, directory_list=[]):
   # If needed, create the remote home, deployment and data directories

+ 66 - 1
desktop/libs/liboozie/src/liboozie/tests.py

@@ -17,9 +17,10 @@
 
 import logging
 
+from django.contrib.auth.models import User
 from nose.tools import assert_equal
-from oozie.tests import MockOozieApi
 
+from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import reformat_xml
 
 from hadoop import cluster
@@ -27,6 +28,7 @@ from hadoop.conf import HDFS_CLUSTERS, MR_CLUSTERS, YARN_CLUSTERS
 from liboozie.types import WorkflowAction, Coordinator
 from liboozie.submittion import Submission
 from liboozie.utils import config_gen
+from oozie.tests import MockOozieApi
 
 
 LOG = logging.getLogger(__name__)
@@ -108,3 +110,66 @@ def test_update_properties():
     cluster.clear_caches()
     for reset in finish:
       reset()
+
+
+class TestSubmission():
+
+  def setUp(self):
+    self.c = make_logged_in_client(is_superuser=False)
+#    grant_access("test", "test", "oozie")
+#    add_to_group("test")
+    self.user = User.objects.get(username='test')
+
+  def test_get_external_parameters(self):
+    xml = """
+<workflow-app name="Pig" xmlns="uri:oozie:workflow:0.4">
+    <start to="Pig"/>
+    <action name="Pig">
+        <pig>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <prepare>
+                  <delete path="${output}"/>
+            </prepare>
+            <script>aggregate.pig</script>
+              <argument>-param</argument>
+              <argument>INPUT=${input}</argument>
+              <argument>-param</argument>
+              <argument>OUTPUT=${output}</argument>
+        </pig>
+        <ok to="end"/>
+        <error to="kill"/>
+    </action>
+    <kill name="kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <end name="end"/>
+</workflow-app>
+    """
+
+    properties = """
+#
+# Licensed to the Hue
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
+    """
+    parameters = Submission(self.user)._get_external_parameters(xml, properties)
+
+    assert_equal({'oozie.use.system.libpath': 'true',
+                   'input': '',
+                   'jobTracker': 'localhost:8021',
+                   'oozie.wf.application.path': '${nameNode}/user/${user.name}/${examplesRoot}/apps/pig',
+                   'examplesRoot': 'examples',
+                   'output': '',
+                   'nameNode': 'hdfs://localhost:8020',
+                   'queueName': 'default'
+                  },
+                 parameters)

Vissa filer visades inte eftersom för många filer har ändrats