Browse Source

HUE-1213 [jobsub] Remove old examples

Abraham Elmahrek 12 năm trước cách đây
mục cha
commit
100843eaa3

BIN
apps/jobsub/data/examples/hadoop-examples.jar


+ 0 - 81
apps/jobsub/data/examples/wordcount.py

@@ -1,81 +0,0 @@
-#!/usr/bin/python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Wordcount example, for Hadoop streaming.
-#
-# Test with:
-#  $(echo "hello moon"; echo "hello sun") | python wordcount.py map | sort | python wordcount.py reduce
-#  hello 2
-#  moon  1
-#  sun 1
-
-import sys
-import re
-import __builtin__
-
-def map(line):
-  for word in re.split("\W", line):
-    if word:
-      emit(word, str(1))
-
-def reduce(word, counts):
-  emit(word, str(sum(__builtin__.map(int, counts))))
-
-def emit(key, value):
-  """
-  Emits a key->value pair.  Key and value should be strings.
-  """
-  print "\t".join( (key, value) )
-
-def run_map():
-  """Calls map() for each input value."""
-  for line in sys.stdin:
-    line = line.rstrip()
-    map(line)
-
-def run_reduce():
-  """Gathers reduce() data in memory, and calls reduce()."""
-  prev_key = None
-  values = []
-  for line in sys.stdin:
-    line = line.rstrip()
-    key, value = re.split("\t", line, 1)
-    if prev_key == key:
-      values.append(value)
-    else:
-      if prev_key is not None:
-        reduce(prev_key, values)
-      prev_key = key
-      values = [ value ]
-
-  if prev_key is not None:
-    reduce(prev_key, values)
-
-def main():
-  """Runs map or reduce code, per arguments."""
-  if len(sys.argv) != 2 or sys.argv[1] not in ("map", "reduce"):
-    print "Usage: %s <map|reduce>" % sys.argv[0]
-    sys.exit(1)
-  if sys.argv[1] == "map":
-    run_map()
-  elif sys.argv[1] == "reduce":
-    run_reduce()
-  else:
-    assert False
-
-if __name__ == "__main__":
-  main()

+ 0 - 19
apps/jobsub/data/script_templates/streaming.pl

@@ -1,19 +0,0 @@
-#!/usr/bin/perl
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-print "Hi there!\n";

+ 0 - 83
apps/jobsub/data/script_templates/streaming.py

@@ -1,83 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# Template for python Hadoop streaming.  Fill in the map() and reduce()
-# functions, which should call emit(), as appropriate.
-#
-# Test your script with
-#  cat input | python wordcount.py map | sort | python wordcount.py reduce
-
-import sys
-import re
-
-def map(line):
-  # Fill this in!
-  pass
-
-def reduce(key, values):
-  # Fill this in
-  pass
-
-
-# Common library code follows:
-
-def emit(key, value):
-  """
-  Emits a key->value pair.  Key and value should be strings.
-  """
-  print "\t".join( (key, value) )
-
-def run_map():
-  """Calls map() for each input value."""
-  for line in sys.stdin:
-    line = line.rstrip()
-    map(line)
-
-def run_reduce():
-  """Gathers reduce() data in memory, and calls reduce()."""
-  prev_key = None
-  values = []
-  for line in sys.stdin:
-    line = line.rstrip()
-    key, value = re.split("\t", line, 1)
-    if prev_key == key:
-      values.append(value)
-    else:
-      if prev_key is not None:
-        reduce(prev_key, values)
-      prev_key = key
-      values = [ value ]
-
-  if prev_key is not None:
-    reduce(prev_key, values)
-
-def main():
-  """Runs map or reduce code, per arguments."""
-  if len(sys.argv) != 2 or sys.argv[1] not in ("map", "reduce"):
-    print "Usage: %s <map|reduce>" % sys.argv[0]
-    sys.exit(1)
-  if sys.argv[1] == "map":
-    run_map()
-  elif sys.argv[1] == "reduce":
-    run_reduce()
-  else:
-    assert False
-
-if __name__ == "__main__":
-  main()

+ 0 - 67
apps/jobsub/src/jobsub/fixtures/example_data.xml

@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<django-objects version="1.0">
-  <!-- Sample User -->
-  <object pk="1100713" model="auth.user">
-    <field type="CharField" name="username">sample</field>
-    <field type="CharField" name="first_name"></field>
-    <field type="CharField" name="last_name"></field>
-    <field type="CharField" name="email"></field>
-    <field type="CharField" name="password">!</field>
-    <field type="BooleanField" name="is_staff">False</field>
-    <field type="BooleanField" name="is_active">False</field>
-    <field type="BooleanField" name="is_superuser">False</field>
-    <field type="DateTimeField" name="last_login">2009-09-18 22:27:14</field>
-    <field type="DateTimeField" name="date_joined">2009-09-18 22:06:38</field>
-    <field to="auth.group" name="groups" rel="ManyToManyRel"></field>
-    <field to="auth.permission" name="user_permissions" rel="ManyToManyRel"></field>
-  </object>
-  <object pk="1" model="jobsub.oozieaction">
-    <field type="CharField" name="action_type">streaming</field>
-  </object>
-  <object pk="2" model="jobsub.oozieaction">
-    <field type="CharField" name="action_type">mapreduce</field>
-  </object>
-  <object pk="3" model="jobsub.oozieaction">
-    <field type="CharField" name="action_type">mapreduce</field>
-  </object>
-  <object pk="1" model="jobsub.ooziedesign">
-    <field to="auth.user" name="owner" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="name">streaming_wordcount</field>
-    <field type="CharField" name="description">[Sample] Wordcount</field>
-    <field type="DateTimeField" name="last_modified">2012-04-02 01:13:31</field>
-    <field to="jobsub.oozieaction" name="root_action" rel="ManyToOneRel">1</field>
-  </object>
-  <object pk="2" model="jobsub.ooziedesign">
-    <field to="auth.user" name="owner" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="name">grep_example</field>
-    <field type="CharField" name="description">[Sample] Grep for dream in some Shakespearean text</field>
-    <field type="DateTimeField" name="last_modified">2012-04-02 02:04:34</field>
-    <field to="jobsub.oozieaction" name="root_action" rel="ManyToOneRel">2</field>
-  </object>
-  <object pk="3" model="jobsub.ooziedesign">
-    <field to="auth.user" name="owner" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="name">sleep_job</field>
-    <field type="CharField" name="description">[Sample] Sleep Job</field>
-    <field type="DateTimeField" name="last_modified">2012-04-02 03:06:02</field>
-    <field to="jobsub.oozieaction" name="root_action" rel="ManyToOneRel">3</field>
-  </object>
-  <object pk="2" model="jobsub.ooziemapreduceaction">
-    <field type="CharField" name="files">[]</field>
-    <field type="CharField" name="archives">[]</field>
-    <field type="CharField" name="job_properties">[{"name":"mapred.mapper.regex","value":"dream"},{"name":"mapred.input.dir","value":"/user/hue/jobsub/sample_data"},{"name":"mapred.output.dir","value":"$output_dir"},{"name":"mapred.mapper.class","value":"org.apache.hadoop.mapred.lib.RegexMapper"},{"name":"mapred.combiner.class","value":"org.apache.hadoop.mapred.lib.LongSumReducer"},{"name":"mapred.reducer.class","value":"org.apache.hadoop.mapred.lib.LongSumReducer"},{"name":"mapred.output.key.class","value":"org.apache.hadoop.io.Text"},{"name":"mapred.output.value.class","value":"org.apache.hadoop.io.LongWritable"}]</field>
-    <field type="CharField" name="jar_path">/user/hue/jobsub/examples/hadoop-examples.jar</field>
-  </object>
-  <object pk="3" model="jobsub.ooziemapreduceaction">
-    <field type="CharField" name="files">[]</field>
-    <field type="CharField" name="archives">[]</field>
-    <field type="CharField" name="job_properties">[{"name":"mapred.map.tasks","value":"$num_maps"},{"name":"mapred.reduce.tasks","value":"$num_reduces"},{"name":"mapred.mapper.class","value":"org.apache.hadoop.examples.SleepJob"},{"name":"mapred.reducer.class","value":"org.apache.hadoop.examples.SleepJob"},{"name":"mapred.mapoutput.key.class","value":"org.apache.hadoop.io.IntWritable"},{"name":"mapred.mapoutput.value.class","value":"org.apache.hadoop.io.NullWritable"},{"name":"mapred.output.format.class","value":"org.apache.hadoop.mapred.lib.NullOutputFormat"},{"name":"mapred.input.format.class","value":"org.apache.hadoop.examples.SleepJob$$SleepInputFormat"},{"name":"mapred.partitioner.class","value":"org.apache.hadoop.examples.SleepJob"},{"name":"mapred.speculative.execution","value":"false"},{"name":"sleep.job.map.sleep.time","value":"$map_sleep_time"},{"name":"sleep.job.reduce.sleep.time","value":"$reduce_sleep_time"}]</field>
-    <field type="CharField" name="jar_path">/user/hue/jobsub/examples/hadoop-examples.jar</field>
-  </object>
-  <object pk="1" model="jobsub.ooziestreamingaction">
-    <field type="CharField" name="files">["/user/hue/jobsub/examples/wordcount.py"]</field>
-    <field type="CharField" name="archives">[]</field>
-    <field type="CharField" name="job_properties">[{"name":"mapred.input.dir","value":"/user/hue/jobsub/sample_data"},{"name":"mapred.output.dir","value":"$output_directory"},{"name":"mapred.reduce.tasks","value":"1"},{"name":"oozie.use.system.libpath","value":"true"}]</field>
-    <field type="CharField" name="mapper">python wordcount.py map</field>
-    <field type="CharField" name="reducer">python wordcount.py reduce</field>
-  </object>
-</django-objects>

+ 0 - 0
apps/jobsub/src/jobsub/management/__init__.py


+ 0 - 0
apps/jobsub/src/jobsub/management/commands/__init__.py


+ 0 - 176
apps/jobsub/src/jobsub/management/commands/jobsub_setup.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Copies a handful of files over to the remote filesystem.
-# The source for this operation ought to be part of the
-# build process; this is currently a bit ad-hoc.
-
-import os
-import posixpath
-import logging
-import shutil
-
-import django.core
-from django.core.management.base import NoArgsCommand
-from django.contrib.auth.models import User
-
-from hadoop import cluster
-import jobsub.conf
-from jobsub.submit import Submission
-
-from django.utils.translation import ugettext as _
-
-LOG = logging.getLogger(__name__)
-
-
-# The setup_level value for the CheckForSetup table
-JOBSUB_SETUP_LEVEL = 200        # Stands for Hue 2.0.0
-
-
-class Command(NoArgsCommand):
-  """Creates file system for testing."""
-  def handle_noargs(self, **options):
-    remote_fs = cluster.get_hdfs()
-    if hasattr(remote_fs, "setuser"):
-      remote_fs.setuser(remote_fs.DEFAULT_USER)
-    LOG.info("Using remote fs: %s" % str(remote_fs))
-
-    # Create remote data directory if needed
-    remote_data_dir = Submission.create_data_dir(remote_fs)
-
-    # Copy over examples/
-    for dirname in ("examples",):
-      local_dir = os.path.join(jobsub.conf.LOCAL_DATA_DIR.get(), dirname)
-      remote_dir = posixpath.join(remote_data_dir, dirname)
-      copy_dir(local_dir, remote_fs, remote_dir)
-
-    # Copy over sample_data/
-    copy_dir(jobsub.conf.SAMPLE_DATA_DIR.get(),
-      remote_fs,
-      posixpath.join(remote_data_dir, "sample_data"))
-
-    # Write out the models too
-    fixture_path = os.path.join(os.path.dirname(__file__), "..", "..", "fixtures", "example_data.xml")
-    examples = django.core.serializers.deserialize("xml", open(fixture_path))
-    sample_user = None
-    sample_oozie_designs = []
-    sample_oozie_abstract_actions = {}      # pk -> object
-    sample_oozie_concrete_actions = {}      # oozieaction_ptr_id -> object
-
-    for example in examples:
-      if isinstance(example.object, User):
-        sample_user = example
-      elif isinstance(example.object, jobsub.models.OozieDesign):
-        sample_oozie_designs.append(example)
-      elif type(example.object) in (jobsub.models.OozieMapreduceAction,
-                                    jobsub.models.OozieJavaAction,
-                                    jobsub.models.OozieStreamingAction):
-        key = example.object.oozieaction_ptr_id
-        sample_oozie_concrete_actions[key] = example
-      elif type(example.object) is jobsub.models.OozieAction:
-        key = example.object.pk
-        sample_oozie_abstract_actions[key] = example
-      else:
-        raise Exception(_("Unexpected fixture type."))
-
-    if sample_user is None:
-      raise Exception(_("Expected sample user fixture."))
-
-    # Create the sample user if it doesn't exist
-    USERNAME = 'sample'
-    try:
-      sample_user.object = User.objects.get(username=USERNAME)
-    except User.DoesNotExist:
-      sample_user.object = User.objects.create(username=USERNAME, password='!', is_active=False, is_superuser=False, id=1100713, pk=1100713)
-
-    # Create the designs
-    for d in sample_oozie_designs:
-      #
-      # OozieDesign          ----many-to-one--->  OozieAction
-      #
-      # OozieMapreduceAction -----one-to-one--->  OozieAction
-      # OozieStreamingAction -----one-to-one--->  OozieAction
-      # OozieJavaAction      -----one-to-one--->  OozieAction
-      #
-      # We find the OozieAction pk and link everything back together
-      #
-      abstract_action_id = d.object.root_action_id
-      abstract_action = sample_oozie_abstract_actions[abstract_action_id]
-      concrete_action = sample_oozie_concrete_actions[str(abstract_action_id)]
-
-      concrete_action.object.action_type = abstract_action.object.action_type
-      concrete_action.object.pk = None
-      concrete_action.object.id = None
-      concrete_action.object.save()
-
-      d.object.id = None
-      d.object.pk = None
-      d.object.owner_id = sample_user.object.id
-      d.object.root_action = concrete_action.object
-      d.object.save()
-
-    # Upon success, write to the database
-    try:
-      entry = jobsub.models.CheckForSetup.objects.get(id=1)
-    except jobsub.models.CheckForSetup.DoesNotExist:
-      entry = jobsub.models.CheckForSetup(id=1)
-    entry.setup_run = True
-    entry.setup_level = JOBSUB_SETUP_LEVEL
-    entry.save()
-
-  def has_been_setup(self):
-    """
-    Returns true if we think job sub examples have been setup.
-    """
-    try:
-      entry = jobsub.models.CheckForSetup.objects.get(id=1)
-    except jobsub.models.CheckForSetup.DoesNotExist:
-      return False
-    return entry.setup_run and entry.setup_level >= JOBSUB_SETUP_LEVEL
-
-
-def copy_dir(local_dir, remote_fs, remote_dir):
-  # Hadoop mkdir is always recursive.
-  remote_fs.mkdir(remote_dir)
-  for f in os.listdir(local_dir):
-    local_src = os.path.join(local_dir, f)
-    remote_dst = posixpath.join(remote_dir, f)
-    copy_file(local_src, remote_fs, remote_dst)
-
-
-CHUNK_SIZE = 65536
-
-def copy_file(local_src, remote_fs, remote_dst):
-  if remote_fs.exists(remote_dst):
-    LOG.info("%s already exists.  Skipping." % remote_dst)
-    return
-  else:
-    LOG.info("%s does not exist. trying to copy" % remote_dst)
-
-  if os.path.isfile(local_src):
-    src = file(local_src)
-    try:
-      dst = remote_fs.open(remote_dst, "w")
-      try:
-        shutil.copyfileobj(src, dst, CHUNK_SIZE)
-        LOG.info("Copied %s -> %s" % (local_src, remote_dst))
-      finally:
-        dst.close()
-    finally:
-      src.close()
-  else:
-    LOG.info("Skipping %s (not a file)" % local_src)

+ 0 - 32
apps/jobsub/src/jobsub/middleware.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from desktop.lib.exceptions import StructuredException
-
-from jobsubd.ttypes import SubmissionError
-
-class SubmissionErrorRecastMiddleware(object):
-  """
-  When this middleware sees a SubmissionError,
-  it adds a response_data field to it.
-
-  We do this instead of "monkey-patching" a response_data
-  property into SubmissionError.
-  """
-  def process_exception(self, request, exception):
-    if isinstance(exception, SubmissionError) and not hasattr(SubmissionError, "response_data"):
-    	raise StructuredException(code="SUBMISSION_ERROR", message=exception.message)

+ 0 - 228
apps/jobsub/src/jobsub/submit.py

@@ -1,228 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Handle design submission.
-"""
-
-import errno
-import logging
-
-from desktop.lib import django_mako
-from desktop.lib.exceptions_renderable import PopupException
-import hadoop.cluster
-from hadoop.fs.hadoopfs import Hdfs
-
-from jobsub import conf, models
-from liboozie.oozie_api import get_oozie
-
-from django.utils.translation import ugettext as _
-
-LOG = logging.getLogger(__name__)
-
-
-class Submission(object):
-  """Represents one submission"""
-  def __init__(self, design_obj, fs):
-    self._design_obj = design_obj
-    self._username = design_obj.owner.username
-    self._action = design_obj.get_root_action()
-    self._fs = fs
-    self._job_id = None       # The oozie workflow instance id
-
-  def __unicode__(self):
-    res = _("Submission for job design '%(name)s' (id %(id)s, owner %(username)s).") % \
-        dict(name=self._design_obj.name, id=self._design_obj.id, username=self._username)
-    if self.job_id:
-      res += " -- " + self.job_id
-    return res
-
-  @property
-  def job_id(self):
-    return self._job_id
-
-  def _do_as(self, username, fn, *args, **kwargs):
-    curr_user = self._fs.setuser(username)
-    try:
-      fn(*args, **kwargs)
-    finally:
-      self._fs.setuser(curr_user)
-
-
-  def run(self):
-    """
-    Take care of all the actions of submitting a workflow/design.
-    Returns the oozie job id if all goes well.
-    """
-    if self.job_id is not None:
-      raise Exception(_("Job design already submitted (Oozie job id %(id)s).") % dict(id=(self.job_id,)))
-
-    fs_defaultfs = self._fs.fs_defaultfs
-    jobtracker = hadoop.cluster.get_cluster_addr_for_job_submission()
-
-    try:
-      wf_dir = self._get_and_create_deployment_dir()
-    except Exception, ex:
-      LOG.exception("Failed to access deployment directory")
-      raise PopupException(message=_("Failed to access deployment directory."),
-                           detail=str(ex))
-
-    wf_xml = self._generate_workflow_xml(fs_defaultfs)
-    self._do_as(self._username, self._copy_files, wf_dir, wf_xml)
-    LOG.info("Prepared deployment directory at '%s' for %s" % (wf_dir, self))
-    LOG.info("Submitting design id %s to %s as `%s'" % (self._design_obj.id, jobtracker, self._username))
-
-    try:
-      prev = get_oozie().setuser(self._username)
-      self._job_id = get_oozie().submit_workflow(
-            self._fs.get_hdfs_path(wf_dir),
-            properties=self._get_properties(jobtracker))
-      LOG.info("Submitted: %s" % (self,))
-
-      # Now we need to run it
-      get_oozie().job_control(self.job_id, 'start')
-      LOG.info("Started: %s" % (self,))
-    finally:
-      get_oozie().setuser(prev)
-
-    return self.job_id
-
-
-  def _get_properties(self, jobtracker_addr):
-    res = { 'jobTracker': jobtracker_addr }
-    if self._design_obj.get_root_action().action_type == \
-          models.OozieStreamingAction.ACTION_TYPE:
-      res['oozie.use.system.libpath'] = 'true'
-    return res
-
-
-  def _copy_files(self, wf_dir, wf_xml):
-    """
-    Copy the files over to the deployment directory. This should run as the
-    design owner.
-    """
-    xml_path = self._fs.join(wf_dir, 'workflow.xml')
-    self._fs.create(xml_path, overwrite=True, permission=0644, data=wf_xml)
-    LOG.debug("Created %s" % (xml_path,))
-
-    # Copy the jar over
-    if self._action.action_type in (models.OozieMapreduceAction.ACTION_TYPE,
-                                    models.OozieJavaAction.ACTION_TYPE):
-      lib_path = self._fs.join(wf_dir, 'lib')
-      if self._fs.exists(lib_path):
-        LOG.debug("Cleaning up old %s" % (lib_path,))
-        self._fs.rmtree(lib_path)
-
-      self._fs.mkdir(lib_path, 0755)
-      LOG.debug("Created %s" % (lib_path,))
-
-      jar = self._action.jar_path
-      self._fs.copyfile(jar, self._fs.join(lib_path, self._fs.basename(jar)))
-
-
-  def _generate_workflow_xml(self, namenode):
-    """Return a string that is the workflow.xml of this workflow"""
-    action_type = self._design_obj.root_action.action_type
-    data = {
-      'design': self._design_obj,
-      'nameNode': namenode,
-    }
-
-    if action_type == models.OozieStreamingAction.ACTION_TYPE:
-      tmpl = "workflow-streaming.xml.mako"
-    elif action_type == models.OozieMapreduceAction.ACTION_TYPE:
-      tmpl = "workflow-mapreduce.xml.mako"
-    elif action_type == models.OozieJavaAction.ACTION_TYPE:
-      tmpl = "workflow-java.xml.mako"
-    return django_mako.render_to_string(tmpl, data)
-
-
-  def _get_and_create_deployment_dir(self):
-    """
-    Return the workflow deployment directory in HDFS,
-    creating it if necessary.
-
-    May raise Exception.
-    """
-    path = self._get_deployment_dir()
-    try:
-      statbuf = self._fs.stats(path)
-      if not statbuf.isDir:
-        msg = "Workflow deployment path is not a directory: %s." % (path,)
-        LOG.error(msg)
-        raise Exception(msg)
-      return path
-    except IOError, ex:
-      if ex.errno != errno.ENOENT:
-        msg = "Error accessing workflow directory '%s': %s." % (path, ex)
-        LOG.exception(msg)
-        raise IOError(ex.errno, msg)
-      self._create_deployment_dir(path)
-      return path
-
-
-  def _create_deployment_dir(self, path):
-    # Make sure the root data dir exists
-    self.create_data_dir(self._fs)
-
-    # The actual deployment dir should be 0711 owned by the user
-    self._do_as(self._username, self._fs.mkdir, path, 0711)
-
-
-  @classmethod
-  def create_data_dir(cls, fs):
-    # If needed, create the remote home and data directories
-    remote_data_dir = conf.REMOTE_DATA_DIR.get()
-    user = fs.user
-
-    try:
-      fs.setuser(fs.DEFAULT_USER)
-      if not fs.exists(remote_data_dir):
-        remote_home_dir = Hdfs.join('/user', fs.user)
-        if remote_data_dir.startswith(remote_home_dir):
-          # Home is 755
-          fs.create_home_dir(remote_home_dir)
-        # Shared by all the users
-        fs.mkdir(remote_data_dir, 01777)
-    finally:
-      fs.setuser(user)
-
-    return remote_data_dir
-
-
-  def _get_deployment_dir(self):
-    """Return the workflow deployment directory"""
-    if self._fs is None:
-      raise PopupException(_("Failed to obtain HDFS reference. "
-                           "Check your configuration."))
-
-    # We could have collision with usernames. But there's no good separator.
-    # Hope people don't create crazy usernames.
-    return self._fs.join(conf.REMOTE_DATA_DIR.get(),
-                         "_%s_-design-%s" % (self._username, self._design_obj.id))
-
-
-  def remove_deployment_dir(self):
-    """Delete the workflow deployment directory. Does not throw."""
-    try:
-      path = self._get_deployment_dir()
-      if self._do_as(self._username, self._fs.exists, path):
-        self._do_as(self._username, self._fs.rmtree, path)
-    except Exception, ex:
-      LOG.warn("Failed to clean up workflow deployment directory for "
-               "%s (owner %s). Caused by: %s",
-               self._design_obj.name, self._design_obj.owner.username, ex)

+ 0 - 3
apps/jobsub/src/jobsub/views.py

@@ -45,9 +45,6 @@ from oozie.forms import design_form_by_type
 from oozie.utils import model_to_dict, format_dict_field_values,\
                         sanitize_node_dict
 
-# To re-enable
-from jobsub.management.commands import jobsub_setup
-
 
 LOG = logging.getLogger(__name__)
 

+ 17 - 12
apps/oozie/src/oozie/tests.py

@@ -30,8 +30,7 @@ from django.core.urlresolvers import reverse
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_permission
-from jobsub.management.commands import jobsub_setup
-from jobsub.models import OozieDesign
+from jobsub.models import OozieDesign, OozieMapreduceAction
 from liboozie import oozie_api
 from liboozie.conf import OOZIE_URL
 from liboozie.oozie_api_test import OozieServerProvider
@@ -2794,16 +2793,22 @@ class GeneralTestsWithOozie(OozieBase):
     OozieBase.setUp(self)
 
   def test_import_jobsub_actions(self):
-    # Setup jobsub examples
-    if not jobsub_setup.Command().has_been_setup():
-      jobsub_setup.Command().handle()
-
-    # There should be 3 from examples
-    jobsub_design = OozieDesign.objects.filter(root_action__action_type='streaming')[0]
-    action = convert_jobsub_design(jobsub_design)
-    assert_equal(jobsub_design.name, action.name)
-    assert_equal(jobsub_design.description, action.description)
-    assert_equal('streaming', action.node_type)
+    design = OozieDesign(owner=self.user, name="test")
+    action = OozieMapreduceAction(jar_path='/tmp/test.jar')
+    action.action_type = OozieMapreduceAction.ACTION_TYPE
+    action.save()
+    design.root_action = action
+    design.save()
+
+    try:
+      # There should be 3 from examples
+      action = convert_jobsub_design(design)
+      assert_equal(design.name, action.name)
+      assert_equal(design.description, action.description)
+      assert_equal(OozieMapreduceAction.ACTION_TYPE, action.node_type)
+    finally:
+      OozieDesign.objects.all().delete()
+      OozieMapreduceAction.objects.all().delete()
 
 
 class TestUtils(OozieMockBase):