浏览代码

HUE-752 [jobsub] Add tests to Oozie API

Setup and start an Oozie instance during the tests
Requires "tools/jenkins/jenkins.sh" to be previously ran because of
https://issues.cloudera.org/browse/HUE-861
Add tests of the Oozie API
Add many tests to the Oozie App
Split Oozie App tests between requires Hadoop and not
Romain Rigaux 13 年之前
父节点
当前提交
6df5059f74

+ 0 - 0
apps/oozie/examples/sleep/empty


+ 0 - 62
apps/oozie/examples/sleep/workflow.xml

@@ -1,62 +0,0 @@
-
-<workflow-app name="SleepWorkflow" xmlns="uri:oozie:workflow:0.2">
-    <start to="Sleep"/>
-    <action name="Sleep">
-        <map-reduce>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property>
-                    <name>mapred.reduce.tasks</name>
-                    <value>1</value>
-                </property>
-                <property>
-                    <name>mapred.mapper.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.reducer.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.key.class</name>
-                    <value>org.apache.hadoop.io.IntWritable</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.value.class</name>
-                    <value>org.apache.hadoop.io.NullWritable</value>
-                </property>
-                <property>
-                    <name>mapred.output.format.class</name>
-                    <value>org.apache.hadoop.mapred.lib.NullOutputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.input.format.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.partitioner.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.speculative.execution</name>
-                    <value>false</value>
-                </property>
-                <property>
-                    <name>sleep.job.map.sleep.time</name>
-                    <value>0</value>
-                </property>
-                <property>
-                    <name>sleep.job.reduce.sleep.time</name>
-                    <value>${REDUCER_SLEEP_TIME}</value>
-                </property>
-            </configuration>
-        </map-reduce>
-        <ok to="end"/>
-        <error to="kill"/>
-    </action>
-    <kill name="kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

+ 1 - 1
apps/oozie/src/oozie/fixtures/initial_example_data.json

@@ -28,7 +28,7 @@
     }
   },
   {
-    "pk": 7,
+    "pk": null,
     "model": "oozie.job",
     "fields": {
       "is_shared": true,

+ 8 - 8
apps/oozie/src/oozie/migrations/0007_auto__chg_field_sqoop_script_path.py

@@ -5,19 +5,19 @@ from south.v2 import SchemaMigration
 from django.db import models
 
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.TextField')(blank=True))
-    
-    
+
+
     def backwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True))
-    
-    
+
+
     models = {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
@@ -251,5 +251,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
     }
-    
+
     complete_apps = ['oozie']

文件差异内容过多而无法显示
+ 545 - 436
apps/oozie/src/oozie/tests.py


+ 22 - 3
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -17,6 +17,7 @@
 
 
 import atexit
+import getpass
 import logging
 import pwd
 import os
@@ -95,6 +96,10 @@ class PseudoHdfs4(object):
   def mr1_env(self):
     return self._mr1_env
 
+  @property
+  def log_dir(self):
+    return self._log_dir
+
   @property
   def fs_default_name(self):
     return self._fs_default_name
@@ -237,6 +242,16 @@ class PseudoHdfs4(object):
     # Start MR1
     self._start_mr1(env)
 
+    # Make sure /tmp is 1777
+    self.fs.setuser(self.superuser)
+    if not self.fs.isdir('/tmp'):
+      self.fs.mkdir('/tmp', 01777)
+    else:
+      self.fs.chmod('/tmp', 01777)
+
+    self.fs.chmod(self._tmpdir, 01777)
+    self.fs.chmod(self._tmpdir + '/hadoop_tmp_dir/mapred', 01777)
+
 
   def _start_mr1(self, env):
     LOG.info("Starting MR1")
@@ -291,7 +306,6 @@ class PseudoHdfs4(object):
       stdout.close()
       stderr.close()
 
-
   def _log_exit(self, proc_name, exit_code):
     """Log the stdout and stderr for a process"""
     LOG.info('%s exited with %s' % (proc_name, exit_code))
@@ -392,8 +406,6 @@ class PseudoHdfs4(object):
       'dfs.datanode.ipc.address': 'localhost:0',
       'dfs.replication': 1,
       'dfs.safemode.min.datanodes': 1,
-      'hadoop.proxyuser.hue.hosts': '*',
-      'hadoop.proxyuser.hue.groups': '*',
     }
     write_config(hdfs_configs, self._tmppath('conf/hdfs-site.xml'))
 
@@ -410,6 +422,12 @@ class PseudoHdfs4(object):
       'hadoop.security.authentication': 'simple',
       'hadoop.proxyuser.%s.groups' % (self.superuser,): 'users,supergroup',
       'hadoop.proxyuser.%s.hosts' % (self.superuser,): 'localhost',
+      'hadoop.proxyuser.hue.hosts': '*',
+      'hadoop.proxyuser.hue.groups': '*',
+      'hadoop.proxyuser.oozie.hosts': '*',
+      'hadoop.proxyuser.oozie.groups': '*',
+      'hadoop.proxyuser.%s.hosts' % getpass.getuser(): '*',
+      'hadoop.proxyuser.%s.groups' % getpass.getuser(): '*',
       'hadoop.tmp.dir': self._tmppath('hadoop_tmp_dir'),
     }
     write_config(core_configs, self._tmppath('conf/core-site.xml'))
@@ -467,6 +485,7 @@ def shared_cluster():
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].PORT.set_for_testing(cluster._jt_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
 

+ 1 - 1
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -225,7 +225,7 @@ class OozieApi(object):
 
   def submit_job(self, properties=None):
     """
-    submit_workflow(properties=None, id=None) -> jobid
+    submit_job(properties=None, id=None) -> jobid
 
     Submit a job to Oozie. May raise PopupException.
     """

+ 49 - 62
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -21,15 +21,14 @@ import time
 import subprocess
 import threading
 
-from nose.plugins.skip import SkipTest
-from nose.tools import assert_equal, assert_true
+from nose.tools import assert_equal
 
-from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.paths import get_run_root
 
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import OOZIE_URL
 from hadoop import pseudo_hdfs4
+import atexit
 
 
 _oozie_running = False
@@ -37,12 +36,11 @@ _oozie_lock = threading.Lock()
 
 LOG = logging.getLogger(__name__)
 
-# TODO HUE-752
+
 class OozieServerProvider(object):
   """
   Setup a Oozie server.
   """
-  STATES_WF_COMPLETION = ('SUCCEEDED' , 'KILLED', 'FAILED')
   OOZIE_TEST_PORT = '18080'
   OOZIE_HOME = get_run_root('ext/oozie/oozie')
 
@@ -50,37 +48,32 @@ class OozieServerProvider(object):
 
   @classmethod
   def setup_class(cls):
-    raise SkipTest
-
     cls.cluster = pseudo_hdfs4.shared_cluster()
-    cls.client = make_logged_in_client()
     cls.oozie, callback = cls._get_shared_oozie_server()
-    cls.shutdown = [ callback ]
+    cls.shutdown = [callback]
 
-  def wait_until_completion(self, jobid, timeout=60.0, step=1):
+  @classmethod
+  def wait_until_completion(cls, oozie_jobid, timeout=70.0, step=1):
     sleep = 0
-    workflow = None
+    job = cls.oozie.get_job(oozie_jobid)
     if step < 0:
       step = 1
     start = time.time()
 
-    while not self.is_job_completed(workflow) and time.time() - start < timeout:
+    while job.is_running() and time.time() - start < timeout:
       time.sleep(sleep)
       sleep = sleep + step
-      LOG.info('Checking status of %s...' % jobid)
-      workflow = self.oozie.get_job(jobid)
-    if not self.is_job_completed(workflow):
-      logs = self.oozie.get_job_log(jobid)
-      raise Exception("%s took too long to complete: %s" % (jobid, logs))
+      LOG.info('Checking status of %s...' % oozie_jobid)
+      job = cls.oozie.get_job(oozie_jobid)
+      LOG.info('Status: %s' % job)
 
-    return workflow
+    if job.is_running():
+      logs = cls.oozie.get_job_log(oozie_jobid)
+      raise Exception("%s took too long to complete: %s" % (oozie_jobid, logs))
+    else:
+      LOG.info('Job duration %s: %d' % (job.id, time.time() - start))
 
-  def is_job_completed(self, workflow):
-    # Only for Workflows so far
-    if workflow is None:
-      return False
-    LOG.info(workflow.status)
-    return workflow.status in OozieServerProvider.STATES_WF_COMPLETION
+    return job
 
   @classmethod
   def _start_oozie(cls, cluster):
@@ -98,6 +91,19 @@ class OozieServerProvider(object):
     process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
     return process
 
+  @classmethod
+  def _reset_oozie(cls):
+    env = os.environ
+
+    args = ['rm', '-r', OozieServerProvider.OOZIE_HOME + '/data/oozie-db']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+    args = [OozieServerProvider.OOZIE_HOME + '/bin/ooziedb.sh',  'create', '-sqlfile', 'oozie.sql', '-run']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+
   @classmethod
   def _get_shared_oozie_server(cls):
     global _oozie_running
@@ -105,16 +111,29 @@ class OozieServerProvider(object):
 
     _oozie_lock.acquire()
     if not _oozie_running:
+      LOG.info('\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')
+      LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
+
       finish = (
         OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
       )
 
       cluster = pseudo_hdfs4.shared_cluster()
+      cls._reset_oozie()
+      p = cls._start_oozie(cluster)
+
+      def kill():
+        LOG.info("Killing Oozie server (pid %d)." % p.pid)
+        os.kill(p.pid, 9)
+        p.wait()
+      atexit.register(kill)
 
       start = time.time()
       started = False
       sleep = 0.01
-      while not started and time.time() - start < 20.0:
+
+      while not started and time.time() - start < 30.0:
+        status = None
         try:
           LOG.info('Check Oozie status...')
           status = get_oozie().get_oozie_status()
@@ -123,8 +142,8 @@ class OozieServerProvider(object):
             break
           time.sleep(sleep)
           sleep *= 2
-        except:
-          LOG.info('Oozie server status not NORMAL yet.')
+        except Exception, e:
+          LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e))
           time.sleep(sleep)
           sleep *= 2
           pass
@@ -143,39 +162,7 @@ class OozieServerProvider(object):
     return get_oozie(), callback
 
 
-class TestoozieWithHadoop(OozieServerProvider):
+class TestMiniOozie(OozieServerProvider):
+
   def test_oozie_status(self):
-    assert_equal(self.oozie.get_oozie_status()['systemMode'], 'NORMAL')
-
-  def test_oozie_example(self):
-    jobid = None
-
-    try:
-      self.cluster.fs.setuser('hue')
-      self.cluster.fs.create_home_dir()
-      home = self.cluster.fs.get_home_dir()
-
-      self.cluster.put(OozieServerProvider + '/oozie/examples', home)
-      self.cluster.put(OozieServerProvider + '/oozie/examples/input-data/text/data.txt', home)
-      self.cluster.chmod(home, '0777')
-
-      application_path = self.cluster._fs_default_name + home + '/examples/apps/map-reduce'
-      assert_true(self.cluster.fs.exists(home + '/examples/apps/map-reduce'))
-
-      jobid = self.oozie.submit_workflow(application_path, {
-          'nameNode': self.cluster._fs_default_name,
-          'jobTracker': self.cluster.mapred_job_tracker,
-          'queueName': 'default',
-          'examplesRoot': 'examples',
-          'outputDir': 'test-out'})
-      assert_true(jobid)
-
-      self.oozie.job_control(jobid, 'start')
-      workflow = self.wait_until_completion(jobid)
-
-      assert_equal('SUCCEEDED', workflow.status)
-      assert_true(self.cluster.fs.exists('output-data'))
-    except:
-      if jobid is not None:
-        print self.oozie.get_job_log(jobid)
-      raise
+    assert_equal(get_oozie().get_oozie_status()['systemMode'], 'NORMAL')

+ 35 - 0
tools/jenkins/build-functions

@@ -94,3 +94,38 @@ build_hive() {
   tar -C $HIVE_DIR -xzf $HIVE_CACHE
   export HIVE_CONF_DIR=$HIVE_HOME/conf
 }
+
+##########
+OOZIE_URL=${OOZIE_URL:-http://nightly.cloudera.com/cdh4/cdh/4/oozie-3.2.0-cdh4.2.0-SNAPSHOT.tar.gz}
+
+OOZIE_TGZ=$(basename $OOZIE_URL)
+OOZIE_VERSION=${OOZIE_TGZ/.tar.gz/}
+OOZIE_CACHE="$HOME/.hue_cache/${OOZIE_TGZ}"
+
+build_oozie() {
+  if [ ! -f $OOZIE_CACHE ]; then
+    mkdir -p $HOME/.hue_cache
+    echo "Downloading $OOZIE_URL..."
+    wget $OOZIE_URL -O $OOZIE_CACHE
+  fi
+
+  OOZIE_DIR=$HUE_ROOT/ext/oozie
+  export OOZIE_HOME="$OOZIE_DIR/${OOZIE_VERSION}"
+
+  mkdir -p $OOZIE_DIR
+  rm -rf $OOZIE_HOME
+  echo "Unpacking $OOZIE_CACHE to $OOZIE_DIR"
+  tar -C $OOZIE_DIR -xzf $OOZIE_CACHE
+  export OOZIE_CONF_DIR=$OOZIE_HOME/conf
+
+  rm -rf $OOZIE_DIR/oozie
+  ln -s $OOZIE_DIR/${OOZIE_VERSION} $OOZIE_DIR/oozie # easier to reference later
+
+  mkdir -p $OOZIE_HOME/libext
+  tar -C $OOZIE_HOME/libext -zxvf $OOZIE_HOME/oozie-hadooplibs-*-cdh*-SNAPSHOT.tar.gz
+  cp $OOZIE_HOME/libext/oozie-*-SNAPSHOT/hadooplibs/hadooplib-*-mr1-cdh*-SNAPSHOT/*jar $OOZIE_HOME/libext/
+  tar -C $OOZIE_HOME -zxvf $OOZIE_HOME/oozie-examples.tar.gz
+
+  $OOZIE_HOME/bin/oozie-setup.sh
+  $OOZIE_HOME/bin/ooziedb.sh create -sqlfile oozie.sql -run
+}

+ 1 - 0
tools/jenkins/jenkins.sh

@@ -41,6 +41,7 @@ fi
 build_hadoop
 build_mr1
 build_hive
+build_oozie
 
 make apps
 

部分文件因为文件数量过多而无法显示