Эх сурвалжийг харах

HUE-752 [jobsub] Add tests to Oozie API

Setup and start an Oozie instance during the tests
Requires "tools/jenkins/jenkins.sh" to be previously ran because of
https://issues.cloudera.org/browse/HUE-861
Add tests of the Oozie API
Add many tests to the Oozie App
Split Oozie App tests between requires Hadoop and not
Romain Rigaux 13 жил өмнө
parent
commit
6df5059f74

+ 0 - 0
apps/oozie/examples/sleep/empty


+ 0 - 62
apps/oozie/examples/sleep/workflow.xml

@@ -1,62 +0,0 @@
-
-<workflow-app name="SleepWorkflow" xmlns="uri:oozie:workflow:0.2">
-    <start to="Sleep"/>
-    <action name="Sleep">
-        <map-reduce>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property>
-                    <name>mapred.reduce.tasks</name>
-                    <value>1</value>
-                </property>
-                <property>
-                    <name>mapred.mapper.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.reducer.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.key.class</name>
-                    <value>org.apache.hadoop.io.IntWritable</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.value.class</name>
-                    <value>org.apache.hadoop.io.NullWritable</value>
-                </property>
-                <property>
-                    <name>mapred.output.format.class</name>
-                    <value>org.apache.hadoop.mapred.lib.NullOutputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.input.format.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.partitioner.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.speculative.execution</name>
-                    <value>false</value>
-                </property>
-                <property>
-                    <name>sleep.job.map.sleep.time</name>
-                    <value>0</value>
-                </property>
-                <property>
-                    <name>sleep.job.reduce.sleep.time</name>
-                    <value>${REDUCER_SLEEP_TIME}</value>
-                </property>
-            </configuration>
-        </map-reduce>
-        <ok to="end"/>
-        <error to="kill"/>
-    </action>
-    <kill name="kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

+ 1 - 1
apps/oozie/src/oozie/fixtures/initial_example_data.json

@@ -28,7 +28,7 @@
     }
   },
   {
-    "pk": 7,
+    "pk": null,
     "model": "oozie.job",
     "fields": {
       "is_shared": true,

+ 8 - 8
apps/oozie/src/oozie/migrations/0007_auto__chg_field_sqoop_script_path.py

@@ -5,19 +5,19 @@ from south.v2 import SchemaMigration
 from django.db import models
 
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.TextField')(blank=True))
-    
-    
+
+
     def backwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True))
-    
-    
+
+
     models = {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
@@ -251,5 +251,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
     }
-    
+
     complete_apps = ['oozie']

Файлын зөрүү хэтэрхий том тул дарагдсан байна
+ 545 - 436
apps/oozie/src/oozie/tests.py


+ 22 - 3
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -17,6 +17,7 @@
 
 
 import atexit
+import getpass
 import logging
 import pwd
 import os
@@ -95,6 +96,10 @@ class PseudoHdfs4(object):
   def mr1_env(self):
     return self._mr1_env
 
+  @property
+  def log_dir(self):
+    return self._log_dir
+
   @property
   def fs_default_name(self):
     return self._fs_default_name
@@ -237,6 +242,16 @@ class PseudoHdfs4(object):
     # Start MR1
     self._start_mr1(env)
 
+    # Make sure /tmp is 1777
+    self.fs.setuser(self.superuser)
+    if not self.fs.isdir('/tmp'):
+      self.fs.mkdir('/tmp', 01777)
+    else:
+      self.fs.chmod('/tmp', 01777)
+
+    self.fs.chmod(self._tmpdir, 01777)
+    self.fs.chmod(self._tmpdir + '/hadoop_tmp_dir/mapred', 01777)
+
 
   def _start_mr1(self, env):
     LOG.info("Starting MR1")
@@ -291,7 +306,6 @@ class PseudoHdfs4(object):
       stdout.close()
       stderr.close()
 
-
   def _log_exit(self, proc_name, exit_code):
     """Log the stdout and stderr for a process"""
     LOG.info('%s exited with %s' % (proc_name, exit_code))
@@ -392,8 +406,6 @@ class PseudoHdfs4(object):
       'dfs.datanode.ipc.address': 'localhost:0',
       'dfs.replication': 1,
       'dfs.safemode.min.datanodes': 1,
-      'hadoop.proxyuser.hue.hosts': '*',
-      'hadoop.proxyuser.hue.groups': '*',
     }
     write_config(hdfs_configs, self._tmppath('conf/hdfs-site.xml'))
 
@@ -410,6 +422,12 @@ class PseudoHdfs4(object):
       'hadoop.security.authentication': 'simple',
       'hadoop.proxyuser.%s.groups' % (self.superuser,): 'users,supergroup',
       'hadoop.proxyuser.%s.hosts' % (self.superuser,): 'localhost',
+      'hadoop.proxyuser.hue.hosts': '*',
+      'hadoop.proxyuser.hue.groups': '*',
+      'hadoop.proxyuser.oozie.hosts': '*',
+      'hadoop.proxyuser.oozie.groups': '*',
+      'hadoop.proxyuser.%s.hosts' % getpass.getuser(): '*',
+      'hadoop.proxyuser.%s.groups' % getpass.getuser(): '*',
       'hadoop.tmp.dir': self._tmppath('hadoop_tmp_dir'),
     }
     write_config(core_configs, self._tmppath('conf/core-site.xml'))
@@ -467,6 +485,7 @@ def shared_cluster():
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].PORT.set_for_testing(cluster._jt_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
 

+ 1 - 1
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -225,7 +225,7 @@ class OozieApi(object):
 
   def submit_job(self, properties=None):
     """
-    submit_workflow(properties=None, id=None) -> jobid
+    submit_job(properties=None, id=None) -> jobid
 
     Submit a job to Oozie. May raise PopupException.
     """

+ 49 - 62
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -21,15 +21,14 @@ import time
 import subprocess
 import threading
 
-from nose.plugins.skip import SkipTest
-from nose.tools import assert_equal, assert_true
+from nose.tools import assert_equal
 
-from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.paths import get_run_root
 
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import OOZIE_URL
 from hadoop import pseudo_hdfs4
+import atexit
 
 
 _oozie_running = False
@@ -37,12 +36,11 @@ _oozie_lock = threading.Lock()
 
 LOG = logging.getLogger(__name__)
 
-# TODO HUE-752
+
 class OozieServerProvider(object):
   """
   Setup a Oozie server.
   """
-  STATES_WF_COMPLETION = ('SUCCEEDED' , 'KILLED', 'FAILED')
   OOZIE_TEST_PORT = '18080'
   OOZIE_HOME = get_run_root('ext/oozie/oozie')
 
@@ -50,37 +48,32 @@ class OozieServerProvider(object):
 
   @classmethod
   def setup_class(cls):
-    raise SkipTest
-
     cls.cluster = pseudo_hdfs4.shared_cluster()
-    cls.client = make_logged_in_client()
     cls.oozie, callback = cls._get_shared_oozie_server()
-    cls.shutdown = [ callback ]
+    cls.shutdown = [callback]
 
-  def wait_until_completion(self, jobid, timeout=60.0, step=1):
+  @classmethod
+  def wait_until_completion(cls, oozie_jobid, timeout=70.0, step=1):
     sleep = 0
-    workflow = None
+    job = cls.oozie.get_job(oozie_jobid)
     if step < 0:
       step = 1
     start = time.time()
 
-    while not self.is_job_completed(workflow) and time.time() - start < timeout:
+    while job.is_running() and time.time() - start < timeout:
       time.sleep(sleep)
       sleep = sleep + step
-      LOG.info('Checking status of %s...' % jobid)
-      workflow = self.oozie.get_job(jobid)
-    if not self.is_job_completed(workflow):
-      logs = self.oozie.get_job_log(jobid)
-      raise Exception("%s took too long to complete: %s" % (jobid, logs))
+      LOG.info('Checking status of %s...' % oozie_jobid)
+      job = cls.oozie.get_job(oozie_jobid)
+      LOG.info('Status: %s' % job)
 
-    return workflow
+    if job.is_running():
+      logs = cls.oozie.get_job_log(oozie_jobid)
+      raise Exception("%s took too long to complete: %s" % (oozie_jobid, logs))
+    else:
+      LOG.info('Job duration %s: %d' % (job.id, time.time() - start))
 
-  def is_job_completed(self, workflow):
-    # Only for Workflows so far
-    if workflow is None:
-      return False
-    LOG.info(workflow.status)
-    return workflow.status in OozieServerProvider.STATES_WF_COMPLETION
+    return job
 
   @classmethod
   def _start_oozie(cls, cluster):
@@ -98,6 +91,19 @@ class OozieServerProvider(object):
     process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
     return process
 
+  @classmethod
+  def _reset_oozie(cls):
+    env = os.environ
+
+    args = ['rm', '-r', OozieServerProvider.OOZIE_HOME + '/data/oozie-db']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+    args = [OozieServerProvider.OOZIE_HOME + '/bin/ooziedb.sh',  'create', '-sqlfile', 'oozie.sql', '-run']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+
   @classmethod
   def _get_shared_oozie_server(cls):
     global _oozie_running
@@ -105,16 +111,29 @@ class OozieServerProvider(object):
 
     _oozie_lock.acquire()
     if not _oozie_running:
+      LOG.info('\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')
+      LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
+
       finish = (
         OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
       )
 
       cluster = pseudo_hdfs4.shared_cluster()
+      cls._reset_oozie()
+      p = cls._start_oozie(cluster)
+
+      def kill():
+        LOG.info("Killing Oozie server (pid %d)." % p.pid)
+        os.kill(p.pid, 9)
+        p.wait()
+      atexit.register(kill)
 
       start = time.time()
       started = False
       sleep = 0.01
-      while not started and time.time() - start < 20.0:
+
+      while not started and time.time() - start < 30.0:
+        status = None
         try:
           LOG.info('Check Oozie status...')
           status = get_oozie().get_oozie_status()
@@ -123,8 +142,8 @@ class OozieServerProvider(object):
             break
           time.sleep(sleep)
           sleep *= 2
-        except:
-          LOG.info('Oozie server status not NORMAL yet.')
+        except Exception, e:
+          LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e))
           time.sleep(sleep)
           sleep *= 2
           pass
@@ -143,39 +162,7 @@ class OozieServerProvider(object):
     return get_oozie(), callback
 
 
-class TestoozieWithHadoop(OozieServerProvider):
+class TestMiniOozie(OozieServerProvider):
+
   def test_oozie_status(self):
-    assert_equal(self.oozie.get_oozie_status()['systemMode'], 'NORMAL')
-
-  def test_oozie_example(self):
-    jobid = None
-
-    try:
-      self.cluster.fs.setuser('hue')
-      self.cluster.fs.create_home_dir()
-      home = self.cluster.fs.get_home_dir()
-
-      self.cluster.put(OozieServerProvider + '/oozie/examples', home)
-      self.cluster.put(OozieServerProvider + '/oozie/examples/input-data/text/data.txt', home)
-      self.cluster.chmod(home, '0777')
-
-      application_path = self.cluster._fs_default_name + home + '/examples/apps/map-reduce'
-      assert_true(self.cluster.fs.exists(home + '/examples/apps/map-reduce'))
-
-      jobid = self.oozie.submit_workflow(application_path, {
-          'nameNode': self.cluster._fs_default_name,
-          'jobTracker': self.cluster.mapred_job_tracker,
-          'queueName': 'default',
-          'examplesRoot': 'examples',
-          'outputDir': 'test-out'})
-      assert_true(jobid)
-
-      self.oozie.job_control(jobid, 'start')
-      workflow = self.wait_until_completion(jobid)
-
-      assert_equal('SUCCEEDED', workflow.status)
-      assert_true(self.cluster.fs.exists('output-data'))
-    except:
-      if jobid is not None:
-        print self.oozie.get_job_log(jobid)
-      raise
+    assert_equal(get_oozie().get_oozie_status()['systemMode'], 'NORMAL')

+ 35 - 0
tools/jenkins/build-functions

@@ -94,3 +94,38 @@ build_hive() {
   tar -C $HIVE_DIR -xzf $HIVE_CACHE
   export HIVE_CONF_DIR=$HIVE_HOME/conf
 }
+
+##########
+OOZIE_URL=${OOZIE_URL:-http://nightly.cloudera.com/cdh4/cdh/4/oozie-3.2.0-cdh4.2.0-SNAPSHOT.tar.gz}
+
+OOZIE_TGZ=$(basename $OOZIE_URL)
+OOZIE_VERSION=${OOZIE_TGZ/.tar.gz/}
+OOZIE_CACHE="$HOME/.hue_cache/${OOZIE_TGZ}"
+
+build_oozie() {
+  if [ ! -f $OOZIE_CACHE ]; then
+    mkdir -p $HOME/.hue_cache
+    echo "Downloading $OOZIE_URL..."
+    wget $OOZIE_URL -O $OOZIE_CACHE
+  fi
+
+  OOZIE_DIR=$HUE_ROOT/ext/oozie
+  export OOZIE_HOME="$OOZIE_DIR/${OOZIE_VERSION}"
+
+  mkdir -p $OOZIE_DIR
+  rm -rf $OOZIE_HOME
+  echo "Unpacking $OOZIE_CACHE to $OOZIE_DIR"
+  tar -C $OOZIE_DIR -xzf $OOZIE_CACHE
+  export OOZIE_CONF_DIR=$OOZIE_HOME/conf
+
+  rm -rf $OOZIE_DIR/oozie
+  ln -s $OOZIE_DIR/${OOZIE_VERSION} $OOZIE_DIR/oozie # easier to reference later
+
+  mkdir -p $OOZIE_HOME/libext
+  tar -C $OOZIE_HOME/libext -zxvf $OOZIE_HOME/oozie-hadooplibs-*-cdh*-SNAPSHOT.tar.gz
+  cp $OOZIE_HOME/libext/oozie-*-SNAPSHOT/hadooplibs/hadooplib-*-mr1-cdh*-SNAPSHOT/*jar $OOZIE_HOME/libext/
+  tar -C $OOZIE_HOME -zxvf $OOZIE_HOME/oozie-examples.tar.gz
+
+  $OOZIE_HOME/bin/oozie-setup.sh
+  $OOZIE_HOME/bin/ooziedb.sh create -sqlfile oozie.sql -run
+}

+ 1 - 0
tools/jenkins/jenkins.sh

@@ -41,6 +41,7 @@ fi
 build_hadoop
 build_mr1
 build_hive
+build_oozie
 
 make apps
 

Энэ ялгаанд хэт олон файл өөрчлөгдсөн тул зарим файлыг харуулаагүй болно