Explorar el Código

HUE-752 [jobsub] Add tests to Oozie API

Setup and start an Oozie instance during the tests
Requires "tools/jenkins/jenkins.sh" to be previously ran because of
https://issues.cloudera.org/browse/HUE-861
Add tests of the Oozie API
Add many tests to the Oozie App
Split Oozie App tests between requires Hadoop and not
Romain Rigaux hace 13 años
padre
commit
6df5059f74

+ 0 - 0
apps/oozie/examples/sleep/empty


+ 0 - 62
apps/oozie/examples/sleep/workflow.xml

@@ -1,62 +0,0 @@
-
-<workflow-app name="SleepWorkflow" xmlns="uri:oozie:workflow:0.2">
-    <start to="Sleep"/>
-    <action name="Sleep">
-        <map-reduce>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property>
-                    <name>mapred.reduce.tasks</name>
-                    <value>1</value>
-                </property>
-                <property>
-                    <name>mapred.mapper.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.reducer.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.key.class</name>
-                    <value>org.apache.hadoop.io.IntWritable</value>
-                </property>
-                <property>
-                    <name>mapred.mapoutput.value.class</name>
-                    <value>org.apache.hadoop.io.NullWritable</value>
-                </property>
-                <property>
-                    <name>mapred.output.format.class</name>
-                    <value>org.apache.hadoop.mapred.lib.NullOutputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.input.format.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
-                </property>
-                <property>
-                    <name>mapred.partitioner.class</name>
-                    <value>org.apache.hadoop.examples.SleepJob</value>
-                </property>
-                <property>
-                    <name>mapred.speculative.execution</name>
-                    <value>false</value>
-                </property>
-                <property>
-                    <name>sleep.job.map.sleep.time</name>
-                    <value>0</value>
-                </property>
-                <property>
-                    <name>sleep.job.reduce.sleep.time</name>
-                    <value>${REDUCER_SLEEP_TIME}</value>
-                </property>
-            </configuration>
-        </map-reduce>
-        <ok to="end"/>
-        <error to="kill"/>
-    </action>
-    <kill name="kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

+ 1 - 1
apps/oozie/src/oozie/fixtures/initial_example_data.json

@@ -28,7 +28,7 @@
     }
     }
   },
   },
   {
   {
-    "pk": 7,
+    "pk": null,
     "model": "oozie.job",
     "model": "oozie.job",
     "fields": {
     "fields": {
       "is_shared": true,
       "is_shared": true,

+ 8 - 8
apps/oozie/src/oozie/migrations/0007_auto__chg_field_sqoop_script_path.py

@@ -5,19 +5,19 @@ from south.v2 import SchemaMigration
 from django.db import models
 from django.db import models
 
 
 class Migration(SchemaMigration):
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
     def forwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.TextField')(blank=True))
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.TextField')(blank=True))
-    
-    
+
+
     def backwards(self, orm):
     def backwards(self, orm):
-        
+
         # Changing field 'Sqoop.script_path'
         # Changing field 'Sqoop.script_path'
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True))
         db.alter_column('oozie_sqoop', 'script_path', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True))
-    
-    
+
+
     models = {
     models = {
         'auth.group': {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
             'Meta': {'object_name': 'Group'},
@@ -251,5 +251,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
         }
     }
     }
-    
+
     complete_apps = ['oozie']
     complete_apps = ['oozie']

La diferencia del archivo ha sido suprimido porque es demasiado grande
+ 545 - 436
apps/oozie/src/oozie/tests.py


+ 22 - 3
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -17,6 +17,7 @@
 
 
 
 
 import atexit
 import atexit
+import getpass
 import logging
 import logging
 import pwd
 import pwd
 import os
 import os
@@ -95,6 +96,10 @@ class PseudoHdfs4(object):
   def mr1_env(self):
   def mr1_env(self):
     return self._mr1_env
     return self._mr1_env
 
 
+  @property
+  def log_dir(self):
+    return self._log_dir
+
   @property
   @property
   def fs_default_name(self):
   def fs_default_name(self):
     return self._fs_default_name
     return self._fs_default_name
@@ -237,6 +242,16 @@ class PseudoHdfs4(object):
     # Start MR1
     # Start MR1
     self._start_mr1(env)
     self._start_mr1(env)
 
 
+    # Make sure /tmp is 1777
+    self.fs.setuser(self.superuser)
+    if not self.fs.isdir('/tmp'):
+      self.fs.mkdir('/tmp', 01777)
+    else:
+      self.fs.chmod('/tmp', 01777)
+
+    self.fs.chmod(self._tmpdir, 01777)
+    self.fs.chmod(self._tmpdir + '/hadoop_tmp_dir/mapred', 01777)
+
 
 
   def _start_mr1(self, env):
   def _start_mr1(self, env):
     LOG.info("Starting MR1")
     LOG.info("Starting MR1")
@@ -291,7 +306,6 @@ class PseudoHdfs4(object):
       stdout.close()
       stdout.close()
       stderr.close()
       stderr.close()
 
 
-
   def _log_exit(self, proc_name, exit_code):
   def _log_exit(self, proc_name, exit_code):
     """Log the stdout and stderr for a process"""
     """Log the stdout and stderr for a process"""
     LOG.info('%s exited with %s' % (proc_name, exit_code))
     LOG.info('%s exited with %s' % (proc_name, exit_code))
@@ -392,8 +406,6 @@ class PseudoHdfs4(object):
       'dfs.datanode.ipc.address': 'localhost:0',
       'dfs.datanode.ipc.address': 'localhost:0',
       'dfs.replication': 1,
       'dfs.replication': 1,
       'dfs.safemode.min.datanodes': 1,
       'dfs.safemode.min.datanodes': 1,
-      'hadoop.proxyuser.hue.hosts': '*',
-      'hadoop.proxyuser.hue.groups': '*',
     }
     }
     write_config(hdfs_configs, self._tmppath('conf/hdfs-site.xml'))
     write_config(hdfs_configs, self._tmppath('conf/hdfs-site.xml'))
 
 
@@ -410,6 +422,12 @@ class PseudoHdfs4(object):
       'hadoop.security.authentication': 'simple',
       'hadoop.security.authentication': 'simple',
       'hadoop.proxyuser.%s.groups' % (self.superuser,): 'users,supergroup',
       'hadoop.proxyuser.%s.groups' % (self.superuser,): 'users,supergroup',
       'hadoop.proxyuser.%s.hosts' % (self.superuser,): 'localhost',
       'hadoop.proxyuser.%s.hosts' % (self.superuser,): 'localhost',
+      'hadoop.proxyuser.hue.hosts': '*',
+      'hadoop.proxyuser.hue.groups': '*',
+      'hadoop.proxyuser.oozie.hosts': '*',
+      'hadoop.proxyuser.oozie.groups': '*',
+      'hadoop.proxyuser.%s.hosts' % getpass.getuser(): '*',
+      'hadoop.proxyuser.%s.groups' % getpass.getuser(): '*',
       'hadoop.tmp.dir': self._tmppath('hadoop_tmp_dir'),
       'hadoop.tmp.dir': self._tmppath('hadoop_tmp_dir'),
     }
     }
     write_config(core_configs, self._tmppath('conf/core-site.xml'))
     write_config(core_configs, self._tmppath('conf/core-site.xml'))
@@ -467,6 +485,7 @@ def shared_cluster():
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
       hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].PORT.set_for_testing(cluster._jt_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
     ]
 
 

+ 1 - 1
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -225,7 +225,7 @@ class OozieApi(object):
 
 
   def submit_job(self, properties=None):
   def submit_job(self, properties=None):
     """
     """
-    submit_workflow(properties=None, id=None) -> jobid
+    submit_job(properties=None, id=None) -> jobid
 
 
     Submit a job to Oozie. May raise PopupException.
     Submit a job to Oozie. May raise PopupException.
     """
     """

+ 49 - 62
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -21,15 +21,14 @@ import time
 import subprocess
 import subprocess
 import threading
 import threading
 
 
-from nose.plugins.skip import SkipTest
-from nose.tools import assert_equal, assert_true
+from nose.tools import assert_equal
 
 
-from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.paths import get_run_root
 from desktop.lib.paths import get_run_root
 
 
 from liboozie.oozie_api import get_oozie
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import OOZIE_URL
 from liboozie.conf import OOZIE_URL
 from hadoop import pseudo_hdfs4
 from hadoop import pseudo_hdfs4
+import atexit
 
 
 
 
 _oozie_running = False
 _oozie_running = False
@@ -37,12 +36,11 @@ _oozie_lock = threading.Lock()
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
-# TODO HUE-752
+
 class OozieServerProvider(object):
 class OozieServerProvider(object):
   """
   """
   Setup a Oozie server.
   Setup a Oozie server.
   """
   """
-  STATES_WF_COMPLETION = ('SUCCEEDED' , 'KILLED', 'FAILED')
   OOZIE_TEST_PORT = '18080'
   OOZIE_TEST_PORT = '18080'
   OOZIE_HOME = get_run_root('ext/oozie/oozie')
   OOZIE_HOME = get_run_root('ext/oozie/oozie')
 
 
@@ -50,37 +48,32 @@ class OozieServerProvider(object):
 
 
   @classmethod
   @classmethod
   def setup_class(cls):
   def setup_class(cls):
-    raise SkipTest
-
     cls.cluster = pseudo_hdfs4.shared_cluster()
     cls.cluster = pseudo_hdfs4.shared_cluster()
-    cls.client = make_logged_in_client()
     cls.oozie, callback = cls._get_shared_oozie_server()
     cls.oozie, callback = cls._get_shared_oozie_server()
-    cls.shutdown = [ callback ]
+    cls.shutdown = [callback]
 
 
-  def wait_until_completion(self, jobid, timeout=60.0, step=1):
+  @classmethod
+  def wait_until_completion(cls, oozie_jobid, timeout=70.0, step=1):
     sleep = 0
     sleep = 0
-    workflow = None
+    job = cls.oozie.get_job(oozie_jobid)
     if step < 0:
     if step < 0:
       step = 1
       step = 1
     start = time.time()
     start = time.time()
 
 
-    while not self.is_job_completed(workflow) and time.time() - start < timeout:
+    while job.is_running() and time.time() - start < timeout:
       time.sleep(sleep)
       time.sleep(sleep)
       sleep = sleep + step
       sleep = sleep + step
-      LOG.info('Checking status of %s...' % jobid)
-      workflow = self.oozie.get_job(jobid)
-    if not self.is_job_completed(workflow):
-      logs = self.oozie.get_job_log(jobid)
-      raise Exception("%s took too long to complete: %s" % (jobid, logs))
+      LOG.info('Checking status of %s...' % oozie_jobid)
+      job = cls.oozie.get_job(oozie_jobid)
+      LOG.info('Status: %s' % job)
 
 
-    return workflow
+    if job.is_running():
+      logs = cls.oozie.get_job_log(oozie_jobid)
+      raise Exception("%s took too long to complete: %s" % (oozie_jobid, logs))
+    else:
+      LOG.info('Job duration %s: %d' % (job.id, time.time() - start))
 
 
-  def is_job_completed(self, workflow):
-    # Only for Workflows so far
-    if workflow is None:
-      return False
-    LOG.info(workflow.status)
-    return workflow.status in OozieServerProvider.STATES_WF_COMPLETION
+    return job
 
 
   @classmethod
   @classmethod
   def _start_oozie(cls, cluster):
   def _start_oozie(cls, cluster):
@@ -98,6 +91,19 @@ class OozieServerProvider(object):
     process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
     process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
     return process
     return process
 
 
+  @classmethod
+  def _reset_oozie(cls):
+    env = os.environ
+
+    args = ['rm', '-r', OozieServerProvider.OOZIE_HOME + '/data/oozie-db']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+    args = [OozieServerProvider.OOZIE_HOME + '/bin/ooziedb.sh',  'create', '-sqlfile', 'oozie.sql', '-run']
+    LOG.info("Executing %s, env %s" % (args, env))
+    subprocess.call(args, env=env)
+
+
   @classmethod
   @classmethod
   def _get_shared_oozie_server(cls):
   def _get_shared_oozie_server(cls):
     global _oozie_running
     global _oozie_running
@@ -105,16 +111,29 @@ class OozieServerProvider(object):
 
 
     _oozie_lock.acquire()
     _oozie_lock.acquire()
     if not _oozie_running:
     if not _oozie_running:
+      LOG.info('\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')
+      LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
+
       finish = (
       finish = (
         OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
         OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
       )
       )
 
 
       cluster = pseudo_hdfs4.shared_cluster()
       cluster = pseudo_hdfs4.shared_cluster()
+      cls._reset_oozie()
+      p = cls._start_oozie(cluster)
+
+      def kill():
+        LOG.info("Killing Oozie server (pid %d)." % p.pid)
+        os.kill(p.pid, 9)
+        p.wait()
+      atexit.register(kill)
 
 
       start = time.time()
       start = time.time()
       started = False
       started = False
       sleep = 0.01
       sleep = 0.01
-      while not started and time.time() - start < 20.0:
+
+      while not started and time.time() - start < 30.0:
+        status = None
         try:
         try:
           LOG.info('Check Oozie status...')
           LOG.info('Check Oozie status...')
           status = get_oozie().get_oozie_status()
           status = get_oozie().get_oozie_status()
@@ -123,8 +142,8 @@ class OozieServerProvider(object):
             break
             break
           time.sleep(sleep)
           time.sleep(sleep)
           sleep *= 2
           sleep *= 2
-        except:
-          LOG.info('Oozie server status not NORMAL yet.')
+        except Exception, e:
+          LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e))
           time.sleep(sleep)
           time.sleep(sleep)
           sleep *= 2
           sleep *= 2
           pass
           pass
@@ -143,39 +162,7 @@ class OozieServerProvider(object):
     return get_oozie(), callback
     return get_oozie(), callback
 
 
 
 
-class TestoozieWithHadoop(OozieServerProvider):
+class TestMiniOozie(OozieServerProvider):
+
   def test_oozie_status(self):
   def test_oozie_status(self):
-    assert_equal(self.oozie.get_oozie_status()['systemMode'], 'NORMAL')
-
-  def test_oozie_example(self):
-    jobid = None
-
-    try:
-      self.cluster.fs.setuser('hue')
-      self.cluster.fs.create_home_dir()
-      home = self.cluster.fs.get_home_dir()
-
-      self.cluster.put(OozieServerProvider + '/oozie/examples', home)
-      self.cluster.put(OozieServerProvider + '/oozie/examples/input-data/text/data.txt', home)
-      self.cluster.chmod(home, '0777')
-
-      application_path = self.cluster._fs_default_name + home + '/examples/apps/map-reduce'
-      assert_true(self.cluster.fs.exists(home + '/examples/apps/map-reduce'))
-
-      jobid = self.oozie.submit_workflow(application_path, {
-          'nameNode': self.cluster._fs_default_name,
-          'jobTracker': self.cluster.mapred_job_tracker,
-          'queueName': 'default',
-          'examplesRoot': 'examples',
-          'outputDir': 'test-out'})
-      assert_true(jobid)
-
-      self.oozie.job_control(jobid, 'start')
-      workflow = self.wait_until_completion(jobid)
-
-      assert_equal('SUCCEEDED', workflow.status)
-      assert_true(self.cluster.fs.exists('output-data'))
-    except:
-      if jobid is not None:
-        print self.oozie.get_job_log(jobid)
-      raise
+    assert_equal(get_oozie().get_oozie_status()['systemMode'], 'NORMAL')

+ 35 - 0
tools/jenkins/build-functions

@@ -94,3 +94,38 @@ build_hive() {
   tar -C $HIVE_DIR -xzf $HIVE_CACHE
   tar -C $HIVE_DIR -xzf $HIVE_CACHE
   export HIVE_CONF_DIR=$HIVE_HOME/conf
   export HIVE_CONF_DIR=$HIVE_HOME/conf
 }
 }
+
+##########
+OOZIE_URL=${OOZIE_URL:-http://nightly.cloudera.com/cdh4/cdh/4/oozie-3.2.0-cdh4.2.0-SNAPSHOT.tar.gz}
+
+OOZIE_TGZ=$(basename $OOZIE_URL)
+OOZIE_VERSION=${OOZIE_TGZ/.tar.gz/}
+OOZIE_CACHE="$HOME/.hue_cache/${OOZIE_TGZ}"
+
+build_oozie() {
+  if [ ! -f $OOZIE_CACHE ]; then
+    mkdir -p $HOME/.hue_cache
+    echo "Downloading $OOZIE_URL..."
+    wget $OOZIE_URL -O $OOZIE_CACHE
+  fi
+
+  OOZIE_DIR=$HUE_ROOT/ext/oozie
+  export OOZIE_HOME="$OOZIE_DIR/${OOZIE_VERSION}"
+
+  mkdir -p $OOZIE_DIR
+  rm -rf $OOZIE_HOME
+  echo "Unpacking $OOZIE_CACHE to $OOZIE_DIR"
+  tar -C $OOZIE_DIR -xzf $OOZIE_CACHE
+  export OOZIE_CONF_DIR=$OOZIE_HOME/conf
+
+  rm -rf $OOZIE_DIR/oozie
+  ln -s $OOZIE_DIR/${OOZIE_VERSION} $OOZIE_DIR/oozie # easier to reference later
+
+  mkdir -p $OOZIE_HOME/libext
+  tar -C $OOZIE_HOME/libext -zxvf $OOZIE_HOME/oozie-hadooplibs-*-cdh*-SNAPSHOT.tar.gz
+  cp $OOZIE_HOME/libext/oozie-*-SNAPSHOT/hadooplibs/hadooplib-*-mr1-cdh*-SNAPSHOT/*jar $OOZIE_HOME/libext/
+  tar -C $OOZIE_HOME -zxvf $OOZIE_HOME/oozie-examples.tar.gz
+
+  $OOZIE_HOME/bin/oozie-setup.sh
+  $OOZIE_HOME/bin/ooziedb.sh create -sqlfile oozie.sql -run
+}

+ 1 - 0
tools/jenkins/jenkins.sh

@@ -41,6 +41,7 @@ fi
 build_hadoop
 build_hadoop
 build_mr1
 build_mr1
 build_hive
 build_hive
+build_oozie
 
 
 make apps
 make apps
 
 

Algunos archivos no se mostraron porque demasiados archivos cambiaron en este cambio