瀏覽代碼

HUE-958 [jobbrowser,jobsub,ooxie] Fix flakey tests

- Clean jobsub examples database per test
- Use FQDN when starting hadoop daemons
abec 13 年之前
父節點
當前提交
1ac2119

+ 6 - 1
apps/jobbrowser/src/jobbrowser/tests.py

@@ -27,7 +27,7 @@ from nose.tools import assert_true, assert_false, assert_equal
 
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
 from desktop.lib.test_utils import grant_access
-from jobsub.models import OozieDesign
+from jobsub.models import OozieDesign, CheckForSetup
 from liboozie.oozie_api_test import OozieServerProvider
 from liboozie.oozie_api_test import OozieServerProvider
 
 
 from jobbrowser import models, views
 from jobbrowser import models, views
@@ -88,6 +88,11 @@ class TestJobBrowserWithHadoop(unittest.TestCase, OozieServerProvider):
 
 
     cls.sleep_design_id = OozieDesign.objects.get(name='sleep_job').id
     cls.sleep_design_id = OozieDesign.objects.get(name='sleep_job').id
 
 
+  @classmethod
+  def teardown_class(cls):
+    OozieDesign.objects.all().delete()
+    CheckForSetup.objects.all().delete()
+
   def setUp(self):
   def setUp(self):
     TestJobBrowserWithHadoop.user_count += 1
     TestJobBrowserWithHadoop.user_count += 1
     self.username = 'test' + str(TestJobBrowserWithHadoop.user_count)
     self.username = 'test' + str(TestJobBrowserWithHadoop.user_count)

+ 5 - 1
apps/jobsub/src/jobsub/tests.py

@@ -27,7 +27,7 @@ from liboozie.oozie_api_test import OozieServerProvider
 
 
 from jobsub import conf
 from jobsub import conf
 from jobsub.management.commands import jobsub_setup
 from jobsub.management.commands import jobsub_setup
-from jobsub.models import OozieDesign, OozieMapreduceAction, OozieStreamingAction
+from jobsub.models import OozieDesign, OozieMapreduceAction, OozieStreamingAction, CheckForSetup
 from jobsub.parameterization import recursive_walk, find_variables, substitute_variables
 from jobsub.parameterization import recursive_walk, find_variables, substitute_variables
 
 
 
 
@@ -191,6 +191,10 @@ class TestJobsubWithHadoop(OozieServerProvider):
     # Ensure access to MR folder
     # Ensure access to MR folder
     self.cluster.fs.do_as_superuser(self.cluster.fs.chmod, '/tmp', 0777, recursive=True)
     self.cluster.fs.do_as_superuser(self.cluster.fs.chmod, '/tmp', 0777, recursive=True)
 
 
+  def tearDown(self):
+    OozieDesign.objects.all().delete()
+    CheckForSetup.objects.all().delete()
+
   def test_jobsub_setup(self):
   def test_jobsub_setup(self):
     # User 'test' triggers the setup of the examples.
     # User 'test' triggers the setup of the examples.
     # 'hue' home will be deleted, the examples installed in the new one
     # 'hue' home will be deleted, the examples installed in the new one

+ 22 - 16
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -23,6 +23,7 @@ import os
 import shutil
 import shutil
 import signal
 import signal
 import subprocess
 import subprocess
+import socket
 import tempfile
 import tempfile
 import textwrap
 import textwrap
 import time
 import time
@@ -79,13 +80,17 @@ class PseudoHdfs4(object):
     self._dn_proc = None
     self._dn_proc = None
     self._jt_proc = None
     self._jt_proc = None
     self._tt_proc = None
     self._tt_proc = None
+    self._fqdn = socket.getfqdn()
 
 
     self.shutdown_hook = None
     self.shutdown_hook = None
 
 
   def __str__(self):
   def __str__(self):
-    return "PseudoHdfs4 (%s) at %s --- MR1 (%s) at http://localhost:%s" % \
-        (self._fs_default_name, self._tmpdir,
-         self.mapred_job_tracker, self._jt_http_port)
+    return "PseudoHdfs4 (%(name)s) at %(dir)s --- MR1 (%(mapreduce)s) at http://%(fqdn)s:%(port)s" % {
+      'name': self._fs_default_name,
+      'dir': self._tmpdir,
+      'mapreduce': self.mapred_job_tracker,
+      'fqdn': self._fqdn,
+      'port': self._jt_http_port}
 
 
   @property
   @property
   def superuser(self):
   def superuser(self):
@@ -121,11 +126,11 @@ class PseudoHdfs4(object):
 
 
   @property
   @property
   def mapred_job_tracker(self):
   def mapred_job_tracker(self):
-    return "localhost:%s" % (self._jt_port,)
+    return "%s:%s" % (self._fqdn, self._jt_port,)
 
 
   @property
   @property
   def mapred_job_tracker_http_address(self):
   def mapred_job_tracker_http_address(self):
-    return "localhost:%s" % (self._jt_http_port,)
+    return "%s:%s" % (self._fqdn, self._jt_http_port,)
 
 
   @property
   @property
   def fs(self):
   def fs(self):
@@ -143,7 +148,7 @@ class PseudoHdfs4(object):
   def jt(self):
   def jt(self):
     """Returns a LiveJobTracker object configured for this cluster."""
     """Returns a LiveJobTracker object configured for this cluster."""
     if self._jt is None:
     if self._jt is None:
-      self._jt = LiveJobTracker("localhost", self.jt_thrift_port)
+      self._jt = LiveJobTracker(self._fqdn, self.jt_thrift_port)
     return self._jt
     return self._jt
 
 
   def stop(self):
   def stop(self):
@@ -392,17 +397,17 @@ class PseudoHdfs4(object):
 
 
   def _write_hdfs_site(self):
   def _write_hdfs_site(self):
     self._dfs_http_port = find_unused_port()
     self._dfs_http_port = find_unused_port()
-    self._dfs_http_address = 'localhost:%s' % (self._dfs_http_port,)
+    self._dfs_http_address = '%s:%s' % (self._fqdn, self._dfs_http_port)
 
 
     hdfs_configs = {
     hdfs_configs = {
       'dfs.webhdfs.enabled': 'true',
       'dfs.webhdfs.enabled': 'true',
       'dfs.http.address': self._dfs_http_address,
       'dfs.http.address': self._dfs_http_address,
       'dfs.namenode.safemode.extension': 1,
       'dfs.namenode.safemode.extension': 1,
       'dfs.namenode.safemode.threshold-pct': 0,
       'dfs.namenode.safemode.threshold-pct': 0,
-      'dfs.datanode.address': 'localhost:0',
+      'dfs.datanode.address': '%s:0' % self._fqdn,
       # Work around webhdfs redirect bug -- bind to all interfaces
       # Work around webhdfs redirect bug -- bind to all interfaces
       'dfs.datanode.http.address': '0.0.0.0:0',
       'dfs.datanode.http.address': '0.0.0.0:0',
-      'dfs.datanode.ipc.address': 'localhost:0',
+      'dfs.datanode.ipc.address': '%s:0' % self._fqdn,
       'dfs.replication': 1,
       'dfs.replication': 1,
       'dfs.safemode.min.datanodes': 1,
       'dfs.safemode.min.datanodes': 1,
     }
     }
@@ -413,7 +418,7 @@ class PseudoHdfs4(object):
     ugm_properties = self._tmppath('ugm.properties')
     ugm_properties = self._tmppath('ugm.properties')
     self._write_static_group_mapping(ugm_properties)
     self._write_static_group_mapping(ugm_properties)
     self._namenode_port = find_unused_port()
     self._namenode_port = find_unused_port()
-    self._fs_default_name = 'hdfs://localhost:%s' % (self._namenode_port,)
+    self._fs_default_name = 'hdfs://%s:%s' % (self._fqdn, self._namenode_port,)
 
 
     core_configs = {
     core_configs = {
       'fs.default.name': self._fs_default_name,
       'fs.default.name': self._fs_default_name,
@@ -434,11 +439,11 @@ class PseudoHdfs4(object):
     self._tt_http_port = find_unused_port()
     self._tt_http_port = find_unused_port()
 
 
     mapred_configs = {
     mapred_configs = {
-      'mapred.job.tracker': 'localhost:%s' % (self._jt_port,),
-      'mapred.job.tracker.http.address': 'localhost:%s' % (self._jt_http_port,),
-      'jobtracker.thrift.address': 'localhost:%s' % (self._jt_thrift_port,),
+      'mapred.job.tracker': '%s:%s' % (self._fqdn, self._jt_port,),
+      'mapred.job.tracker.http.address': '%s:%s' % (self._fqdn, self._jt_http_port,),
+      'jobtracker.thrift.address': '%s:%s' % (self._fqdn, self._jt_thrift_port,),
       'mapred.jobtracker.plugins': 'org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin',
       'mapred.jobtracker.plugins': 'org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin',
-      'mapred.task.tracker.http.address': 'localhost:%s' % (self._tt_http_port,),
+      'mapred.task.tracker.http.address': '%s:%s' % (self._fqdn, self._tt_http_port,),
     }
     }
     write_config(mapred_configs, self._tmppath('conf/mapred-site.xml'))
     write_config(mapred_configs, self._tmppath('conf/mapred-site.xml'))
 
 
@@ -475,11 +480,12 @@ def shared_cluster():
       LOG.exception("Failed to fully bring up test cluster: %s" % (ex,))
       LOG.exception("Failed to fully bring up test cluster: %s" % (ex,))
 
 
     # Fix config to reflect the cluster setup.
     # Fix config to reflect the cluster setup.
-    webhdfs_url = "http://localhost:%s/webhdfs/v1" % (cluster.dfs_http_port,)
+    fqdn = socket.getfqdn()
+    webhdfs_url = "http://%s:%s/webhdfs/v1" % (fqdn, cluster.dfs_http_port,)
     closers = [
     closers = [
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
-      hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing(fqdn),
       hadoop.conf.MR_CLUSTERS['default'].PORT.set_for_testing(cluster._jt_port),
       hadoop.conf.MR_CLUSTERS['default'].PORT.set_for_testing(cluster._jt_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
     ]

+ 2 - 1
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -18,6 +18,7 @@
 import atexit
 import atexit
 import logging
 import logging
 import os
 import os
+import socket
 import subprocess
 import subprocess
 import threading
 import threading
 import time
 import time
@@ -118,7 +119,7 @@ class OozieServerProvider(object):
       LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
       LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
 
 
       finish = (
       finish = (
-        OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
+        OOZIE_URL.set_for_testing("http://%s:%s/oozie" % (socket.getfqdn(), OozieServerProvider.OOZIE_TEST_PORT)),
       )
       )
 
 
       # Setup
       # Setup