Эх сурвалжийг харах

HUE-2477 [impala] Run tests on a live cluster and make them re-entrant

Romain Rigaux 10 жил өмнө
parent
commit
219f3c0

+ 23 - 5
apps/impala/src/impala/tests.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 import re
 import sys
@@ -35,10 +36,15 @@ from beeswax.models import SavedQuery, QueryHistory
 from beeswax.server import dbms
 from beeswax.test_base import get_query_server_config, wait_for_query_to_finish, fetch_query_result_data
 from beeswax.tests import _make_query
+from hadoop.pseudo_hdfs4 import get_db_prefix, is_live_cluser
 
 from impala.conf import SERVER_HOST
 
 
+LOG = logging.getLogger(__name__)
+
+
+
 class MockDbms:
   def get_databases(self):
     return ['db1', 'db2']
@@ -96,13 +102,12 @@ class TestMockedImpala:
 
 
 class TestImpalaIntegration:
-  DATABASE = 'test_hue_impala'
 
   def setUp(self):
     self.finish = []
 
     # We need a real Impala cluster currently
-    if not 'impala' in sys.argv and not os.environ.get('TEST_IMPALAD_HOST'):
+    if (not 'impala' in sys.argv and not os.environ.get('TEST_IMPALAD_HOST')) or not is_live_cluser():
       raise SkipTest
 
     if os.environ.get('TEST_IMPALAD_HOST'):
@@ -112,6 +117,7 @@ class TestImpalaIntegration:
     self.user = User.objects.get(username='test')
     add_to_group('test')
     self.db = dbms.get(self.user, get_query_server_config(name='impala'))
+    self.DATABASE = get_db_prefix()
 
     hql = """
       USE default;
@@ -138,9 +144,21 @@ class TestImpalaIntegration:
     resp = _make_query(self.client, hql, database=self.DATABASE, local=False, server_name='impala')
     resp = wait_for_query_to_finish(self.client, resp, max=30.0)
 
-    def tearDown(self):
-      for f in self.finish:
-        f()
+  def tearDown(self):
+    try:
+      # We need to drop tables before dropping the database
+      hql = """
+      USE default;
+      DROP TABLE IF EXISTS %(db)s.tweets;
+      DROP DATABASE %(db)s;
+      """ % {'db': self.DATABASE}
+      resp = _make_query(self.client, hql, database='default', local=False, server_name='impala')
+      resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    except Exception, e:
+      LOG.exception('Problem deleting Impala integration test DB')
+
+    for f in self.finish:
+      f()
 
   def test_basic_flow(self):
     dbs = self.db.get_databases()

+ 10 - 4
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -45,18 +45,24 @@ STARTUP_DEADLINE = 60.0
 CLEANUP_TMP_DIR = os.environ.get('MINI_CLUSTER_CLEANUP', 'true')
 
 
-def _get_fs_prefix(fs):
+def is_live_cluser():
+  return os.environ.get('LIVE_CLUSTER', 'false').lower() == 'true'
+
+def get_fs_prefix(fs):
   prefix = '/tmp/hue_tests_%s' % str(time.time())
   fs.mkdir(prefix, 0777)
   return prefix
 
+def get_db_prefix():
+  return 'hue_test__%s' % str(time.time()).replace('.', '')
+
 
 class LiveHdfs():
   def __init__(self):
     self.fs = cluster.get_hdfs('default')
     # Assumes /tmp exists and is 1777
 
-    self.fs_prefix = _get_fs_prefix(self.fs)
+    self.fs_prefix = get_fs_prefix(self.fs)
     LOG.info('Using %s as FS root' % self.fs_prefix)
 
     # Might need more
@@ -287,7 +293,7 @@ class PseudoHdfs4(object):
     self.fs.do_as_user('test', self.fs.create_home_dir, '/user/test')
     self.fs.do_as_user('hue', self.fs.create_home_dir, '/user/hue')
 
-    self.fs_prefix = _get_fs_prefix(self.fs)
+    self.fs_prefix = get_fs_prefix(self.fs)
 
 
   def _start_mr2(self, env):
@@ -538,7 +544,7 @@ def shared_cluster():
   global _shared_cluster
 
   if _shared_cluster is None:
-    if os.environ.get('LIVE_CLUSTER', 'false').lower() == 'true':
+    if is_live_cluser():
       cluster = LiveHdfs()
     else:
       cluster = PseudoHdfs4()