Эх сурвалжийг харах

HUE-7803 [core] Remove old imports to JT and NN plugins in minicluster

Romain Rigaux 8 жил өмнө
parent
commit
004deabdee

+ 0 - 12
desktop/libs/hadoop/src/hadoop/conf.py

@@ -168,7 +168,6 @@ def config_validator(user):
 
 
   Called by core check_config() view.
   Called by core check_config() view.
   """
   """
-  from hadoop import job_tracker
   from hadoop.fs import webhdfs
   from hadoop.fs import webhdfs
 
 
   res = []
   res = []
@@ -184,17 +183,6 @@ def config_validator(user):
   if not has_default:
   if not has_default:
     res.append(("hadoop.hdfs_clusters", "You should have an HDFS called 'default'."))
     res.append(("hadoop.hdfs_clusters", "You should have an HDFS called 'default'."))
 
 
-  # MR_CLUSTERS
-  mr_down = []
-  for name in MR_CLUSTERS.keys():
-    cluster = MR_CLUSTERS[name]
-    if cluster.SUBMIT_TO.get():
-      mr_down.extend(job_tracker.test_jt_configuration(cluster))
-      submit_to.append('mapred_clusters.' + name)
-  # If HA still failing
-  if mr_down and len(mr_down) == len(MR_CLUSTERS.keys()):
-    res.extend(mr_down)
-
   # YARN_CLUSTERS
   # YARN_CLUSTERS
   if YARN_CLUSTERS.keys():
   if YARN_CLUSTERS.keys():
     res.extend(test_yarn_configurations(user))
     res.extend(test_yarn_configurations(user))

+ 4 - 14
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -45,7 +45,6 @@ import logging
 import sys
 import sys
 import signal
 import signal
 import shutil
 import shutil
-import socket
 import time
 import time
 import tempfile
 import tempfile
 import json
 import json
@@ -55,8 +54,6 @@ import urllib2
 from desktop.lib import python_util
 from desktop.lib import python_util
 from desktop.lib.test_utils import clear_sys_caches, restore_sys_caches
 from desktop.lib.test_utils import clear_sys_caches, restore_sys_caches
 
 
-from hadoop.fs.hadoopfs import HadoopFileSystem
-from hadoop.job_tracker import LiveJobTracker
 import hadoop.cluster
 import hadoop.cluster
 
 
 # Starts mini cluster suspended until a debugger attaches to it.
 # Starts mini cluster suspended until a debugger attaches to it.
@@ -334,20 +331,13 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
 
 
   @property
   @property
   def fs(self):
   def fs(self):
-    """Creates a HadoopFileSystem object configured for this cluster."""
-    if self._fs is None:
-      self._fs = HadoopFileSystem("localhost", 
-        thrift_port=self.namenode_thrift_port,
-        hdfs_port=self.namenode_port,
-        hadoop_bin_path=hadoop.conf.HADOOP_BIN.get())
-    return self._fs
+    # Deprecated
+    return None
 
 
   @property
   @property
   def jt(self):
   def jt(self):
-    """Creates a LiveJobTracker object configured for this cluster."""
-    if self._jt is None:
-      self._jt = LiveJobTracker("localhost", self.jobtracker_thrift_port)
-    return self._jt
+    # Deprecated
+    return None
 
 
   @property
   @property
   def superuser(self):
   def superuser(self):

+ 2 - 5
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -27,14 +27,13 @@ import tempfile
 import textwrap
 import textwrap
 import time
 import time
 
 
+from desktop.lib.paths import get_run_root
 from desktop.lib.python_util import find_unused_port
 from desktop.lib.python_util import find_unused_port
 from desktop.lib.test_utils import clear_sys_caches, restore_sys_caches
 from desktop.lib.test_utils import clear_sys_caches, restore_sys_caches
 
 
 import hadoop
 import hadoop
 from hadoop import cluster
 from hadoop import cluster
 from hadoop.mini_cluster import write_config
 from hadoop.mini_cluster import write_config
-from hadoop.job_tracker import LiveJobTracker
-from desktop.lib.paths import get_run_root
 
 
 
 
 _shared_cluster = None
 _shared_cluster = None
@@ -171,9 +170,7 @@ class PseudoHdfs4(object):
 
 
   @property
   @property
   def jt(self):
   def jt(self):
-    if self._jt is None:
-      self._jt = LiveJobTracker(self._fqdn, 0)
-    return self._jt
+    return None
 
 
   def stop(self):
   def stop(self):
     def _kill_proc(name, proc):
     def _kill_proc(name, proc):

+ 0 - 30
desktop/libs/hadoop/src/hadoop/tests.py

@@ -33,34 +33,6 @@ from hadoop import confparse
 from hadoop import pseudo_hdfs4
 from hadoop import pseudo_hdfs4
 
 
 
 
-@attr('requires_hadoop')
-def test_live_jobtracker():
-  """
-  Checks that LiveJobTracker never raises
-  exceptions for most of its calls.
-  """
-  raise SkipTest
-
-  minicluster = pseudo_hdfs4.shared_cluster()
-
-  jt = minicluster.jt
-  # Make sure that none of the following
-  # raise.
-  assert_true(jt.queues())
-  assert_true(jt.cluster_status())
-  assert_true(jt.all_task_trackers())
-  assert_true(jt.active_trackers())
-  assert_true(jt.blacklisted_trackers())
-  # not tested: task_tracker
-  assert_true(jt.running_jobs())
-  assert_true(jt.completed_jobs())
-  assert_true(jt.failed_jobs())
-  assert_true(jt.all_jobs())
-  # not tested: get_job_counters
-  assert_true(jt.get_current_time())
-  # not tested: get_job_xml
-
-
 def test_confparse():
 def test_confparse():
   data = """
   data = """
     <configuration>
     <configuration>
@@ -164,9 +136,7 @@ def test_non_default_cluster():
   try:
   try:
     # This is indeed the only hdfs/mr cluster
     # This is indeed the only hdfs/mr cluster
     assert_equal(1, len(cluster.get_all_hdfs()))
     assert_equal(1, len(cluster.get_all_hdfs()))
-    assert_equal(1, len(cluster.all_mrclusters()))
     assert_true(cluster.get_hdfs(NON_DEFAULT_NAME))
     assert_true(cluster.get_hdfs(NON_DEFAULT_NAME))
-    assert_true(cluster.get_mrcluster(NON_DEFAULT_NAME))
 
 
     cli = make_logged_in_client()
     cli = make_logged_in_client()
     # That we can get to a view without errors means that the middlewares work
     # That we can get to a view without errors means that the middlewares work

+ 2 - 2
desktop/libs/liboozie/src/liboozie/submittion2_tests.py

@@ -207,6 +207,8 @@ class TestSubmission(OozieMockBase):
 
 
   def test_update_properties(self):
   def test_update_properties(self):
     finish = []
     finish = []
+    finish.append(MR_CLUSTERS.set_for_testing({'default': {}}))
+    finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     try:
     try:
@@ -247,8 +249,6 @@ class TestSubmission(OozieMockBase):
       })
       })
       submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       assert_equal(properties, submission.properties)
-      submission._update_properties('jtaddress', 'deployment-directory')
-      assert_equal(final_properties, submission.properties)
     finally:
     finally:
       clear_sys_caches()
       clear_sys_caches()
       for reset in finish:
       for reset in finish:

+ 2 - 2
desktop/libs/liboozie/src/liboozie/submittion_tests.py

@@ -166,6 +166,8 @@ class TestSubmission(OozieMockBase):
 
 
   def test_update_properties(self):
   def test_update_properties(self):
     finish = []
     finish = []
+    finish.append(MR_CLUSTERS.set_for_testing({'default': {}}))
+    finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS.set_for_testing({'default': {}}))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
     try:
     try:
@@ -205,8 +207,6 @@ class TestSubmission(OozieMockBase):
       })
       })
       submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=None)
       assert_equal(properties, submission.properties)
       assert_equal(properties, submission.properties)
-      submission._update_properties('jtaddress', 'deployment-directory')
-      assert_equal(final_properties, submission.properties)
     finally:
     finally:
       clear_sys_caches()
       clear_sys_caches()
       for reset in finish:
       for reset in finish: