浏览代码

[hadoop] Support HA for checkconfig YARN check

Romain Rigaux 10 年之前
父节点
当前提交
9bc3653

+ 1 - 1
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -68,7 +68,7 @@ def rm_ha(funct):
       return funct(api, *args, **kwargs)
     except Exception, ex:
       ex_message = str(ex)
-      if 'Connection refused' in ex_message or 'standby RM' in ex_message:
+      if 'Connection refused' in ex_message or 'Connection aborted' in ex_message or 'standby RM' in ex_message:
         LOG.info('Resource Manager not available, trying another RM: %s.' % ex)
         rm_ha = get_next_ha_yarncluster()
         if rm_ha is not None:

+ 11 - 10
desktop/libs/hadoop/src/hadoop/conf.py

@@ -161,8 +161,8 @@ def config_validator(user):
 
   Called by core check_config() view.
   """
-  from hadoop.fs import webhdfs
   from hadoop import job_tracker
+  from hadoop.fs import webhdfs
 
   res = []
   submit_to = []
@@ -189,10 +189,11 @@ def config_validator(user):
     res.extend(mr_down)
 
   # YARN_CLUSTERS
+  if YARN_CLUSTERS.keys():
+    res.extend(test_yarn_configurations(user))
   for name in YARN_CLUSTERS.keys():
     cluster = YARN_CLUSTERS[name]
     if cluster.SUBMIT_TO.get():
-      res.extend(test_yarn_configurations(user))
       submit_to.append('yarn_clusters.' + name)
 
   if not submit_to:
@@ -203,18 +204,18 @@ def config_validator(user):
 
 
 def test_yarn_configurations(user):
-  # Single cluster for now
-  from hadoop.yarn.resource_manager_api import get_resource_manager
-
   result = []
 
   try:
-    url = ''
-    api = get_resource_manager(user.username)
-    url = api._url
-    api.apps()
+    from jobbrowser.api import get_api # Required for cluster HA testing
+  except Exception, e:
+    LOG.warn('Jobbrowser is disabled, skipping test_yarn_configurations')
+    return result
+
+  try:
+    get_api(user, None).get_jobs(user, username=user.username, state='all', text='')
   except Exception, e:
-    msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
+    msg = 'Failed to contact an active Resource Manager: %s' % e
     LOG.exception(msg)
     result.append(('Resource Manager', msg))
 

+ 2 - 1
desktop/libs/hadoop/src/hadoop/yarn/mapreduce_api.py

@@ -41,7 +41,8 @@ def get_mapreduce_api(user):
     try:
       if _api_cache is None:
         yarn_cluster = cluster.get_cluster_conf_for_job_submission()
-        _api_cache = MapreduceApi(user, yarn_cluster.PROXY_API_URL.get(), yarn_cluster.SECURITY_ENABLED.get(), yarn_cluster.SSL_CERT_CA_VERIFY.get())
+        if yarn_cluster is not None:
+          _api_cache = MapreduceApi(user, yarn_cluster.PROXY_API_URL.get(), yarn_cluster.SECURITY_ENABLED.get(), yarn_cluster.SSL_CERT_CA_VERIFY.get())
     finally:
       _api_cache_lock.release()
   return _api_cache