瀏覽代碼

[desktop] Fix pseudo_hdfs4 starting up a yarn nodemanager

Erick Tryzelaar 11 年之前
父節點
當前提交
52d57d8a8d
共有 2 個文件被更改,包括 12 次插入6 次删除
  1. 8 5
      desktop/core/src/desktop/lib/python_util.py
  2. 4 1
      desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

+ 8 - 5
desktop/core/src/desktop/lib/python_util.py

@@ -90,11 +90,14 @@ def find_unused_port():
   Unfortunately, this port may not be available by the time
   Unfortunately, this port may not be available by the time
   the subprocess uses it, but this generally works.
   the subprocess uses it, but this generally works.
   """
   """
-  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
-  sock.bind(('127.0.0.1', 0))
-  sock.listen(socket.SOMAXCONN)
-  _, port = sock.getsockname()
-  sock.close()
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  try:
+    sock.bind(('127.0.0.1', 0))
+    sock.listen(socket.SOMAXCONN)
+    _, port = sock.getsockname()
+  finally:
+    sock.close()
+
   return port
   return port
 
 
 
 

+ 4 - 1
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -270,6 +270,9 @@ class PseudoHdfs4(object):
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
 
 
+    # Give them a moment to actually start
+    time.sleep(1)
+
     # Make sure they're running
     # Make sure they're running
     deadline = time.time() + STARTUP_DEADLINE
     deadline = time.time() + STARTUP_DEADLINE
     while not self._is_mr2_ready(self.mr2_env):
     while not self._is_mr2_ready(self.mr2_env):
@@ -455,7 +458,7 @@ class PseudoHdfs4(object):
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
-      'yarn.nodemanager.webapp.address': self._nm_webapp_port,
+      'yarn.nodemanager.webapp.address': '%s:%s' % (self._fqdn, self._nm_webapp_port,),
 
 
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',