Просмотр исходного кода

[desktop] Fix pseudo_hdfs4 starting up a yarn nodemanager

Erick Tryzelaar 11 лет назад
Родитель
Сommit
52d57d8
2 измененных файлов с 12 добавлено и 6 удалено
  1. 8 5
      desktop/core/src/desktop/lib/python_util.py
  2. 4 1
      desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

+ 8 - 5
desktop/core/src/desktop/lib/python_util.py

@@ -90,11 +90,14 @@ def find_unused_port():
   Unfortunately, this port may not be available by the time
   Unfortunately, this port may not be available by the time
   the subprocess uses it, but this generally works.
   the subprocess uses it, but this generally works.
   """
   """
-  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
-  sock.bind(('127.0.0.1', 0))
-  sock.listen(socket.SOMAXCONN)
-  _, port = sock.getsockname()
-  sock.close()
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  try:
+    sock.bind(('127.0.0.1', 0))
+    sock.listen(socket.SOMAXCONN)
+    _, port = sock.getsockname()
+  finally:
+    sock.close()
+
   return port
   return port
 
 
 
 

+ 4 - 1
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -270,6 +270,9 @@ class PseudoHdfs4(object):
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
 
 
+    # Give them a moment to actually start
+    time.sleep(1)
+
     # Make sure they're running
     # Make sure they're running
     deadline = time.time() + STARTUP_DEADLINE
     deadline = time.time() + STARTUP_DEADLINE
     while not self._is_mr2_ready(self.mr2_env):
     while not self._is_mr2_ready(self.mr2_env):
@@ -455,7 +458,7 @@ class PseudoHdfs4(object):
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
-      'yarn.nodemanager.webapp.address': self._nm_webapp_port,
+      'yarn.nodemanager.webapp.address': '%s:%s' % (self._fqdn, self._nm_webapp_port,),
 
 
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',