Browse Source

[desktop] Fix pseudo_hdfs4 starting up a yarn nodemanager

Erick Tryzelaar 11 years ago
parent
commit
52d57d8

+ 8 - 5
desktop/core/src/desktop/lib/python_util.py

@@ -90,11 +90,14 @@ def find_unused_port():
   Unfortunately, this port may not be available by the time
   Unfortunately, this port may not be available by the time
   the subprocess uses it, but this generally works.
   the subprocess uses it, but this generally works.
   """
   """
-  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
-  sock.bind(('127.0.0.1', 0))
-  sock.listen(socket.SOMAXCONN)
-  _, port = sock.getsockname()
-  sock.close()
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  try:
+    sock.bind(('127.0.0.1', 0))
+    sock.listen(socket.SOMAXCONN)
+    _, port = sock.getsockname()
+  finally:
+    sock.close()
+
   return port
   return port
 
 
 
 

+ 4 - 1
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -270,6 +270,9 @@ class PseudoHdfs4(object):
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._nm_proc = self._start_daemon('nodemanager', self.hadoop_conf_dir, self.mr2_env, self._get_yarn_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
     self._hs_proc = self._start_daemon('historyserver', self.hadoop_conf_dir, self.mr2_env, self._get_mapred_bin(self.mr2_env))
 
 
+    # Give them a moment to actually start
+    time.sleep(1)
+
     # Make sure they're running
     # Make sure they're running
     deadline = time.time() + STARTUP_DEADLINE
     deadline = time.time() + STARTUP_DEADLINE
     while not self._is_mr2_ready(self.mr2_env):
     while not self._is_mr2_ready(self.mr2_env):
@@ -455,7 +458,7 @@ class PseudoHdfs4(object):
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.localizer.address' : '%s:%s' % (self._fqdn, self._nm_port,),
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
       'yarn.nodemanager.aux-services.mapreduce.shuffle.class': 'org.apache.hadoop.mapred.ShuffleHandler',
-      'yarn.nodemanager.webapp.address': self._nm_webapp_port,
+      'yarn.nodemanager.webapp.address': '%s:%s' % (self._fqdn, self._nm_webapp_port,),
 
 
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',
       'yarn.app.mapreduce.am.staging-dir': '/tmp/hadoop-yarn/staging',