Browse Source

[hadoop] Change default location of HADOOP_CONF_DIR to `/etc/hadoop/conf'

We used to look under HADOOP_HOME/conf. Things have now changed.
Add NN http port back to config. We need it as the default webhdfs port.
bc Wong 13 năm trước cách đây
mục cha
commit
012bf83126

+ 3 - 5
desktop/libs/hadoop/src/hadoop/conf.py

@@ -48,8 +48,8 @@ HADOOP_BIN = Config("hadoop_bin",
 # TODO(philip): This will need more love for dealing with multiple clusters.
 HADOOP_CONF_DIR = Config(
   key="hadoop_conf_dir",
-  default=None,
-  help="If set, directory to pass to hadoop_bin (from hadoop configuration) as the --config flag.",
+  default="/etc/hadoop/conf",
+  help="Directory to pass to hadoop_bin (from Hadoop configuration) as the --config flag.",
 )
 
 def find_file_recursive(desired_glob, root=None):
@@ -119,8 +119,6 @@ CREDENTIALS_MERGER_JAR = Config("hadoop_credentials_merger_jar",
   private=True)
 
 
-DEFAULT_NN_HTTP_PORT = 50070
-
 HDFS_CLUSTERS = UnspecifiedConfigSection(
   "hdfs_clusters",
   help="One entry for each HDFS cluster",
@@ -134,9 +132,9 @@ HDFS_CLUSTERS = UnspecifiedConfigSection(
                             type=int),
       NN_HDFS_PORT=Config("hdfs_port", help="Hadoop IPC port for the name node", default=8020,
                             type=int),
+      # End deprecation
       NN_HTTP_PORT=Config("http_port", help="Hadoop HTTP port for the name node", default=50070,
                             type=int),
-      # End deprecation
       FS_DEFAULTFS=Config("fs_defaultfs", help="The equivalent of fs.defaultFS (aka fs.default.name)",
                           default="hdfs://localhost:8020"),
       WEBHDFS_URL=Config("webhdfs_url",

+ 1 - 1
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -538,7 +538,7 @@ def _get_service_url(hdfs_config):
   fs_defaultfs = hdfs_config.FS_DEFAULTFS.get()
   netloc = Hdfs.urlsplit(fs_defaultfs)[1]
   host = netloc.split(':')[0]
-  port = hadoop.conf.DEFAULT_NN_HTTP_PORT
+  port = hdfs_config.NN_HTTP_PORT.get()
   return "http://%s:%s/webhdfs/v1" % (host, port)
 
 

+ 1 - 0
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -460,6 +460,7 @@ def shared_cluster():
     # Fix config to reflect the cluster setup.
     closers = [
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
+      hadoop.conf.HDFS_CLUSTERS['default'].NN_HTTP_PORT.set_for_testing(cluster.dfs_http_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_HOST.set_for_testing('localhost'),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]