Procházet zdrojové kódy

HUE-532. HDFS thrift plugin port is also in thriftfs-site.xml

* Removed thriftfs-default.xml. Defined constants in ThriftFsConfig.
  Use the default values in the code.
* Keep loading the `thriftfs-site.xml' resource, for backward
  compatibility.
* Convert a static block to inject config resource into a static
  method, and invoke that explicitly.
bc Wong před 14 roky
rodič
revize
351f2a00c5

+ 2 - 5
desktop/libs/hadoop/java/src/main/java/org/apache/hadoop/thriftfs/DatanodePlugin.java

@@ -48,10 +48,6 @@ public class DatanodePlugin
   extends org.apache.hadoop.hdfs.server.datanode.DatanodePlugin
   extends org.apache.hadoop.hdfs.server.datanode.DatanodePlugin
   implements Configurable {
   implements Configurable {
 
 
-
-  /** Name of the configuration property of the Thrift server address */
-  public static final String THRIFT_ADDRESS_PROPERTY =
-      "dfs.thrift.datanode.address";
   /**
   /**
    * Default address and port this server will bind to, in case nothing is found
    * Default address and port this server will bind to, in case nothing is found
    * in the configuration object.
    * in the configuration object.
@@ -161,10 +157,11 @@ public class DatanodePlugin
 
 
   @Override
   @Override
   public void start(Object service) {
   public void start(Object service) {
+    ThriftUtils.initConfigResource();
     this.datanode = (DataNode)service;
     this.datanode = (DataNode)service;
     try {
     try {
       InetSocketAddress address = NetUtils.createSocketAddr(
       InetSocketAddress address = NetUtils.createSocketAddr(
-        conf.get(THRIFT_ADDRESS_PROPERTY, DEFAULT_THRIFT_ADDRESS));
+        conf.get(ThriftFsConfig.DFS_THRIFT_DATANODE_ADDR_KEY, DEFAULT_THRIFT_ADDRESS));
 
 
       thriftServer = new ThriftPluginServer(
       thriftServer = new ThriftPluginServer(
         address, new ProcessorFactory());
         address, new ProcessorFactory());

+ 6 - 6
desktop/libs/hadoop/java/src/main/java/org/apache/hadoop/thriftfs/NamenodePlugin.java

@@ -65,9 +65,6 @@ import org.apache.thrift.transport.TTransport;
 public class NamenodePlugin extends org.apache.hadoop.hdfs.server.namenode.NamenodePlugin implements
 public class NamenodePlugin extends org.apache.hadoop.hdfs.server.namenode.NamenodePlugin implements
     Configurable {
     Configurable {
 
 
-  /** Name of the configuration property of the Thrift server address */
-  public static final String THRIFT_ADDRESS_PROPERTY = "dfs.thrift.address";
-
   /**
   /**
    * Default address and port this server will bind to, in case nothing is found
    * Default address and port this server will bind to, in case nothing is found
    * in the configuration object.
    * in the configuration object.
@@ -427,16 +424,19 @@ public class NamenodePlugin extends org.apache.hadoop.hdfs.server.namenode.Namen
 
 
   @Override
   @Override
   public void start(Object service) {
   public void start(Object service) {
+    ThriftUtils.initConfigResource();
     this.namenode = (NameNode) service;
     this.namenode = (NameNode) service;
     try {
     try {
-      InetSocketAddress address = NetUtils.createSocketAddr(conf.get(THRIFT_ADDRESS_PROPERTY,
-          DEFAULT_THRIFT_ADDRESS));
+      InetSocketAddress address = NetUtils.createSocketAddr(
+          conf.get(ThriftFsConfig.DFS_THRIFT_ADDR_KEY,
+                   DEFAULT_THRIFT_ADDRESS));
 
 
       this.thriftServer = new ThriftPluginServer(address, new ProcessorFactory());
       this.thriftServer = new ThriftPluginServer(address, new ProcessorFactory());
       thriftServer.setConf(conf);
       thriftServer.setConf(conf);
       thriftServer.start();
       thriftServer.start();
       // The port may have been 0, so we update it.
       // The port may have been 0, so we update it.
-      conf.set(THRIFT_ADDRESS_PROPERTY, address.getHostName() + ":" + thriftServer.getPort());
+      conf.set(ThriftFsConfig.DFS_THRIFT_ADDR_KEY,
+               address.getHostName() + ":" + thriftServer.getPort());
     } catch (Exception e) {
     } catch (Exception e) {
       throw new RuntimeException("Cannot start Thrift namenode plug-in", e);
       throw new RuntimeException("Cannot start Thrift namenode plug-in", e);
     }
     }

+ 44 - 0
desktop/libs/hadoop/java/src/main/java/org/apache/hadoop/thriftfs/ThriftFsConfig.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.thriftfs;
+
+/**
+ * Thrift FS config constants
+ */
+public class ThriftFsConfig {
+  /** Datanode thrift plugin <host>:<port> */
+  public static final String DFS_THRIFT_DATANODE_ADDR_KEY = "dfs.thrift.datanode.address";
+
+  /** Namenode thrift plugin <host>:<port> */
+  public static final String DFS_THRIFT_ADDR_KEY = "dfs.thrift.address";
+
+  /** Min number of threads for thrift server */
+  public static final String DFS_THRIFT_THREADS_MIN_KEY = "dfs.thrift.threads.min";
+
+  /** Max number of threads for thrift server */
+  public static final String DFS_THRIFT_THREADS_MAX_KEY = "dfs.thrift.threads.max";
+
+  /** Timeout (in seconds) for thrift server threads */
+  public static final String DFS_THRIFT_TIMEOUT_KEY = "dfs.thrift.timeout";
+
+  /** Read timeout (in milliseconds) for thrift socket */
+  public static final String DFS_THRIFT_SOCKET_TIMEOUT_KEY = "dfs.thrift.socket.timeout";
+
+  /** Queue size for thrift server */
+  public static final String DFS_THRIFT_QUEUE_SIZE_KEY = "dfs.thrift.queue.size";
+}

+ 7 - 10
desktop/libs/hadoop/java/src/main/java/org/apache/hadoop/thriftfs/ThriftPluginServer.java

@@ -54,11 +54,6 @@ public class ThriftPluginServer implements Configurable, Runnable {
 
 
   static final int SOCKET_READ_TIMEOUT = 5000; // 5sec
   static final int SOCKET_READ_TIMEOUT = 5000; // 5sec
 
 
-  static {
-    Configuration.addDefaultResource("thriftfs-default.xml");
-    Configuration.addDefaultResource("thriftfs-site.xml");
-  }
-
   public ThriftPluginServer(InetSocketAddress address,
   public ThriftPluginServer(InetSocketAddress address,
                             TProcessorFactory processorFactory)
                             TProcessorFactory processorFactory)
     throws TTransportException {
     throws TTransportException {
@@ -107,15 +102,17 @@ public class ThriftPluginServer implements Configurable, Runnable {
         sock.bind(address);
         sock.bind(address);
       }
       }
 
 
-      int socketTimeout = conf.getInt("dfs.thrift.socket.timeout", SOCKET_READ_TIMEOUT);
+      int socketTimeout = conf.getInt(ThriftFsConfig.DFS_THRIFT_SOCKET_TIMEOUT_KEY,
+                                      SOCKET_READ_TIMEOUT);
 
 
       TServerTransport transport = new TServerSocket(sock, socketTimeout);
       TServerTransport transport = new TServerSocket(sock, socketTimeout);
       SanerThreadPoolServer.Options options = new SanerThreadPoolServer.Options();
       SanerThreadPoolServer.Options options = new SanerThreadPoolServer.Options();
-      options.minWorkerThreads = conf.getInt("dfs.thrift.threads.min", 5);
-      options.maxWorkerThreads = conf.getInt("dfs.thrift.threads.max", 20);
-      options.stopTimeoutVal = conf.getInt("dfs.thrift.timeout", 60);
+      options.minWorkerThreads = conf.getInt(ThriftFsConfig.DFS_THRIFT_THREADS_MIN_KEY, 5);
+      options.maxWorkerThreads = conf.getInt(ThriftFsConfig.DFS_THRIFT_THREADS_MAX_KEY, 20);
+      options.stopTimeoutVal = conf.getInt(ThriftFsConfig.DFS_THRIFT_TIMEOUT_KEY, 60);
       options.stopTimeoutUnit = TimeUnit.SECONDS;
       options.stopTimeoutUnit = TimeUnit.SECONDS;
-      options.queueSize = conf.getInt("dfs.thrift.queue.size", 4*options.maxWorkerThreads);
+      options.queueSize = conf.getInt(ThriftFsConfig.DFS_THRIFT_QUEUE_SIZE_KEY,
+                                      4*options.maxWorkerThreads);
 
 
       server = new SanerThreadPoolServer(
       server = new SanerThreadPoolServer(
         processorFactory, transport,
         processorFactory, transport,

+ 7 - 1
desktop/libs/hadoop/java/src/main/java/org/apache/hadoop/thriftfs/ThriftUtils.java

@@ -69,6 +69,10 @@ public class ThriftUtils {
   public static final String HUE_USER_NAME_KEY = "hue.kerberos.principal.shortname";
   public static final String HUE_USER_NAME_KEY = "hue.kerberos.principal.shortname";
   public static final String HUE_USER_NAME_DEFAULT = "hue";
   public static final String HUE_USER_NAME_DEFAULT = "hue";
 
 
+  public static void initConfigResource() {
+    Configuration.addDefaultResource("thriftfs-site.xml");
+  }
+
   public static LocatedBlock fromThrift(Block block) {
   public static LocatedBlock fromThrift(Block block) {
     if (block == null) {
     if (block == null) {
       return null;
       return null;
@@ -263,7 +267,8 @@ public class ThriftUtils {
    */
    */
   public static Namenode.Client createNamenodeClient(Configuration conf)
   public static Namenode.Client createNamenodeClient(Configuration conf)
       throws Exception {
       throws Exception {
-    String s = conf.get(NamenodePlugin.THRIFT_ADDRESS_PROPERTY, NamenodePlugin.DEFAULT_THRIFT_ADDRESS);
+    String s = conf.get(ThriftFsConfig.DFS_THRIFT_ADDR_KEY,
+                        NamenodePlugin.DEFAULT_THRIFT_ADDRESS);
     // TODO(todd) use fs.default.name here if set to 0.0.0.0 - but share this with the code in
     // TODO(todd) use fs.default.name here if set to 0.0.0.0 - but share this with the code in
     // SecondaryNameNode that does the same
     // SecondaryNameNode that does the same
     InetSocketAddress addr = NetUtils.createSocketAddr(s);
     InetSocketAddress addr = NetUtils.createSocketAddr(s);
@@ -276,6 +281,7 @@ public class ThriftUtils {
       addr = new InetSocketAddress(nnAddr.getAddress(), addr.getPort());
       addr = new InetSocketAddress(nnAddr.getAddress(), addr.getPort());
     }
     }
 
 
+    LOG.info("Creating NameNode client against " + addr);
     TTransport t = new TSocket(addr.getHostName(), addr.getPort());
     TTransport t = new TSocket(addr.getHostName(), addr.getPort());
     if (UserGroupInformation.isSecurityEnabled()) {
     if (UserGroupInformation.isSecurityEnabled()) {
       t = new HadoopThriftAuthBridge.Client()
       t = new HadoopThriftAuthBridge.Client()

+ 0 - 57
desktop/libs/hadoop/java/src/main/resources/thriftfs-default.xml

@@ -1,57 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed to Cloudera, Inc. under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  Cloudera, Inc. licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<!-- Do not modify this file directly.  Instead, copy entries that you    -->
-<!-- wish to modify from this file into thriftfs-site.xml and change them -->
-<!-- there.  If thriftfs-site.xml does not already exist, create it.      -->
-
-<configuration>
-<property>
-  <name>dfs.thrift.address</name>
-  <value>0.0.0.0:10090</value>
-  <description>
-    The address where the Thrift namenode server will listen to.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-<property>
-  <name>dfs.thrift.datanode.address</name>
-  <value>0.0.0.0:0</value>
-  <description>
-    The address where the Thrift datanode server will listen to.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-<property>
-  <name>dfs.thrift.threads.min</name>
-  <value>5</value>
-  <descrition>Minimum number of Thrift server threads.</descrition>
-</property>
-<property>
-  <name>dfs.thrift.threads.max</name>
-  <value>20</value>
-  <descrition>Maximum number of Thrift server threads.</descrition>
-</property>
-<property>
-  <name>dfs.thrift.timeout</name>
-  <value>60</value>
-  <descrition>Timeout in seconds for Thrift server threads.</descrition>
-</property>
-</configuration>

+ 3 - 8
desktop/libs/hadoop/java/src/test/java/org/apache/hadoop/thriftfs/Helper.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.thriftfs.api.Datanode;
 import org.apache.hadoop.thriftfs.api.Datanode;
 import org.apache.hadoop.thriftfs.api.DatanodeInfo;
 import org.apache.hadoop.thriftfs.api.DatanodeInfo;
 import org.apache.hadoop.thriftfs.api.RequestContext;
 import org.apache.hadoop.thriftfs.api.RequestContext;
+import org.apache.hadoop.thriftfs.ThriftFsConfig;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.protocol.TProtocol;
@@ -50,20 +51,14 @@ import org.apache.thrift.transport.TTransport;
  */
  */
 public class Helper {
 public class Helper {
 
 
-  public static final String NAMENODE_ADDRESS_PROPERTY =
-    org.apache.hadoop.thriftfs.NamenodePlugin.THRIFT_ADDRESS_PROPERTY;
-
-  public static final String DATANODE_ADDRESS_PROPERTY =
-    org.apache.hadoop.thriftfs.DatanodePlugin.THRIFT_ADDRESS_PROPERTY;
-
   public static final String TEST_USER="hadoop";
   public static final String TEST_USER="hadoop";
   public static final String TEST_GROUP="supergroup";
   public static final String TEST_GROUP="supergroup";
 
 
   /** Create a configuration object for the unit tests. */
   /** Create a configuration object for the unit tests. */
   public static Configuration createConf() {
   public static Configuration createConf() {
     Configuration conf = new Configuration();
     Configuration conf = new Configuration();
-    conf.set(NAMENODE_ADDRESS_PROPERTY, "127.0.0.1:10090");
-    conf.set(DATANODE_ADDRESS_PROPERTY, "127.0.0.1:0");
+    conf.set(ThriftFsConfig.DFS_THRIFT_ADDR_KEY, "127.0.0.1:10090");
+    conf.set(ThriftFsConfig.DFS_THRIFT_DATANODE_ADDR_KEY, "127.0.0.1:0");
     conf.set("slave.host.name", "127.0.0.1");
     conf.set("slave.host.name", "127.0.0.1");
     conf.setStrings("dfs.namenode.plugins", NamenodePlugin.class.getName());
     conf.setStrings("dfs.namenode.plugins", NamenodePlugin.class.getName());
     conf.setStrings("dfs.datanode.plugins", DatanodePlugin.class.getName());
     conf.setStrings("dfs.datanode.plugins", DatanodePlugin.class.getName());