Browse Source

HUE-219. Make Hue work with CDH3 trunk

Many changes to make Hue work against the latest build of CDH3. The most
substantial changes are to the plugins and to the Jobsub interposition code.
Note that this commit does not address any Beeswax changes, and in fact removes
Beeswax from the apps/Makefile. That work will come in a subsequent commit.
Aaron T. Myers 15 years ago
parent
commit
b8dbf1fa57
34 changed files with 1270 additions and 3255 deletions
  1. 0 1
      apps/Makefile
  2. 1 1
      apps/jobbrowser/src/jobbrowser/models.py
  3. 18 18
      apps/jobbrowser/src/jobbrowser/tests.py
  4. 1 1
      apps/jobsub/src/jobsub/java/build.xml
  5. 2 2
      apps/jobsub/src/jobsub/java/src/META-INF/aop-ajc.xml
  6. 0 59
      apps/jobsub/src/jobsub/java/src/com/cloudera/jobsub/UgiFixer.aj
  7. 3 3
      apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/JobClientTrace.aj
  8. 2 2
      apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/JobClientTracer.java
  9. 99 0
      apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/UgiFixer.aj
  10. 9 5
      apps/jobsub/src/jobsub/management/commands/jobsub_setup.py
  11. 1 1
      apps/jobsub/src/jobsub/server.py
  12. 7 4
      apps/jobsub/src/jobsub/tests.py
  13. 2 2
      desktop/core/src/desktop/middleware.py
  14. 18 2
      desktop/libs/hadoop/Makefile
  15. 0 14
      desktop/libs/hadoop/gen-py/hadoop/api/hdfs/Namenode-remote
  16. 29 460
      desktop/libs/hadoop/gen-py/hadoop/api/hdfs/Namenode.py
  17. 20 8
      desktop/libs/hadoop/gen-py/hadoop/api/hdfs/ttypes.py
  18. 159 63
      desktop/libs/hadoop/java/gen-java/org/apache/hadoop/thriftfs/api/Block.java
  19. 32 1841
      desktop/libs/hadoop/java/gen-java/org/apache/hadoop/thriftfs/api/Namenode.java
  20. 7 18
      desktop/libs/hadoop/java/if/hdfs.thrift
  21. 293 210
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/mapred/ThriftJobTrackerPlugin.java
  22. 57 54
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/DatanodePlugin.java
  23. 232 349
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/NamenodePlugin.java
  24. 25 45
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftHandlerBase.java
  25. 0 7
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftServerContext.java
  26. 2 3
      desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftUtils.java
  27. 11 16
      desktop/libs/hadoop/java/src/test/org/apache/hadoop/thriftfs/Helper.java
  28. 2 19
      desktop/libs/hadoop/java/src/test/org/apache/hadoop/thriftfs/TestNamenodePlugin.java
  29. 6 0
      desktop/libs/hadoop/src/hadoop/conf.py
  30. 13 17
      desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py
  31. 9 14
      desktop/libs/hadoop/src/hadoop/job_tracker.py
  32. 59 16
      desktop/libs/hadoop/src/hadoop/mini_cluster.py
  33. 75 0
      desktop/libs/hadoop/static-group-mapping/build.xml
  34. 76 0
      desktop/libs/hadoop/static-group-mapping/src/org/apache/hadoop/security/StaticUserGroupMapping.java

+ 0 - 1
apps/Makefile

@@ -31,7 +31,6 @@ default: env-install
 .PHONY: clean default distclean install
 
 APPS := about \
-	beeswax \
 	filebrowser \
 	help \
 	jobbrowser \

+ 1 - 1
apps/jobbrowser/src/jobbrowser/models.py

@@ -413,7 +413,7 @@ class TaskAttempt(object):
                       '%s:%s' % (tracker.host, tracker.httpPort),
                       'tasklog',
                       None,
-                      'taskid=%s' % (self.attemptId,),
+                      'attemptid=%s' % (self.attemptId,),
                       None))
     LOGGER.info('Retrieving %s' % (url,))
     try:

+ 18 - 18
apps/jobbrowser/src/jobbrowser/tests.py

@@ -67,6 +67,11 @@ class TestJobBrowserWithHadoop(object):
     if not cluster.fs.exists("/user/test"):
       cluster.fs.mkdir("/user/test")
     cluster.fs.chown("/user/test", "test", "test")
+
+    if not cluster.fs.exists("/tmp"):
+      cluster.fs.mkdir("/tmp")
+    cluster.fs.chmod("/tmp", int('777', 8))
+
     cluster.fs.setuser("test")
 
     cls.cluster = cluster
@@ -184,24 +189,17 @@ class TestJobBrowserWithHadoop(object):
     assert_equal(
       response.context['job'].counters['FileSystemCounters']['counters']['FILE_BYTES_WRITTEN'],
       {'map': 44L, 'reduce': 12L, 'displayName': 'FILE_BYTES_WRITTEN', 'name': 'FILE_BYTES_WRITTEN'}
-      )
-    assert_equal(
-      response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter'],
-      {'displayName': 'Job Counters ',
-       'name': 'org.apache.hadoop.mapred.JobInProgress$Counter',
-       'counters': {
-          'TOTAL_LAUNCHED_MAPS': {
-            'total': 1L,
-            'displayName': 'Launched map tasks',
-            'name': 'TOTAL_LAUNCHED_MAPS'
-            },
-          'TOTAL_LAUNCHED_REDUCES': {
-            'total': 1L,
-            'displayName': 'Launched reduce tasks',
-            'name': 'TOTAL_LAUNCHED_REDUCES'
-            }
-          }
-       })
+    )
+
+    # We can't just check the complete contents of the python map because the
+    # SLOTS_MILLIS_* entries have a variable number of milliseconds from
+    # run-to-run.
+    assert_equal(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['TOTAL_LAUNCHED_MAPS']['total'], 1)
+    assert_equal(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['TOTAL_LAUNCHED_REDUCES']['total'], 1)
+    assert_equal(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['FALLOW_SLOTS_MILLIS_MAPS']['total'], 0)
+    assert_equal(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['FALLOW_SLOTS_MILLIS_REDUCES']['total'], 0)
+    assert_true(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['SLOTS_MILLIS_MAPS']['total'] > 0)
+    assert_true(response.context['job'].counters['org.apache.hadoop.mapred.JobInProgress$Counter']['counters']['SLOTS_MILLIS_REDUCES']['total'] > 0)
 
     # Check conf keys made it
     assert_equal(response.context['job'].conf_keys['mapredReducerClass'],
@@ -272,4 +270,6 @@ class TestJobBrowserWithHadoop(object):
     # Test dock jobs
     response = self.client.get('/jobbrowser/dock_jobs/')
     assert_true('completed' in response.content)
+    # TODO(atm): I'm pretty sure the following test only passes because of
+    # failed jobs which are run in test_failed_jobs
     assert_true('failed' in response.content)

+ 1 - 1
apps/jobsub/src/jobsub/java/build.xml

@@ -43,7 +43,7 @@
 
   <target name="build" depends="init">
     <iajc sourceroots="${src.dir}"
-          destDir="${build.dir}/out">
+          destDir="${build.dir}/out" source="1.5">
       <classpath refid="build.classpath" />
     </iajc>
   </target>

+ 2 - 2
apps/jobsub/src/jobsub/java/src/META-INF/aop-ajc.xml

@@ -17,8 +17,8 @@
 -->
 <aspectj>
 <aspects>
-<aspect name="com.cloudera.jobsub.JobClientTrace"/>
-<aspect name="com.cloudera.jobsub.UgiFixer"/>
+<aspect name="org.apache.hadoop.security.JobClientTrace"/>
+<aspect name="org.apache.hadoop.security.UgiFixer"/>
 </aspects>
 </aspectj>
 

+ 0 - 59
apps/jobsub/src/jobsub/java/src/com/cloudera/jobsub/UgiFixer.aj

@@ -1,59 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.cloudera.jobsub;
-
-import org.apache.hadoop.security.UnixUserGroupInformation;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Overrides the 'whoami' and 'groups' calls that Hadoop does with
- * environment variables, specifying the username and group.
- */
-public aspect UgiFixer {
-  private static final Log LOG = LogFactory.getLog(UgiFixer.class);
-  private static final String USER_VAR = "HUE_JOBSUB_USER";
-  private static final String DEFAULT_USER = "default_jobsub_user";
-  
-  // TODO(philip): Full-on group support would mean
-  // dealing with comma-delimited values here.
-  private static final String GROUPS_VAR = "HUE_JOBSUB_GROUPS";
-  private static final String DEFAULT_GROUP = "default_jobsub_group";
-
-  UgiFixer() {
-    LOG.info("Hue UGI fixer aspect loaded.");
-  }
-
-  pointcut getUserName():
-        call(String UnixUserGroupInformation.getUnixUserName());
-
-  pointcut getUnixGroups():
-        call(String[] UnixUserGroupInformation.getUnixGroups());
-
-  String around(): getUserName() {
-    String user = System.getenv(USER_VAR);
-    return user == null ? DEFAULT_USER : user;
-  }
-
-  String[] around(): getUnixGroups() {
-    String group = System.getenv(GROUPS_VAR);
-    if (group == null) {
-      return new String[] { DEFAULT_GROUP };
-    } else {
-      return group.split(",");
-    }
-  }
-}

+ 3 - 3
apps/jobsub/src/jobsub/java/src/com/cloudera/jobsub/JobClientTrace.aj → apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/JobClientTrace.aj

@@ -13,7 +13,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package com.cloudera.jobsub;
+package org.apache.hadoop.security;
 
 import org.apache.hadoop.mapred.*;
 import org.apache.commons.logging.Log;
@@ -25,9 +25,9 @@ public aspect JobClientTrace {
   JobClientTrace() {
     LOG.info("Hue job submission aspect loaded.");
   }
+
   RunningJob around(JobConf conf):
-    call(RunningJob JobClient.submitJobInternal(JobConf))
-    && args(conf) {
+    call(RunningJob JobClient.submitJobInternal(JobConf)) && args(conf) {
     RunningJob ret = proceed(conf);
     JobClientTracer.getInstance().submittedJob(ret);
     return ret;

+ 2 - 2
apps/jobsub/src/jobsub/java/src/com/cloudera/jobsub/JobClientTracer.java → apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/JobClientTracer.java

@@ -13,7 +13,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package com.cloudera.jobsub;
+package org.apache.hadoop.security;
 
 import org.apache.hadoop.mapred.RunningJob;
 import java.io.PrintStream;
@@ -26,7 +26,7 @@ public class JobClientTracer {
   private static JobClientTracer instance = new JobClientTracer();
   private PrintStream reportStream;
 
-  public static final String DEFAULT_JOB_LOG="reported-jobs.txt";
+  public static final String DEFAULT_JOB_LOG = "reported-jobs.txt";
 
   public synchronized static JobClientTracer getInstance() {
     return instance;

+ 99 - 0
apps/jobsub/src/jobsub/java/src/org/apache/hadoop/security/UgiFixer.aj

@@ -0,0 +1,99 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package org.apache.hadoop.security;
+
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
+import javax.security.auth.login.LoginException;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.spi.LoginModule;
+
+import org.apache.hadoop.security.User;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Overrides the HadoopConfiguration.getAppConfigurationEntry to return the
+ * user that Jobsub is running this job as.
+ */
+public aspect UgiFixer {
+  private static final Log LOG = LogFactory.getLog(UgiFixer.class);
+
+  static {
+    LOG.info("Hue UGI fixer aspect loaded.");
+  }
+
+  private static final String USER_VAR = "HUE_JOBSUB_USER";
+  private static final String DEFAULT_USER = "default_jobsub_user";
+
+  private static final AppConfigurationEntry JOBSUB_LOGIN =
+    new AppConfigurationEntry(JobsubLoginModule.class.getName(),
+                              LoginModuleControlFlag.REQUIRED,
+                              new HashMap<String,String>());
+
+  private static final AppConfigurationEntry[] JOBSUB_CONF =
+    new AppConfigurationEntry[]{ JOBSUB_LOGIN };
+
+  public static class JobsubLoginModule implements LoginModule {
+    private Subject subject;
+
+    public boolean abort() throws LoginException {
+      return true;
+    }
+
+    public boolean commit() throws LoginException {
+      if (!subject.getPrincipals(User.class).isEmpty()) {
+        return true;
+      }
+      String user = System.getenv(USER_VAR);
+      subject.getPrincipals().add(new User(user == null ? DEFAULT_USER : user));
+      return true;
+    }
+
+    public void initialize(Subject subject, CallbackHandler callbackHandler,
+                           Map<String, ?> sharedState, Map<String, ?> options) {
+      this.subject = subject;
+    }
+
+    public boolean login() throws LoginException {
+      return true;
+    }
+
+    public boolean logout() throws LoginException {
+      return true;
+    }
+  }
+
+  pointcut getAppConfigurationEntry(String appName):
+    execution(AppConfigurationEntry[] Configuration.getAppConfigurationEntry(String)) && args(appName) && within(UserGroupInformation);
+
+  AppConfigurationEntry[] around(String appName):
+    getAppConfigurationEntry(appName) {
+      if (!appName.equals("hadoop-simple")) {
+        LOG.warn("getAppConfigurationEntry() called for auth method other than simple: " + appName);
+        return proceed(appName);
+      } else {
+        return JOBSUB_CONF;
+      }
+    }
+}

+ 9 - 5
apps/jobsub/src/jobsub/management/commands/jobsub_setup.py

@@ -32,13 +32,15 @@ from hadoop import cluster
 import hadoop.conf
 import jobsub.conf
 
+LOG = logging.getLogger(__name__)
+
 class Command(NoArgsCommand):
   """Creates file system for testing."""
   def handle_noargs(self, **options):
     remote_fs = cluster.get_hdfs()
     if hasattr(remote_fs, "setuser"):
-      remote_fs.setuser("hue", ["supergroup"])
-    logging.info("Using remote fs: %s" % str(remote_fs))
+      remote_fs.setuser(remote_fs.superuser)
+    LOG.info("Using remote fs: %s" % str(remote_fs))
 
     # Copy over examples/ and script_templates/ directories
     for dirname in ("examples", "script_templates"):
@@ -113,8 +115,10 @@ def copy_dir(local_dir, remote_fs, remote_dir):
 
 def copy_file(local_src, remote_fs, remote_dst):
   if remote_fs.exists(remote_dst):
-    logging.info("%s already exists.  Skipping." % remote_dst)
+    LOG.info("%s already exists.  Skipping." % remote_dst)
     return
+  else:
+    LOG.info("%s does not exist. trying to copy" % remote_dst)
     
   if os.path.isfile(local_src):
     src = file(local_src)
@@ -122,10 +126,10 @@ def copy_file(local_src, remote_fs, remote_dst):
       dst = remote_fs.open(remote_dst, "w")
       try:
         shutil.copyfileobj(src, dst)
-        logging.info("Copied %s -> %s" % (local_src, remote_dst))
+        LOG.info("Copied %s -> %s" % (local_src, remote_dst))
       finally:
         dst.close()
     finally:
       src.close()
   else:
-    logging.info("Skipping %s (not a file)" % local_src)
+    LOG.info("Skipping %s (not a file)" % local_src)

+ 1 - 1
apps/jobsub/src/jobsub/server.py

@@ -226,7 +226,7 @@ class PlanRunner(object):
     env = {      
       'HADOOP_HOME': hadoop.conf.HADOOP_HOME.get(), 
       'HADOOP_OPTS': "-javaagent:%s %s" % (jobsub.conf.ASPECTJWEAVER.get(), java_prop_str),
-      'HADOOP_CLASSPATH': jobsub.conf.ASPECTPATH.get(),
+      'HADOOP_CLASSPATH': ':'.join([jobsub.conf.ASPECTPATH.get(), hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()]),
       'HUE_JOBTRACE_LOG': self.internal_file_name("jobs"),
       'HUE_JOBSUB_USER': self.plan.user,
       'HUE_JOBSUB_GROUPS': ",".join(self.plan.groups),

+ 7 - 4
apps/jobsub/src/jobsub/tests.py

@@ -174,7 +174,7 @@ def test_job_design_cycle():
   response = c.post("/jobsub/delete/%d" % not_mine.id)
   assert_true("Permission Denied." in response.context["error"])
 
-def create_test_user_homedir(cluster):
+def setup_cluster_fs(cluster):
   """
   Irritatingly, pi doesn't run unless /user/test exists.
   """
@@ -182,8 +182,11 @@ def create_test_user_homedir(cluster):
   if not cluster.fs.exists("/user/test"):
     cluster.fs.mkdir("/user/test")
   cluster.fs.chown("/user/test", "test", "test")
+  if not cluster.fs.exists("/tmp"):
+    cluster.fs.mkdir("/tmp")
+  cluster.fs.chmod("/tmp", int('777', 8))
   cluster.fs.setuser("test")
-create_test_user_homedir.__test__ = False # Don't confuse nose.
+setup_cluster_fs.__test__ = False # Don't confuse nose.
 
 @attr('requires_hadoop')
 def test_job_submission():
@@ -208,7 +211,7 @@ def test_job_submission():
       assert_true("<form " in response.content)
 
       # Create home dir
-      create_test_user_homedir(cluster)
+      setup_cluster_fs(cluster)
 
       # Prepare sample data
       f = cluster.fs.open("/user/test/input", "w")
@@ -323,7 +326,7 @@ def test_jobsub_setup_and_samples():
     # And now submit and run the samples
     # pi Example
     # Irritatingly, /user/test needs to exist first
-    create_test_user_homedir(cluster)
+    setup_cluster_fs(cluster)
     id = JobDesign.objects.get(name__contains="Example: Pi").id
     response = c.get("/jobsub/submit/%d" % id)
     assert_true("Iterations per mapper" in response.content)

+ 2 - 2
desktop/core/src/desktop/middleware.py

@@ -135,12 +135,12 @@ class ClusterMiddleware(object):
         raise
 
     if request.user.is_authenticated() and request.fs is not None:
-      request.fs.setuser(request.user.username, request.user.get_groups())
+      request.fs.setuser(request.user.username)
 
     if request.user.is_authenticated() and has_hadoop:
       request.jt = cluster.get_mrcluster()
       if request.jt is not None:
-        request.jt.setuser(request.user.username, request.user.get_groups())
+        request.jt.setuser(request.user.username)
     else:
       request.jt = None
 

+ 18 - 2
desktop/libs/hadoop/Makefile

@@ -45,10 +45,16 @@ PLUGIN_JAVA_DIR := $(PLUGIN_DIR)/java
 PLUGIN := $(PLUGIN_JAVA_LIB)/hue-plugins-$(DESKTOP_VERSION).jar
 BLD_DIR_PLUGINS := $(PLUGIN_DIR)/build/java
 
-compile: $(PLUGIN)
+STATIC_GROUP_MAPPING_DIR := $(realpath .)/static-group-mapping
+STATIC_GROUP_MAPPING_SRC := $(STATIC_GROUP_MAPPING_DIR)/src
+STATIC_GROUP_MAPPING_BUILD := $(STATIC_GROUP_MAPPING_DIR)/build
+STATIC_GROUP_MAPPING_LIB := $(STATIC_GROUP_MAPPING_DIR)/java-lib
+STATIC_GROUP_MAPPING := $(STATIC_GROUP_MAPPING_LIB)/static-group-mapping-$(DESKTOP_VERSION).jar
+
+compile: $(PLUGIN) $(STATIC_GROUP_MAPPING)
 
 clean::
-	rm -Rf $(PLUGIN_JAVA_LIB)
+	rm -Rf $(PLUGIN_JAVA_LIB) $(STATIC_GROUP_MAPPING_LIB)
 
 ifneq (,$(wildcard $(PLUGIN_JAVA_DIR)))
 PLUGIN_OPTS := -Dbuild.dir=$(BLD_DIR_PLUGINS) \
@@ -65,6 +71,16 @@ $(PLUGIN):
 	$(error Cannot build hadoop plugin without source)
 endif
 
+STATIC_GROUP_MAPPING_OPTS := -Dbuild.dir=$(STATIC_GROUP_MAPPING_BUILD) \
+         -Dhadoop.home=$(HADOOP_HOME) \
+         -Dstatic-group-mapping.version=$(DESKTOP_VERSION)
+
+$(STATIC_GROUP_MAPPING): $(shell find $(STATIC_GROUP_MAPPING_SRC))
+	mkdir -p $(STATIC_GROUP_MAPPING_LIB)
+	@echo "--- Building static UserGroupMappingService"
+	cd $(STATIC_GROUP_MAPPING_DIR) && ant $(ANT_OPTS) $(STATIC_GROUP_MAPPING_OPTS) jar
+	cp $(STATIC_GROUP_MAPPING_BUILD)/$(notdir $(STATIC_GROUP_MAPPING)) $(STATIC_GROUP_MAPPING)
+
 #
 # Tell the `bdist' target to exclude our java source.
 #

+ 0 - 14
desktop/libs/hadoop/gen-py/hadoop/api/hdfs/Namenode-remote

@@ -26,8 +26,6 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '   df(RequestContext ctx)'
   print '  void enterSafeMode(RequestContext ctx)'
   print '   getBlocks(RequestContext ctx, string path, i64 offset, i64 length)'
-  print '   getDatanodeReport(RequestContext ctx, DatanodeReportType type)'
-  print '  DFSHealthReport getHealthReport(RequestContext ctx)'
   print '  i64 getPreferredBlockSize(RequestContext ctx, string path)'
   print '  bool isInSafeMode(RequestContext ctx)'
   print '  void leaveSafeMode(RequestContext ctx)'
@@ -123,18 +121,6 @@ elif cmd == 'getBlocks':
     sys.exit(1)
   pp.pprint(client.getBlocks(eval(args[0]),args[1],eval(args[2]),eval(args[3]),))
 
-elif cmd == 'getDatanodeReport':
-  if len(args) != 2:
-    print 'getDatanodeReport requires 2 args'
-    sys.exit(1)
-  pp.pprint(client.getDatanodeReport(eval(args[0]),eval(args[1]),))
-
-elif cmd == 'getHealthReport':
-  if len(args) != 1:
-    print 'getHealthReport requires 1 args'
-    sys.exit(1)
-  pp.pprint(client.getHealthReport(eval(args[0]),))
-
 elif cmd == 'getPreferredBlockSize':
   if len(args) != 2:
     print 'getPreferredBlockSize requires 2 args'

+ 29 - 460
desktop/libs/hadoop/gen-py/hadoop/api/hdfs/Namenode.py

@@ -82,27 +82,6 @@ class Iface(hadoop.api.common.HadoopServiceBase.Iface):
     """
     pass
 
-  def getDatanodeReport(self, ctx, type):
-    """
-    Get a report on the system's current data nodes.
-    Note that ctx is currently ignored by the server.
-    
-    Parameters:
-     - ctx
-     - type: Type of data nodes to return
-    information about.
-    """
-    pass
-
-  def getHealthReport(self, ctx):
-    """
-    Get a health report of DFS.  Note that ctx is ignored by the server.
-    
-    Parameters:
-     - ctx
-    """
-    pass
-
   def getPreferredBlockSize(self, ctx, path):
     """
     Get the preferred block size for the given file.
@@ -442,8 +421,6 @@ class Client(hadoop.api.common.HadoopServiceBase.Client, Iface):
     self._iprot.readMessageEnd()
     if result.success != None:
       return result.success
-    if result.err != None:
-      raise result.err
     raise TApplicationException(TApplicationException.MISSING_RESULT, "df failed: unknown result");
 
   def enterSafeMode(self, ctx):
@@ -518,78 +495,6 @@ class Client(hadoop.api.common.HadoopServiceBase.Client, Iface):
       raise result.err
     raise TApplicationException(TApplicationException.MISSING_RESULT, "getBlocks failed: unknown result");
 
-  def getDatanodeReport(self, ctx, type):
-    """
-    Get a report on the system's current data nodes.
-    Note that ctx is currently ignored by the server.
-    
-    Parameters:
-     - ctx
-     - type: Type of data nodes to return
-    information about.
-    """
-    self.send_getDatanodeReport(ctx, type)
-    return self.recv_getDatanodeReport()
-
-  def send_getDatanodeReport(self, ctx, type):
-    self._oprot.writeMessageBegin('getDatanodeReport', TMessageType.CALL, self._seqid)
-    args = getDatanodeReport_args()
-    args.ctx = ctx
-    args.type = type
-    args.write(self._oprot)
-    self._oprot.writeMessageEnd()
-    self._oprot.trans.flush()
-
-  def recv_getDatanodeReport(self, ):
-    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
-    if mtype == TMessageType.EXCEPTION:
-      x = TApplicationException()
-      x.read(self._iprot)
-      self._iprot.readMessageEnd()
-      raise x
-    result = getDatanodeReport_result()
-    result.read(self._iprot)
-    self._iprot.readMessageEnd()
-    if result.success != None:
-      return result.success
-    if result.err != None:
-      raise result.err
-    raise TApplicationException(TApplicationException.MISSING_RESULT, "getDatanodeReport failed: unknown result");
-
-  def getHealthReport(self, ctx):
-    """
-    Get a health report of DFS.  Note that ctx is ignored by the server.
-    
-    Parameters:
-     - ctx
-    """
-    self.send_getHealthReport(ctx)
-    return self.recv_getHealthReport()
-
-  def send_getHealthReport(self, ctx):
-    self._oprot.writeMessageBegin('getHealthReport', TMessageType.CALL, self._seqid)
-    args = getHealthReport_args()
-    args.ctx = ctx
-    args.write(self._oprot)
-    self._oprot.writeMessageEnd()
-    self._oprot.trans.flush()
-
-  def recv_getHealthReport(self, ):
-    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
-    if mtype == TMessageType.EXCEPTION:
-      x = TApplicationException()
-      x.read(self._iprot)
-      self._iprot.readMessageEnd()
-      raise x
-    result = getHealthReport_result()
-    result.read(self._iprot)
-    self._iprot.readMessageEnd()
-    if result.success != None:
-      return result.success
-    if result.err != None:
-      raise result.err
-    raise TApplicationException(TApplicationException.MISSING_RESULT, "getHealthReport failed: unknown result");
-
   def getPreferredBlockSize(self, ctx, path):
     """
     Get the preferred block size for the given file.
@@ -1248,8 +1153,6 @@ class Processor(hadoop.api.common.HadoopServiceBase.Processor, Iface, TProcessor
     self._processMap["df"] = Processor.process_df
     self._processMap["enterSafeMode"] = Processor.process_enterSafeMode
     self._processMap["getBlocks"] = Processor.process_getBlocks
-    self._processMap["getDatanodeReport"] = Processor.process_getDatanodeReport
-    self._processMap["getHealthReport"] = Processor.process_getHealthReport
     self._processMap["getPreferredBlockSize"] = Processor.process_getPreferredBlockSize
     self._processMap["isInSafeMode"] = Processor.process_isInSafeMode
     self._processMap["leaveSafeMode"] = Processor.process_leaveSafeMode
@@ -1316,10 +1219,7 @@ class Processor(hadoop.api.common.HadoopServiceBase.Processor, Iface, TProcessor
     args.read(iprot)
     iprot.readMessageEnd()
     result = df_result()
-    try:
-      result.success = self._handler.df(args.ctx)
-    except hadoop.api.common.ttypes.IOException, err:
-      result.err = err
+    result.success = self._handler.df(args.ctx)
     oprot.writeMessageBegin("df", TMessageType.REPLY, seqid)
     result.write(oprot)
     oprot.writeMessageEnd()
@@ -1353,34 +1253,6 @@ class Processor(hadoop.api.common.HadoopServiceBase.Processor, Iface, TProcessor
     oprot.writeMessageEnd()
     oprot.trans.flush()
 
-  def process_getDatanodeReport(self, seqid, iprot, oprot):
-    args = getDatanodeReport_args()
-    args.read(iprot)
-    iprot.readMessageEnd()
-    result = getDatanodeReport_result()
-    try:
-      result.success = self._handler.getDatanodeReport(args.ctx, args.type)
-    except hadoop.api.common.ttypes.IOException, err:
-      result.err = err
-    oprot.writeMessageBegin("getDatanodeReport", TMessageType.REPLY, seqid)
-    result.write(oprot)
-    oprot.writeMessageEnd()
-    oprot.trans.flush()
-
-  def process_getHealthReport(self, seqid, iprot, oprot):
-    args = getHealthReport_args()
-    args.read(iprot)
-    iprot.readMessageEnd()
-    result = getHealthReport_result()
-    try:
-      result.success = self._handler.getHealthReport(args.ctx)
-    except hadoop.api.common.ttypes.IOException, err:
-      result.err = err
-    oprot.writeMessageBegin("getHealthReport", TMessageType.REPLY, seqid)
-    result.write(oprot)
-    oprot.writeMessageEnd()
-    oprot.trans.flush()
-
   def process_getPreferredBlockSize(self, seqid, iprot, oprot):
     args = getPreferredBlockSize_args()
     args.read(iprot)
@@ -1987,17 +1859,14 @@ class df_result(object):
   """
   Attributes:
    - success
-   - err
   """
 
   thrift_spec = (
     (0, TType.LIST, 'success', (TType.I64,None), None, ), # 0
-    (1, TType.STRUCT, 'err', (hadoop.api.common.ttypes.IOException, hadoop.api.common.ttypes.IOException.thrift_spec), None, ), # 1
   )
 
-  def __init__(self, success=None, err=None,):
+  def __init__(self, success=None,):
     self.success = success
-    self.err = err
 
   def read(self, iprot):
     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -2018,12 +1887,6 @@ class df_result(object):
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
-      elif fid == 1:
-        if ftype == TType.STRUCT:
-          self.err = hadoop.api.common.ttypes.IOException()
-          self.err.read(iprot)
-        else:
-          iprot.skip(ftype)
       else:
         iprot.skip(ftype)
       iprot.readFieldEnd()
@@ -2041,10 +1904,6 @@ class df_result(object):
         oprot.writeI64(iter13)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
-    if self.err != None:
-      oprot.writeFieldBegin('err', TType.STRUCT, 1)
-      self.err.write(oprot)
-      oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
@@ -2358,296 +2217,6 @@ class getBlocks_result(object):
   def __ne__(self, other):
     return not (self == other)
 
-class getDatanodeReport_args(object):
-  """
-  Attributes:
-   - ctx
-   - type: Type of data nodes to return
-  information about.
-  """
-
-  thrift_spec = (
-    None, # 0
-    (1, TType.I32, 'type', None, None, ), # 1
-    None, # 2
-    None, # 3
-    None, # 4
-    None, # 5
-    None, # 6
-    None, # 7
-    None, # 8
-    None, # 9
-    (10, TType.STRUCT, 'ctx', (hadoop.api.common.ttypes.RequestContext, hadoop.api.common.ttypes.RequestContext.thrift_spec), None, ), # 10
-  )
-
-  def __init__(self, ctx=None, type=None,):
-    self.ctx = ctx
-    self.type = type
-
-  def read(self, iprot):
-    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-      return
-    iprot.readStructBegin()
-    while True:
-      (fname, ftype, fid) = iprot.readFieldBegin()
-      if ftype == TType.STOP:
-        break
-      if fid == 10:
-        if ftype == TType.STRUCT:
-          self.ctx = hadoop.api.common.ttypes.RequestContext()
-          self.ctx.read(iprot)
-        else:
-          iprot.skip(ftype)
-      elif fid == 1:
-        if ftype == TType.I32:
-          self.type = iprot.readI32();
-        else:
-          iprot.skip(ftype)
-      else:
-        iprot.skip(ftype)
-      iprot.readFieldEnd()
-    iprot.readStructEnd()
-
-  def write(self, oprot):
-    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-      return
-    oprot.writeStructBegin('getDatanodeReport_args')
-    if self.type != None:
-      oprot.writeFieldBegin('type', TType.I32, 1)
-      oprot.writeI32(self.type)
-      oprot.writeFieldEnd()
-    if self.ctx != None:
-      oprot.writeFieldBegin('ctx', TType.STRUCT, 10)
-      self.ctx.write(oprot)
-      oprot.writeFieldEnd()
-    oprot.writeFieldStop()
-    oprot.writeStructEnd()
-
-  def __repr__(self):
-    L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
-    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-  def __eq__(self, other):
-    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
-
-  def __ne__(self, other):
-    return not (self == other)
-
-class getDatanodeReport_result(object):
-  """
-  Attributes:
-   - success
-   - err
-  """
-
-  thrift_spec = (
-    (0, TType.LIST, 'success', (TType.STRUCT,(DatanodeInfo, DatanodeInfo.thrift_spec)), None, ), # 0
-    (1, TType.STRUCT, 'err', (hadoop.api.common.ttypes.IOException, hadoop.api.common.ttypes.IOException.thrift_spec), None, ), # 1
-  )
-
-  def __init__(self, success=None, err=None,):
-    self.success = success
-    self.err = err
-
-  def read(self, iprot):
-    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-      return
-    iprot.readStructBegin()
-    while True:
-      (fname, ftype, fid) = iprot.readFieldBegin()
-      if ftype == TType.STOP:
-        break
-      if fid == 0:
-        if ftype == TType.LIST:
-          self.success = []
-          (_etype24, _size21) = iprot.readListBegin()
-          for _i25 in xrange(_size21):
-            _elem26 = DatanodeInfo()
-            _elem26.read(iprot)
-            self.success.append(_elem26)
-          iprot.readListEnd()
-        else:
-          iprot.skip(ftype)
-      elif fid == 1:
-        if ftype == TType.STRUCT:
-          self.err = hadoop.api.common.ttypes.IOException()
-          self.err.read(iprot)
-        else:
-          iprot.skip(ftype)
-      else:
-        iprot.skip(ftype)
-      iprot.readFieldEnd()
-    iprot.readStructEnd()
-
-  def write(self, oprot):
-    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-      return
-    oprot.writeStructBegin('getDatanodeReport_result')
-    if self.success != None:
-      oprot.writeFieldBegin('success', TType.LIST, 0)
-      oprot.writeListBegin(TType.STRUCT, len(self.success))
-      for iter27 in self.success:
-        iter27.write(oprot)
-      oprot.writeListEnd()
-      oprot.writeFieldEnd()
-    if self.err != None:
-      oprot.writeFieldBegin('err', TType.STRUCT, 1)
-      self.err.write(oprot)
-      oprot.writeFieldEnd()
-    oprot.writeFieldStop()
-    oprot.writeStructEnd()
-
-  def __repr__(self):
-    L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
-    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-  def __eq__(self, other):
-    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
-
-  def __ne__(self, other):
-    return not (self == other)
-
-class getHealthReport_args(object):
-  """
-  Attributes:
-   - ctx
-  """
-
-  thrift_spec = (
-    None, # 0
-    None, # 1
-    None, # 2
-    None, # 3
-    None, # 4
-    None, # 5
-    None, # 6
-    None, # 7
-    None, # 8
-    None, # 9
-    (10, TType.STRUCT, 'ctx', (hadoop.api.common.ttypes.RequestContext, hadoop.api.common.ttypes.RequestContext.thrift_spec), None, ), # 10
-  )
-
-  def __init__(self, ctx=None,):
-    self.ctx = ctx
-
-  def read(self, iprot):
-    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-      return
-    iprot.readStructBegin()
-    while True:
-      (fname, ftype, fid) = iprot.readFieldBegin()
-      if ftype == TType.STOP:
-        break
-      if fid == 10:
-        if ftype == TType.STRUCT:
-          self.ctx = hadoop.api.common.ttypes.RequestContext()
-          self.ctx.read(iprot)
-        else:
-          iprot.skip(ftype)
-      else:
-        iprot.skip(ftype)
-      iprot.readFieldEnd()
-    iprot.readStructEnd()
-
-  def write(self, oprot):
-    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-      return
-    oprot.writeStructBegin('getHealthReport_args')
-    if self.ctx != None:
-      oprot.writeFieldBegin('ctx', TType.STRUCT, 10)
-      self.ctx.write(oprot)
-      oprot.writeFieldEnd()
-    oprot.writeFieldStop()
-    oprot.writeStructEnd()
-
-  def __repr__(self):
-    L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
-    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-  def __eq__(self, other):
-    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
-
-  def __ne__(self, other):
-    return not (self == other)
-
-class getHealthReport_result(object):
-  """
-  Attributes:
-   - success
-   - err
-  """
-
-  thrift_spec = (
-    (0, TType.STRUCT, 'success', (DFSHealthReport, DFSHealthReport.thrift_spec), None, ), # 0
-    (1, TType.STRUCT, 'err', (hadoop.api.common.ttypes.IOException, hadoop.api.common.ttypes.IOException.thrift_spec), None, ), # 1
-  )
-
-  def __init__(self, success=None, err=None,):
-    self.success = success
-    self.err = err
-
-  def read(self, iprot):
-    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-      return
-    iprot.readStructBegin()
-    while True:
-      (fname, ftype, fid) = iprot.readFieldBegin()
-      if ftype == TType.STOP:
-        break
-      if fid == 0:
-        if ftype == TType.STRUCT:
-          self.success = DFSHealthReport()
-          self.success.read(iprot)
-        else:
-          iprot.skip(ftype)
-      elif fid == 1:
-        if ftype == TType.STRUCT:
-          self.err = hadoop.api.common.ttypes.IOException()
-          self.err.read(iprot)
-        else:
-          iprot.skip(ftype)
-      else:
-        iprot.skip(ftype)
-      iprot.readFieldEnd()
-    iprot.readStructEnd()
-
-  def write(self, oprot):
-    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-      return
-    oprot.writeStructBegin('getHealthReport_result')
-    if self.success != None:
-      oprot.writeFieldBegin('success', TType.STRUCT, 0)
-      self.success.write(oprot)
-      oprot.writeFieldEnd()
-    if self.err != None:
-      oprot.writeFieldBegin('err', TType.STRUCT, 1)
-      self.err.write(oprot)
-      oprot.writeFieldEnd()
-    oprot.writeFieldStop()
-    oprot.writeStructEnd()
-
-  def __repr__(self):
-    L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
-    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-  def __eq__(self, other):
-    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
-
-  def __ne__(self, other):
-    return not (self == other)
-
 class getPreferredBlockSize_args(object):
   """
   Attributes:
@@ -3155,11 +2724,11 @@ class ls_result(object):
       if fid == 0:
         if ftype == TType.LIST:
           self.success = []
-          (_etype31, _size28) = iprot.readListBegin()
-          for _i32 in xrange(_size28):
-            _elem33 = Stat()
-            _elem33.read(iprot)
-            self.success.append(_elem33)
+          (_etype24, _size21) = iprot.readListBegin()
+          for _i25 in xrange(_size21):
+            _elem26 = Stat()
+            _elem26.read(iprot)
+            self.success.append(_elem26)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -3182,8 +2751,8 @@ class ls_result(object):
     if self.success != None:
       oprot.writeFieldBegin('success', TType.LIST, 0)
       oprot.writeListBegin(TType.STRUCT, len(self.success))
-      for iter34 in self.success:
-        iter34.write(oprot)
+      for iter27 in self.success:
+        iter27.write(oprot)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     if self.err != None:
@@ -3682,11 +3251,11 @@ class reportBadBlocks_args(object):
       elif fid == 1:
         if ftype == TType.LIST:
           self.blocks = []
-          (_etype38, _size35) = iprot.readListBegin()
-          for _i39 in xrange(_size35):
-            _elem40 = Block()
-            _elem40.read(iprot)
-            self.blocks.append(_elem40)
+          (_etype31, _size28) = iprot.readListBegin()
+          for _i32 in xrange(_size28):
+            _elem33 = Block()
+            _elem33.read(iprot)
+            self.blocks.append(_elem33)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -3703,8 +3272,8 @@ class reportBadBlocks_args(object):
     if self.blocks != None:
       oprot.writeFieldBegin('blocks', TType.LIST, 1)
       oprot.writeListBegin(TType.STRUCT, len(self.blocks))
-      for iter41 in self.blocks:
-        iter41.write(oprot)
+      for iter34 in self.blocks:
+        iter34.write(oprot)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     if self.ctx != None:
@@ -4117,10 +3686,10 @@ class multiGetContentSummary_args(object):
       elif fid == 1:
         if ftype == TType.LIST:
           self.paths = []
-          (_etype45, _size42) = iprot.readListBegin()
-          for _i46 in xrange(_size42):
-            _elem47 = iprot.readString();
-            self.paths.append(_elem47)
+          (_etype38, _size35) = iprot.readListBegin()
+          for _i39 in xrange(_size35):
+            _elem40 = iprot.readString();
+            self.paths.append(_elem40)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -4137,8 +3706,8 @@ class multiGetContentSummary_args(object):
     if self.paths != None:
       oprot.writeFieldBegin('paths', TType.LIST, 1)
       oprot.writeListBegin(TType.STRING, len(self.paths))
-      for iter48 in self.paths:
-        oprot.writeString(iter48)
+      for iter41 in self.paths:
+        oprot.writeString(iter41)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     if self.ctx != None:
@@ -4187,11 +3756,11 @@ class multiGetContentSummary_result(object):
       if fid == 0:
         if ftype == TType.LIST:
           self.success = []
-          (_etype52, _size49) = iprot.readListBegin()
-          for _i53 in xrange(_size49):
-            _elem54 = ContentSummary()
-            _elem54.read(iprot)
-            self.success.append(_elem54)
+          (_etype45, _size42) = iprot.readListBegin()
+          for _i46 in xrange(_size42):
+            _elem47 = ContentSummary()
+            _elem47.read(iprot)
+            self.success.append(_elem47)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -4214,8 +3783,8 @@ class multiGetContentSummary_result(object):
     if self.success != None:
       oprot.writeFieldBegin('success', TType.LIST, 0)
       oprot.writeListBegin(TType.STRUCT, len(self.success))
-      for iter55 in self.success:
-        iter55.write(oprot)
+      for iter48 in self.success:
+        iter48.write(oprot)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     if self.err != None:

+ 20 - 8
desktop/libs/hadoop/gen-py/hadoop/api/hdfs/ttypes.py

@@ -242,8 +242,9 @@ class Block(object):
    - path: Path of the file which this block belongs to.
    - numBytes: Length of this block.
    - genStamp: Generational stamp of this block.
-   - startOffset: Offset of the first byte of the block relative to the start of the file
    - nodes: List of data nodes with copies  of this block.
+   - startOffset: Offset of the first byte of the block relative to the start of the file
+   - token: The serialized token associated with this block.
   """
 
   thrift_spec = (
@@ -254,15 +255,17 @@ class Block(object):
     (4, TType.I64, 'genStamp', None, None, ), # 4
     (5, TType.LIST, 'nodes', (TType.STRUCT,(DatanodeInfo, DatanodeInfo.thrift_spec)), None, ), # 5
     (6, TType.I64, 'startOffset', None, None, ), # 6
+    (7, TType.STRING, 'token', None, None, ), # 7
   )
 
-  def __init__(self, blockId=None, path=None, numBytes=None, genStamp=None, startOffset=None, nodes=None,):
+  def __init__(self, blockId=None, path=None, numBytes=None, genStamp=None, nodes=None, startOffset=None, token=None,):
     self.blockId = blockId
     self.path = path
     self.numBytes = numBytes
     self.genStamp = genStamp
-    self.startOffset = startOffset
     self.nodes = nodes
+    self.startOffset = startOffset
+    self.token = token
 
   def read(self, iprot):
     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -293,11 +296,6 @@ class Block(object):
           self.genStamp = iprot.readI64();
         else:
           iprot.skip(ftype)
-      elif fid == 6:
-        if ftype == TType.I64:
-          self.startOffset = iprot.readI64();
-        else:
-          iprot.skip(ftype)
       elif fid == 5:
         if ftype == TType.LIST:
           self.nodes = []
@@ -309,6 +307,16 @@ class Block(object):
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
+      elif fid == 6:
+        if ftype == TType.I64:
+          self.startOffset = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 7:
+        if ftype == TType.STRING:
+          self.token = iprot.readString();
+        else:
+          iprot.skip(ftype)
       else:
         iprot.skip(ftype)
       iprot.readFieldEnd()
@@ -346,6 +354,10 @@ class Block(object):
       oprot.writeFieldBegin('startOffset', TType.I64, 6)
       oprot.writeI64(self.startOffset)
       oprot.writeFieldEnd()
+    if self.token != None:
+      oprot.writeFieldBegin('token', TType.STRING, 7)
+      oprot.writeString(self.token)
+      oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 

+ 159 - 63
desktop/libs/hadoop/java/gen-java/org/apache/hadoop/thriftfs/api/Block.java

@@ -35,8 +35,9 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
   private static final TField PATH_FIELD_DESC = new TField("path", TType.STRING, (short)2);
   private static final TField NUM_BYTES_FIELD_DESC = new TField("numBytes", TType.I64, (short)3);
   private static final TField GEN_STAMP_FIELD_DESC = new TField("genStamp", TType.I64, (short)4);
-  private static final TField START_OFFSET_FIELD_DESC = new TField("startOffset", TType.I64, (short)6);
   private static final TField NODES_FIELD_DESC = new TField("nodes", TType.LIST, (short)5);
+  private static final TField START_OFFSET_FIELD_DESC = new TField("startOffset", TType.I64, (short)6);
+  private static final TField TOKEN_FIELD_DESC = new TField("token", TType.STRING, (short)7);
 
   /**
    * Block ID (unique among all blocks in a filesystem).
@@ -54,14 +55,18 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
    * Generational stamp of this block.
    */
   public long genStamp;
+  /**
+   * List of data nodes with copies  of this block.
+   */
+  public List<DatanodeInfo> nodes;
   /**
    * Offset of the first byte of the block relative to the start of the file
    */
   public long startOffset;
   /**
-   * List of data nodes with copies  of this block.
+   * The serialized token associated with this block.
    */
-  public List<DatanodeInfo> nodes;
+  public String token;
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements TFieldIdEnum {
@@ -81,14 +86,18 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
      * Generational stamp of this block.
      */
     GEN_STAMP((short)4, "genStamp"),
+    /**
+     * List of data nodes with copies  of this block.
+     */
+    NODES((short)5, "nodes"),
     /**
      * Offset of the first byte of the block relative to the start of the file
      */
     START_OFFSET((short)6, "startOffset"),
     /**
-     * List of data nodes with copies  of this block.
+     * The serialized token associated with this block.
      */
-    NODES((short)5, "nodes");
+    TOKEN((short)7, "token");
 
     private static final Map<Integer, _Fields> byId = new HashMap<Integer, _Fields>();
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -157,11 +166,13 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
         new FieldValueMetaData(TType.I64)));
     put(_Fields.GEN_STAMP, new FieldMetaData("genStamp", TFieldRequirementType.DEFAULT, 
         new FieldValueMetaData(TType.I64)));
-    put(_Fields.START_OFFSET, new FieldMetaData("startOffset", TFieldRequirementType.DEFAULT, 
-        new FieldValueMetaData(TType.I64)));
     put(_Fields.NODES, new FieldMetaData("nodes", TFieldRequirementType.DEFAULT, 
         new ListMetaData(TType.LIST, 
             new StructMetaData(TType.STRUCT, DatanodeInfo.class))));
+    put(_Fields.START_OFFSET, new FieldMetaData("startOffset", TFieldRequirementType.DEFAULT, 
+        new FieldValueMetaData(TType.I64)));
+    put(_Fields.TOKEN, new FieldMetaData("token", TFieldRequirementType.DEFAULT, 
+        new FieldValueMetaData(TType.STRING)));
   }});
 
   static {
@@ -176,8 +187,9 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     String path,
     long numBytes,
     long genStamp,
+    List<DatanodeInfo> nodes,
     long startOffset,
-    List<DatanodeInfo> nodes)
+    String token)
   {
     this();
     this.blockId = blockId;
@@ -187,9 +199,10 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     setNumBytesIsSet(true);
     this.genStamp = genStamp;
     setGenStampIsSet(true);
+    this.nodes = nodes;
     this.startOffset = startOffset;
     setStartOffsetIsSet(true);
-    this.nodes = nodes;
+    this.token = token;
   }
 
   /**
@@ -204,7 +217,6 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     }
     this.numBytes = other.numBytes;
     this.genStamp = other.genStamp;
-    this.startOffset = other.startOffset;
     if (other.isSetNodes()) {
       List<DatanodeInfo> __this__nodes = new ArrayList<DatanodeInfo>();
       for (DatanodeInfo other_element : other.nodes) {
@@ -212,6 +224,10 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
       }
       this.nodes = __this__nodes;
     }
+    this.startOffset = other.startOffset;
+    if (other.isSetToken()) {
+      this.token = other.token;
+    }
   }
 
   public Block deepCopy() {
@@ -340,6 +356,51 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     __isset_bit_vector.set(__GENSTAMP_ISSET_ID, value);
   }
 
+  public int getNodesSize() {
+    return (this.nodes == null) ? 0 : this.nodes.size();
+  }
+
+  public java.util.Iterator<DatanodeInfo> getNodesIterator() {
+    return (this.nodes == null) ? null : this.nodes.iterator();
+  }
+
+  public void addToNodes(DatanodeInfo elem) {
+    if (this.nodes == null) {
+      this.nodes = new ArrayList<DatanodeInfo>();
+    }
+    this.nodes.add(elem);
+  }
+
+  /**
+   * List of data nodes with copies  of this block.
+   */
+  public List<DatanodeInfo> getNodes() {
+    return this.nodes;
+  }
+
+  /**
+   * List of data nodes with copies  of this block.
+   */
+  public Block setNodes(List<DatanodeInfo> nodes) {
+    this.nodes = nodes;
+    return this;
+  }
+
+  public void unsetNodes() {
+    this.nodes = null;
+  }
+
+  /** Returns true if field nodes is set (has been asigned a value) and false otherwise */
+  public boolean isSetNodes() {
+    return this.nodes != null;
+  }
+
+  public void setNodesIsSet(boolean value) {
+    if (!value) {
+      this.nodes = null;
+    }
+  }
+
   /**
    * Offset of the first byte of the block relative to the start of the file
    */
@@ -369,48 +430,33 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     __isset_bit_vector.set(__STARTOFFSET_ISSET_ID, value);
   }
 
-  public int getNodesSize() {
-    return (this.nodes == null) ? 0 : this.nodes.size();
-  }
-
-  public java.util.Iterator<DatanodeInfo> getNodesIterator() {
-    return (this.nodes == null) ? null : this.nodes.iterator();
-  }
-
-  public void addToNodes(DatanodeInfo elem) {
-    if (this.nodes == null) {
-      this.nodes = new ArrayList<DatanodeInfo>();
-    }
-    this.nodes.add(elem);
-  }
-
   /**
-   * List of data nodes with copies  of this block.
+   * The serialized token associated with this block.
    */
-  public List<DatanodeInfo> getNodes() {
-    return this.nodes;
+  public String getToken() {
+    return this.token;
   }
 
   /**
-   * List of data nodes with copies  of this block.
+   * The serialized token associated with this block.
    */
-  public Block setNodes(List<DatanodeInfo> nodes) {
-    this.nodes = nodes;
+  public Block setToken(String token) {
+    this.token = token;
     return this;
   }
 
-  public void unsetNodes() {
-    this.nodes = null;
+  public void unsetToken() {
+    this.token = null;
   }
 
-  /** Returns true if field nodes is set (has been asigned a value) and false otherwise */
-  public boolean isSetNodes() {
-    return this.nodes != null;
+  /** Returns true if field token is set (has been asigned a value) and false otherwise */
+  public boolean isSetToken() {
+    return this.token != null;
   }
 
-  public void setNodesIsSet(boolean value) {
+  public void setTokenIsSet(boolean value) {
     if (!value) {
-      this.nodes = null;
+      this.token = null;
     }
   }
 
@@ -448,6 +494,14 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
       }
       break;
 
+    case NODES:
+      if (value == null) {
+        unsetNodes();
+      } else {
+        setNodes((List<DatanodeInfo>)value);
+      }
+      break;
+
     case START_OFFSET:
       if (value == null) {
         unsetStartOffset();
@@ -456,11 +510,11 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
       }
       break;
 
-    case NODES:
+    case TOKEN:
       if (value == null) {
-        unsetNodes();
+        unsetToken();
       } else {
-        setNodes((List<DatanodeInfo>)value);
+        setToken((String)value);
       }
       break;
 
@@ -485,11 +539,14 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     case GEN_STAMP:
       return new Long(getGenStamp());
 
+    case NODES:
+      return getNodes();
+
     case START_OFFSET:
       return new Long(getStartOffset());
 
-    case NODES:
-      return getNodes();
+    case TOKEN:
+      return getToken();
 
     }
     throw new IllegalStateException();
@@ -510,10 +567,12 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
       return isSetNumBytes();
     case GEN_STAMP:
       return isSetGenStamp();
-    case START_OFFSET:
-      return isSetStartOffset();
     case NODES:
       return isSetNodes();
+    case START_OFFSET:
+      return isSetStartOffset();
+    case TOKEN:
+      return isSetToken();
     }
     throw new IllegalStateException();
   }
@@ -571,6 +630,15 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
         return false;
     }
 
+    boolean this_present_nodes = true && this.isSetNodes();
+    boolean that_present_nodes = true && that.isSetNodes();
+    if (this_present_nodes || that_present_nodes) {
+      if (!(this_present_nodes && that_present_nodes))
+        return false;
+      if (!this.nodes.equals(that.nodes))
+        return false;
+    }
+
     boolean this_present_startOffset = true;
     boolean that_present_startOffset = true;
     if (this_present_startOffset || that_present_startOffset) {
@@ -580,12 +648,12 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
         return false;
     }
 
-    boolean this_present_nodes = true && this.isSetNodes();
-    boolean that_present_nodes = true && that.isSetNodes();
-    if (this_present_nodes || that_present_nodes) {
-      if (!(this_present_nodes && that_present_nodes))
+    boolean this_present_token = true && this.isSetToken();
+    boolean that_present_token = true && that.isSetToken();
+    if (this_present_token || that_present_token) {
+      if (!(this_present_token && that_present_token))
         return false;
-      if (!this.nodes.equals(that.nodes))
+      if (!this.token.equals(that.token))
         return false;
     }
 
@@ -637,6 +705,14 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     if (lastComparison != 0) {
       return lastComparison;
     }
+    lastComparison = Boolean.valueOf(isSetNodes()).compareTo(isSetNodes());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    lastComparison = TBaseHelper.compareTo(nodes, typedOther.nodes);
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
     lastComparison = Boolean.valueOf(isSetStartOffset()).compareTo(isSetStartOffset());
     if (lastComparison != 0) {
       return lastComparison;
@@ -645,11 +721,11 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     if (lastComparison != 0) {
       return lastComparison;
     }
-    lastComparison = Boolean.valueOf(isSetNodes()).compareTo(isSetNodes());
+    lastComparison = Boolean.valueOf(isSetToken()).compareTo(isSetToken());
     if (lastComparison != 0) {
       return lastComparison;
     }
-    lastComparison = TBaseHelper.compareTo(nodes, typedOther.nodes);
+    lastComparison = TBaseHelper.compareTo(token, typedOther.token);
     if (lastComparison != 0) {
       return lastComparison;
     }
@@ -701,14 +777,6 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
               TProtocolUtil.skip(iprot, field.type);
             }
             break;
-          case START_OFFSET:
-            if (field.type == TType.I64) {
-              this.startOffset = iprot.readI64();
-              setStartOffsetIsSet(true);
-            } else { 
-              TProtocolUtil.skip(iprot, field.type);
-            }
-            break;
           case NODES:
             if (field.type == TType.LIST) {
               {
@@ -727,6 +795,21 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
               TProtocolUtil.skip(iprot, field.type);
             }
             break;
+          case START_OFFSET:
+            if (field.type == TType.I64) {
+              this.startOffset = iprot.readI64();
+              setStartOffsetIsSet(true);
+            } else { 
+              TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
+          case TOKEN:
+            if (field.type == TType.STRING) {
+              this.token = iprot.readString();
+            } else { 
+              TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
         }
         iprot.readFieldEnd();
       }
@@ -770,6 +853,11 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     oprot.writeFieldBegin(START_OFFSET_FIELD_DESC);
     oprot.writeI64(this.startOffset);
     oprot.writeFieldEnd();
+    if (this.token != null) {
+      oprot.writeFieldBegin(TOKEN_FIELD_DESC);
+      oprot.writeString(this.token);
+      oprot.writeFieldEnd();
+    }
     oprot.writeFieldStop();
     oprot.writeStructEnd();
   }
@@ -799,15 +887,23 @@ public class Block implements TBase<Block._Fields>, java.io.Serializable, Clonea
     sb.append(this.genStamp);
     first = false;
     if (!first) sb.append(", ");
+    sb.append("nodes:");
+    if (this.nodes == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.nodes);
+    }
+    first = false;
+    if (!first) sb.append(", ");
     sb.append("startOffset:");
     sb.append(this.startOffset);
     first = false;
     if (!first) sb.append(", ");
-    sb.append("nodes:");
-    if (this.nodes == null) {
+    sb.append("token:");
+    if (this.token == null) {
       sb.append("null");
     } else {
-      sb.append(this.nodes);
+      sb.append(this.token);
     }
     first = false;
     sb.append(")");

File diff suppressed because it is too large
+ 32 - 1841
desktop/libs/hadoop/java/gen-java/org/apache/hadoop/thriftfs/api/Namenode.java


+ 7 - 18
desktop/libs/hadoop/java/if/hdfs.thrift

@@ -118,11 +118,14 @@ struct Block {
   /** Generational stamp of this block. */
   4: i64 genStamp,
 
-  /** Offset of the first byte of the block relative to the start of the file */  
-  6: i64 startOffset;
-
   /** List of data nodes with copies  of this block. */
   5: list<DatanodeInfo> nodes,
+
+  /** Offset of the first byte of the block relative to the start of the file */
+  6: i64 startOffset,
+
+  /** The serialized token associated with this block. */
+  7: string token,
 }
 
 /**
@@ -281,7 +284,7 @@ service Namenode extends common.HadoopServiceBase {
    *   (index 1) The total used space of the file system (in bytes).
    *   (index 2) The available storage of the file system (in bytes).
    */
-  list<i64> df(10: common.RequestContext ctx) throws (1: common.IOException err),
+  list<i64> df(10: common.RequestContext ctx),
 
   /**
    * Enter safe mode.
@@ -299,20 +302,6 @@ service Namenode extends common.HadoopServiceBase {
                         /** Length of the region */
                         3:  i64 length) throws (1: common.IOException err),
   
-  /** Get a report on the system's current data nodes. 
-      Note that ctx is currently ignored by the server. */
-
-  list<DatanodeInfo> getDatanodeReport(10: common.RequestContext ctx,
-                                       /**
-                                        * Type of data nodes to return
-                                        * information about.
-                                        */
-                                       1: DatanodeReportType type)
-                                          throws (1: common.IOException err),
-
-  /** Get a health report of DFS.  Note that ctx is ignored by the server. */
-  DFSHealthReport getHealthReport(10: common.RequestContext ctx) throws (1: common.IOException err),
-
   /**
    * Get the preferred block size for the given file.
    *

+ 293 - 210
desktop/libs/hadoop/java/src/java/org/apache/hadoop/mapred/ThriftJobTrackerPlugin.java

@@ -20,9 +20,10 @@ package org.apache.hadoop.mapred;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -41,6 +42,7 @@ import org.apache.hadoop.mapred.Counters.Counter;
 import org.apache.hadoop.mapred.Counters.Group;
 import org.apache.hadoop.mapred.JobTracker.State;
 import org.apache.hadoop.mapred.TaskStatus.Phase;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.thriftfs.ThriftHandlerBase;
 import org.apache.hadoop.thriftfs.ThriftPluginServer;
@@ -240,7 +242,6 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
             // could cause inconsistency between the values copied above and
             // the tasks themselves, but no deadlocks/CMEs.
             if (includeTasks) {
-                final boolean do_sort = true;
                 ret.setTasks(getInitialViewTaskList(job, tracker));
             }
             return ret;
@@ -281,7 +282,6 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
             return ret;
         }
 
-        @SuppressWarnings("deprecation")
         public static List<ThriftCounterGroup> toThrift(Counters jcs) {
           Collection<String> groupNames = null;
           List<ThriftCounterGroup> ret = null;
@@ -360,8 +360,8 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
             ttts.setLastSeen(t.getLastSeen());
             ttts.setMapCount(t.countMapTasks());
             ttts.setReduceCount(t.countReduceTasks());
-            ttts.setMaxMapTasks(t.getMaxMapTasks());
-            ttts.setMaxReduceTasks(t.getMaxReduceTasks());
+            ttts.setMaxMapTasks(t.getMaxMapSlots());
+            ttts.setMaxReduceTasks(t.getMaxReduceSlots());
 
             ttts.setTotalPhysicalMemory(t.getResourceStatus().getTotalPhysicalMemory());
             ttts.setTotalVirtualMemory(t.getResourceStatus().getTotalVirtualMemory());
@@ -486,12 +486,12 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
                                                         JobTracker jobTracker) {
             List<TaskInProgress> allTips = new ArrayList<TaskInProgress>();
             synchronized(job) {
-                allTips.addAll(Arrays.asList(job.getMapTasks()));
-                allTips.addAll(Arrays.asList(job.getReduceTasks()));
+                allTips.addAll(Arrays.asList(job.getTasks(TaskType.MAP)));
+                allTips.addAll(Arrays.asList(job.getTasks(TaskType.REDUCE)));
                 allTips.addAll(Arrays.asList(
-                      JTThriftUtils.sanitizeCleanupSetupTask(job.getCleanupTasks())));
+                      JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_CLEANUP))));
                 allTips.addAll(Arrays.asList(
-                      JTThriftUtils.sanitizeCleanupSetupTask(job.getSetupTasks())));
+                      JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_SETUP))));
             }
 
             // Sort by reverse time, but put all the genuine failures in front, and the
@@ -688,45 +688,52 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
 
         /** Returns the JobTracker's name */
         public String getJobTrackerName(RequestContext ctx) {
-            assumeUserContext(ctx);
-            return jobTracker.getJobTrackerMachine();
+          return assumeUserContextAndExecute(ctx, new PrivilegedAction<String>() {
+            public String run() {
+              return jobTracker.getJobTrackerMachine();
+            }
+          });
         }
 
         /** Returns a large clusterstatus object, augmented with some extra
          * detail from the JobTracker
          */
         public ThriftClusterStatus getClusterStatus(RequestContext ctx) {
-            assumeUserContext(ctx);
-            ClusterStatus cs = jobTracker.getClusterStatus(true);
-            return JTThriftUtils.toThrift(cs,jobTracker);
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftClusterStatus>() {
+              public ThriftClusterStatus run() {
+                ClusterStatus cs = jobTracker.getClusterStatus(true);
+                return JTThriftUtils.toThrift(cs,jobTracker);
+              }
+            });
         }
 
         /** Returns a list of all run-queues available to the JobTracker */
         public ThriftJobQueueList getQueues(RequestContext ctx) throws IOException, TException {
-            assumeUserContext(ctx);
-            JobQueueInfo queues[] = null;
-            try {
+            return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<ThriftJobQueueList>() {
+              public ThriftJobQueueList run() throws java.io.IOException {
+                JobQueueInfo queues[] = null;
                 queues = jobTracker.getQueues();
-            } catch (Throwable t) {
-                LOG.info("getQueues failed", t);
-                throw ThriftUtils.toThrift(t);
-            }
 
-            ArrayList<ThriftJobQueueInfo> ret =
-                new ArrayList<ThriftJobQueueInfo>(queues.length);
+                ArrayList<ThriftJobQueueInfo> ret =
+                    new ArrayList<ThriftJobQueueInfo>(queues.length);
 
-            for (JobQueueInfo q : queues) {
-                ThriftJobQueueInfo tq = JTThriftUtils.toThrift(q);
-                ret.add(tq);
-            }
-            return new ThriftJobQueueList(ret);
+                for (JobQueueInfo q : queues) {
+                    ThriftJobQueueInfo tq = JTThriftUtils.toThrift(q);
+                    ret.add(tq);
+                }
+                return new ThriftJobQueueList(ret);
+              }
+            });
         }
 
         /** Returns job by id (including task info) */
         public ThriftJobInProgress getJob(RequestContext ctx, ThriftJobID jobID) throws JobNotFoundException {
-            assumeUserContext(ctx);
-            JobID jid = JTThriftUtils.fromThrift(jobID);
-            JobInProgress job = jobTracker.getJob(jid);
+            final JobID jid = JTThriftUtils.fromThrift(jobID);
+            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(jid);
+              }
+            });
             if (job == null) {
               throw new JobNotFoundException();
             }
@@ -735,118 +742,137 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
 
         /** Returns all running jobs (does not include task info) */
         public ThriftJobList getRunningJobs(RequestContext ctx) {
-            assumeUserContext(ctx);
-            // Atomic copy
-            List<JobInProgress> jobs = jobTracker.getRunningJobs();
-            ArrayList<ThriftJobInProgress> ret =
-                new ArrayList<ThriftJobInProgress>(jobs.size());
-
-            for (JobInProgress job : jobs) {
-                ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
-            }
-            return new ThriftJobList(ret);
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
+              public ThriftJobList run() {
+                // Atomic copy
+                List<JobInProgress> jobs = jobTracker.getRunningJobs();
+                ArrayList<ThriftJobInProgress> ret =
+                    new ArrayList<ThriftJobInProgress>(jobs.size());
+
+                for (JobInProgress job : jobs) {
+                    ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+                }
+                return new ThriftJobList(ret);
+              }
+            });
         }
 
         /** Returns all completed jobs (does not include task info) */
         public ThriftJobList getCompletedJobs(RequestContext ctx) {
-            assumeUserContext(ctx);
-            List<JobInProgress> jobs = null;
-            synchronized(jobTracker){
-                jobs = jobTracker.completedJobs();
-            }
-            ArrayList<ThriftJobInProgress> ret =
-                new ArrayList<ThriftJobInProgress>(jobs.size());
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
+              public ThriftJobList run() {
+                List<JobInProgress> jobs = null;
+                synchronized(jobTracker){
+                    jobs = jobTracker.completedJobs();
+                }
+                ArrayList<ThriftJobInProgress> ret =
+                    new ArrayList<ThriftJobInProgress>(jobs.size());
 
-            for (JobInProgress job : jobs) {
-                ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
-            }
-            return new ThriftJobList(ret);
+                for (JobInProgress job : jobs) {
+                    ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+                }
+                return new ThriftJobList(ret);
+              }
+            });
         }
 
         /** Returns all failed jobs (does not include task info) */
         public ThriftJobList getFailedJobs(RequestContext ctx) {
-            assumeUserContext(ctx);
-            List<JobInProgress> jobs = null;
-            synchronized(jobTracker){
-                jobs = jobTracker.failedJobs();
-            }
-            List<ThriftJobInProgress> ret =
-                new ArrayList<ThriftJobInProgress>(jobs.size());
-            for (JobInProgress job : jobs) {
-                if (job.getStatus().getRunState() == JobStatus.FAILED) {
-                    ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
+              public ThriftJobList run() {
+                List<JobInProgress> jobs = null;
+                synchronized(jobTracker){
+                    jobs = jobTracker.failedJobs();
                 }
-            }
-            return new ThriftJobList(ret);
+                List<ThriftJobInProgress> ret =
+                    new ArrayList<ThriftJobInProgress>(jobs.size());
+                for (JobInProgress job : jobs) {
+                    if (job.getStatus().getRunState() == JobStatus.FAILED) {
+                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+                    }
+                }
+                return new ThriftJobList(ret);
+              }
+            });
         }
 
         /** Returns all killed jobs (does not include task info) */
         public ThriftJobList getKilledJobs(RequestContext ctx) {
-            assumeUserContext(ctx);
-            List<JobInProgress> jobs = null;
-            synchronized(jobTracker){
-                jobs = jobTracker.failedJobs();
-            }
-            List<ThriftJobInProgress> ret =
-                new ArrayList<ThriftJobInProgress>(jobs.size());
-            for (JobInProgress job : jobs) {
-                if (job.getStatus().getRunState() == JobStatus.KILLED) {
-                    ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
+              public ThriftJobList run() {
+                List<JobInProgress> jobs = null;
+                synchronized(jobTracker){
+                    jobs = jobTracker.failedJobs();
                 }
-            }
-            return new ThriftJobList(ret);
+                List<ThriftJobInProgress> ret =
+                    new ArrayList<ThriftJobInProgress>(jobs.size());
+                for (JobInProgress job : jobs) {
+                    if (job.getStatus().getRunState() == JobStatus.KILLED) {
+                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+                    }
+                }
+                return new ThriftJobList(ret);
+              }
+            });
         }
 
         /** Returns all running / failed / completed jobs (does not include task info) */
         public ThriftJobList getAllJobs(RequestContext ctx) {
-            assumeUserContext(ctx);
-            List<JobInProgress> jobList = new ArrayList<JobInProgress>();
-            jobList.addAll(jobTracker.getRunningJobs());
-            synchronized(jobTracker){
-                jobList.addAll(jobTracker.failedJobs());
-                jobList.addAll(jobTracker.completedJobs());
-            }
-            List<ThriftJobInProgress> ret =
-                new ArrayList<ThriftJobInProgress>();
-            for (JobInProgress job : jobList) {
-                    ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
+              public ThriftJobList run() {
+                List<JobInProgress> jobList = new ArrayList<JobInProgress>();
+                jobList.addAll(jobTracker.getRunningJobs());
+                synchronized(jobTracker){
+                    jobList.addAll(jobTracker.failedJobs());
+                    jobList.addAll(jobTracker.completedJobs());
                 }
-            return new ThriftJobList(ret);
+                List<ThriftJobInProgress> ret =
+                    new ArrayList<ThriftJobInProgress>();
+                for (JobInProgress job : jobList) {
+                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
+                    }
+                return new ThriftJobList(ret);
+              }
+            });
         }
 
         /**
          * Return the count of jobs, broken down by status, for a given user.
          */
-        public ThriftUserJobCounts getUserJobCounts(RequestContext ctx, String user) {
-            assumeUserContext(ctx);
-            ThriftUserJobCounts ret = new ThriftUserJobCounts(0, 0, 0, 0, 0);
-
-            JobStatus[] allJobs = jobTracker.getAllJobs();
-            for (JobStatus js : allJobs) {
-                if (!js.getUsername().equals(user))
-                    continue;
-                switch (js.getRunState()) {
-                    case JobStatus.PREP:
-                        ++ret.nPrep;
-                        break;
-                    case JobStatus.RUNNING:
-                        ++ret.nRunning;
-                        break;
-                    case JobStatus.SUCCEEDED:
-                        ++ret.nSucceeded;
-                        break;
-                    case JobStatus.FAILED:
-                        ++ret.nFailed;
-                        break;
-                    case JobStatus.KILLED:
-                        ++ret.nKilled;
-                        break;
-                    default:
-                        LOG.error("Unknown JobStatus " + js.getRunState() +
-                                  " for job id " + js.getJobID().getId());
+        public ThriftUserJobCounts getUserJobCounts(RequestContext ctx, final String user) {
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftUserJobCounts>() {
+              public ThriftUserJobCounts run() {
+                ThriftUserJobCounts ret = new ThriftUserJobCounts(0, 0, 0, 0, 0);
+
+                JobStatus[] allJobs = jobTracker.getAllJobs();
+                for (JobStatus js : allJobs) {
+                  System.out.println("jid: " + js.getJobId() + " js: " + js.getRunState());
+                    if (!js.getUsername().equals(user))
+                        continue;
+                    switch (js.getRunState()) {
+                        case JobStatus.PREP:
+                            ++ret.nPrep;
+                            break;
+                        case JobStatus.RUNNING:
+                            ++ret.nRunning;
+                            break;
+                        case JobStatus.SUCCEEDED:
+                            ++ret.nSucceeded;
+                            break;
+                        case JobStatus.FAILED:
+                            ++ret.nFailed;
+                            break;
+                        case JobStatus.KILLED:
+                            ++ret.nKilled;
+                            break;
+                        default:
+                            LOG.error("Unknown JobStatus " + js.getRunState() +
+                                      " for job id " + js.getJobID().getId());
+                    }
                 }
-            }
-            return ret;
+                return ret;
+              }
+            });
         }
 
         /**
@@ -860,25 +886,29 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
                                   String text,
                                   int count,
                                   int offset) throws JobNotFoundException {
-            assumeUserContext(ctx);
-            JobID jid = JTThriftUtils.fromThrift(thriftJobID);
-            JobInProgress job = jobTracker.getJob(jid);
+            final JobID jid = JTThriftUtils.fromThrift(thriftJobID);
+            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(jid);
+              }
+            });
+
             if (job == null)
-                throw new JobNotFoundException();
+              throw new JobNotFoundException();
 
             // Gather all the tasks of the matching type
             List<TaskInProgress> allTips = new ArrayList<TaskInProgress>();
             synchronized(job) {
                 if (types.contains(ThriftTaskType.MAP))
-                    allTips.addAll(Arrays.asList(job.getMapTasks()));
+                    allTips.addAll(Arrays.asList(job.getTasks(TaskType.MAP)));
                 if (types.contains(ThriftTaskType.REDUCE))
-                    allTips.addAll(Arrays.asList(job.getReduceTasks()));
+                    allTips.addAll(Arrays.asList(job.getTasks(TaskType.REDUCE)));
                 if (types.contains(ThriftTaskType.JOB_CLEANUP))
                     allTips.addAll(Arrays.asList(
-                          JTThriftUtils.sanitizeCleanupSetupTask(job.getCleanupTasks())));
+                          JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_CLEANUP))));
                 if (types.contains(ThriftTaskType.JOB_SETUP))
                     allTips.addAll(Arrays.asList(
-                          JTThriftUtils.sanitizeCleanupSetupTask(job.getSetupTasks())));
+                          JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_SETUP))));
             }
 
             // Are the arguments out of bound?
@@ -942,13 +972,20 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
         /** Returns the task identified by the id */
         public ThriftTaskInProgress getTask(RequestContext ctx, ThriftTaskID ttaskId)
                 throws JobNotFoundException, TaskNotFoundException {
-            assumeUserContext(ctx);
-            TaskID taskId = JTThriftUtils.fromThrift(ttaskId);
-            JobID jobId = JTThriftUtils.fromThrift(ttaskId.getJobID());
-            JobInProgress job = jobTracker.getJob(jobId);
+            final TaskID taskId = JTThriftUtils.fromThrift(ttaskId);
+            final JobID jobId = JTThriftUtils.fromThrift(ttaskId.getJobID());
+            final JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(jobId);
+              }
+            });
             if (job == null)
                 throw new JobNotFoundException();
-            TaskInProgress tip = job.getTaskInProgress(taskId);
+            TaskInProgress tip = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskInProgress>() {
+              public TaskInProgress run() {
+                return job.getTaskInProgress(taskId);
+              }
+            });
             if (tip == null)
                 throw new TaskNotFoundException();
             return JTThriftUtils.toThrift(tip, jobTracker);
@@ -956,22 +993,33 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
 
 
         /** Returns the set of counters associated with a given job */
-        public ThriftGroupList getJobCounters(RequestContext ctx, ThriftJobID jobID)
+        public ThriftGroupList getJobCounters(RequestContext ctx, final ThriftJobID jobID)
             throws JobNotFoundException
         {
-            assumeUserContext(ctx);
-            Counters jcs = jobTracker.getJobCounters(JTThriftUtils.fromThrift(jobID));
+            Counters jcs;
+            try {
+              jcs = assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Counters>() {
+                public Counters run() throws java.io.IOException {
+                    return jobTracker.getJobCounters(JTThriftUtils.fromThrift(jobID));
+                }
+              });
+            } catch (IOException e) {
+              throw new JobNotFoundException();
+            }
             if (jcs == null) {
                 throw new JobNotFoundException();
             }
             return new ThriftGroupList(JTThriftUtils.toThrift(jcs));
         }
 
-        public ThriftJobCounterRollups getJobCounterRollups(RequestContext ctx, ThriftJobID jobID)
+        public ThriftJobCounterRollups getJobCounterRollups(RequestContext ctx, final ThriftJobID jobID)
             throws JobNotFoundException
         {
-            assumeUserContext(ctx);
-            JobInProgress jip = jobTracker.getJob(JTThriftUtils.fromThrift(jobID));
+            JobInProgress jip = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(JTThriftUtils.fromThrift(jobID));
+              }
+            });
             if (jip == null) {
                 throw new JobNotFoundException();
             }
@@ -990,50 +1038,66 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
 
         /** Returns only active TaskTrackerStatus objects */
         public ThriftTaskTrackerStatusList getActiveTrackers(RequestContext ctx) {
-            assumeUserContext(ctx);
-            Collection<TaskTrackerStatus> active = jobTracker.activeTaskTrackers();
-            List<ThriftTaskTrackerStatus> trackers =
-                new ArrayList<ThriftTaskTrackerStatus>(active.size());
-            for (TaskTrackerStatus t : active) {
-                trackers.add(JTThriftUtils.toThrift(t));
-            }
-            return new ThriftTaskTrackerStatusList(trackers);
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
+              public ThriftTaskTrackerStatusList run() {
+                Collection<TaskTrackerStatus> active = jobTracker.activeTaskTrackers();
+                List<ThriftTaskTrackerStatus> trackers =
+                    new ArrayList<ThriftTaskTrackerStatus>(active.size());
+                for (TaskTrackerStatus t : active) {
+                    trackers.add(JTThriftUtils.toThrift(t));
+                }
+                return new ThriftTaskTrackerStatusList(trackers);
+              }
+            });
         }
 
         /** Returns only blacklisted TaskTrackerStatus objects */
         public ThriftTaskTrackerStatusList getBlacklistedTrackers(RequestContext ctx) {
-            assumeUserContext(ctx);
-            Collection<TaskTrackerStatus> black = jobTracker.blacklistedTaskTrackers();
-            List<ThriftTaskTrackerStatus> trackers =
-                new ArrayList<ThriftTaskTrackerStatus>(black.size());
-            for (TaskTrackerStatus t : black) {
-                trackers.add(JTThriftUtils.toThrift(t));
-            }
-            return new ThriftTaskTrackerStatusList(trackers);
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
+              public ThriftTaskTrackerStatusList run() {
+                Collection<TaskTrackerStatus> black = jobTracker.blacklistedTaskTrackers();
+                List<ThriftTaskTrackerStatus> trackers =
+                    new ArrayList<ThriftTaskTrackerStatus>(black.size());
+                for (TaskTrackerStatus t : black) {
+                    trackers.add(JTThriftUtils.toThrift(t));
+                }
+                return new ThriftTaskTrackerStatusList(trackers);
+              }
+            });
         }
 
         /** Returns all TaskTrackerStatus objects */
         public ThriftTaskTrackerStatusList getAllTrackers(RequestContext ctx) {
-            assumeUserContext(ctx);
-            Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
-            List<ThriftTaskTrackerStatus> trackers =
-                new ArrayList<ThriftTaskTrackerStatus>(all.size());
-            for (TaskTrackerStatus t : all) {
-                trackers.add(JTThriftUtils.toThrift(t));
-            }
-            return new ThriftTaskTrackerStatusList(trackers);
+            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
+              public ThriftTaskTrackerStatusList run() {
+                Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
+                List<ThriftTaskTrackerStatus> trackers =
+                    new ArrayList<ThriftTaskTrackerStatus>(all.size());
+                for (TaskTrackerStatus t : all) {
+                    trackers.add(JTThriftUtils.toThrift(t));
+                }
+                return new ThriftTaskTrackerStatusList(trackers);
+              }
+            });
         }
 
         /** Returns a single TaskTrackerStatus object by name */
-        public ThriftTaskTrackerStatus getTracker(RequestContext ctx, String name)
+        public ThriftTaskTrackerStatus getTracker(RequestContext ctx, final String name)
             throws TaskTrackerNotFoundException {
-            assumeUserContext(ctx);
-            Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
-            for (TaskTrackerStatus t : all) {
-                if (t.getTrackerName().equals(name))
-                    return JTThriftUtils.toThrift(t);
-            }
-            throw new TaskTrackerNotFoundException();
+            ThriftTaskTrackerStatus ret = assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatus>() {
+              public ThriftTaskTrackerStatus run() {
+                Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
+                for (TaskTrackerStatus t : all) {
+                    if (t.getTrackerName().equals(name))
+                        return JTThriftUtils.toThrift(t);
+                }
+                return null;
+              }
+            });
+            if (ret != null)
+              return ret;
+            else
+              throw new TaskTrackerNotFoundException();
         }
 
         /** Returns the current time in ms on this machine */
@@ -1043,22 +1107,21 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
         }
 
         /** Reads the local jobconf XML file for a given job */
-        public String getJobConfXML(RequestContext ctx, ThriftJobID jobID) throws IOException {
-            assumeUserContext(ctx);
-            /* This always returns a filename of hadoop.log.dir + "/" + jobid + "_conf.xml"
-             * Better check that jobid doesn't contain anything nasty.
-             */
-            JobID jid = JTThriftUtils.fromThrift(jobID);
-            String jidstring = jid.toString();
-            if (jidstring.contains(File.separator) || jidstring.contains(File.pathSeparator)) {
-                throw ThriftUtils.toThrift(
-                        new IllegalArgumentException("jobConf arguments can't contain path separators"));
-            }
-            String jobFilePath = JobTracker.getLocalJobFilePath(jid);
+        public String getJobConfXML(RequestContext ctx, final ThriftJobID jobID) throws IOException {
+            return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<String>() {
+              public String run() throws java.io.IOException {
+                /* This always returns a filename of hadoop.log.dir + "/" + jobid + "_conf.xml"
+                 * Better check that jobid doesn't contain anything nasty.
+                 */
+                JobID jid = JTThriftUtils.fromThrift(jobID);
+                String jidstring = jid.toString();
+                if (jidstring.contains(File.separator) || jidstring.contains(File.pathSeparator)) {
+                    throw new IllegalArgumentException("jobConf arguments can't contain path separators");
+                }
+                String jobFilePath = JobTracker.getLocalJobFilePath(jid);
 
-            StringBuffer fileData = new StringBuffer(1000);
-            BufferedReader reader;
-            try {
+                StringBuffer fileData = new StringBuffer(1000);
+                BufferedReader reader;
                 reader = new BufferedReader(
                         new FileReader(jobFilePath));
                 char[] buf = new char[1024];
@@ -1067,17 +1130,18 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
                     fileData.append(buf, 0, numRead);
                 }
                 reader.close();
-            } catch (Throwable t) {
-                LOG.error("getJobConfXML failed", t);
-                throw ThriftUtils.toThrift(t);
-            }
-            return fileData.toString();
+                return fileData.toString();
+              }
+            });
         }
 
         /** Kill a job by jobid */
-        public void killJob(RequestContext ctx, ThriftJobID jobID) throws IOException, JobNotFoundException {
-            assumeUserContext(ctx);
-            ThriftJobInProgress job = getJob(ctx, jobID);
+        public void killJob(final RequestContext ctx, final ThriftJobID jobID) throws IOException, JobNotFoundException {
+            ThriftJobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<ThriftJobInProgress>() {
+              public ThriftJobInProgress run() throws JobNotFoundException {
+                return getJob(ctx, jobID);
+              }
+            });
             if (job == null) {
                 throw new JobNotFoundException();
             }
@@ -1093,46 +1157,65 @@ public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configur
         /** Kill a task attempt by taskattemptid */
         public void killTaskAttempt(RequestContext ctx, ThriftTaskAttemptID attemptID)
             throws IOException, TaskAttemptNotFoundException, JobNotFoundException {
-            assumeUserContext(ctx);
-            TaskAttemptID taskid = JTThriftUtils.fromThrift(attemptID);
-            JobID jid = JTThriftUtils.fromThrift(attemptID.taskID.jobID);
-            JobInProgress job = jobTracker.getJob(jid);
+            final TaskAttemptID taskid = JTThriftUtils.fromThrift(attemptID);
+            final JobID jid = JTThriftUtils.fromThrift(attemptID.taskID.jobID);
+
+            final JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(jid);
+              }
+            });
             if (job == null) {
               throw new JobNotFoundException();
             }
-            TaskInProgress tip = job.getTaskInProgress(taskid.getTaskID());
+
+            final TaskInProgress tip = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskInProgress>() {
+              public TaskInProgress run() {
+                return job.getTaskInProgress(taskid.getTaskID());
+              }
+            });
             if (tip == null) {
                 throw new TaskAttemptNotFoundException();
             }
-            TaskStatus status = tip.getTaskStatus(taskid);
+
+            TaskStatus status = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskStatus>() {
+              public TaskStatus run() {
+                return tip.getTaskStatus(taskid);
+              }
+            });
             if (status == null) {
                 throw new TaskAttemptNotFoundException();
             }
-            try {
+
+            assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+              public Void run() throws java.io.IOException {
                 // Second parameter means always kill, don't fail
                 if (!jobTracker.killTask(taskid, true)) {
                     throw new RuntimeException();
                 }
-            } catch (Throwable t) {
-                LOG.info("killTask failed", t);
-                throw ThriftUtils.toThrift(t);
-            }
+                return null;
+              }
+            });
         }
 
         /** Set a job's priority */
-        public void setJobPriority(RequestContext ctx, ThriftJobID jobID, ThriftJobPriority priority)
+        public void setJobPriority(RequestContext ctx, final ThriftJobID jobID, final ThriftJobPriority priority)
             throws IOException, JobNotFoundException {
-            JobID jid = JTThriftUtils.fromThrift(jobID);
-            JobInProgress job = jobTracker.getJob(jid);
+            final JobID jid = JTThriftUtils.fromThrift(jobID);
+            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
+              public JobInProgress run() {
+                return jobTracker.getJob(jid);
+              }
+            });
             if (job == null) {
               throw new JobNotFoundException();
             }
-            try {
+            assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+              public Void run() throws java.io.IOException {
                 jobTracker.setJobPriority(jid, priority.toString());
-            } catch (Throwable t) {
-                LOG.info("setJobPriority failed", t);
-                throw ThriftUtils.toThrift(t);
-            }
+                return null;
+              }
+            });
         }
     }
 

+ 57 - 54
desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/DatanodePlugin.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.thriftfs;
 import java.io.EOFException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
+import java.security.PrivilegedExceptionAction;
 import java.util.zip.CRC32;
 
 import org.apache.commons.logging.Log;
@@ -27,16 +28,16 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.thriftfs.api.Block;
 import org.apache.hadoop.thriftfs.api.BlockData;
 import org.apache.hadoop.thriftfs.api.Datanode;
 import org.apache.hadoop.thriftfs.api.IOException;
 import org.apache.hadoop.thriftfs.api.Namenode;
 import org.apache.hadoop.thriftfs.api.RequestContext;
-import org.apache.hadoop.util.ServicePlugin;
 import org.apache.thrift.TException;
 import org.apache.thrift.TProcessor;
 import org.apache.thrift.TProcessorFactory;
@@ -81,62 +82,64 @@ public class DatanodePlugin
       this.summer = new CRC32();
     }
 
-    public BlockData readBlock(RequestContext ctx, Block block, long offset, int length)
-        throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("readBlock(" + block.blockId + "," + offset + "," + length
-          + "): Entering");
-
-      BlockData ret = new BlockData();
-      DFSClient.BlockReader reader = null;
-      try {
-        reader = DFSClient.BlockReader.newBlockReader(
-          getSocket(), block.path,
-          block.blockId, block.genStamp,
-          offset, length, bufferSize, true, serverContext.getClientName());
-        byte[] buf = new byte[length];
-        int n = reader.read(buf, 0, length);
-        if (n == -1) {
-          throw new EOFException("EOF reading " + length + " bytes at offset "
-              + offset + " from " + block);
-        }
-        LOG.debug("readBlock(" + block.blockId + ", " + offset + ", " + length
-            + "): Read " + n + " bytes");
-
-        if (n == length) {
-            // If we read exactly the same number of bytes that was asked for,
-            // we can simply return the buffer directly
-            ret.data = buf;
-        } else {
-            assert n < length;
-            // If we read fewer bytes than they asked for, we need to write
-            // back a smaller byte array. With the appropriate thrift hook
-            // we could avoid this copy, too.
-            ret.data = new byte[n];
-            System.arraycopy(buf, 0, ret.data, 0, n);
-        }
-        ret.length = n;
-
-        summer.update(ret.data);
-        ret.crc = (int) summer.getValue();
-        summer.reset();
-        LOG.debug("readBlock(" + block.blockId + ", " + offset + ", " + length
-            + "): CRC32: " + ret.crc);
-      } catch (Throwable t) {
-        LOG.info("readBlock(" + block.blockId + ", " + offset + ", " + length
-            + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      } finally {
-        if (reader != null) {
+    public BlockData readBlock(RequestContext ctx, final Block block, final long offset,
+        final int length) throws IOException, TException {
+      LOG.debug("readBlock(" + block.blockId + "," + offset + "," + length + "): Entering");
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<BlockData>() {
+        public BlockData run() throws IOException {
+
+          BlockData ret = new BlockData();
+          DFSClient.BlockReader reader = null;
           try {
-            reader.close();
+            Token<BlockTokenIdentifier> token = new Token<BlockTokenIdentifier>();
+            token.decodeFromUrlString(block.token);
+            reader = DFSClient.BlockReader.newBlockReader(getSocket(), block.path, block.blockId,
+                token, block.genStamp, offset, length, bufferSize, true, serverContext
+                    .getClientName());
+            byte[] buf = new byte[length];
+            int n = reader.read(buf, 0, length);
+            if (n == -1) {
+              throw new EOFException("EOF reading " + length + " bytes at offset " + offset
+                  + " from " + block);
+            }
+            LOG.debug("readBlock(" + block.blockId + ", " + offset + ", " + length + "): Read " + n
+                + " bytes");
+
+            if (n == length) {
+              // If we read exactly the same number of bytes that was asked for,
+              // we can simply return the buffer directly
+              ret.data = buf;
+            } else {
+              assert n < length;
+              // If we read fewer bytes than they asked for, we need to write
+              // back a smaller byte array. With the appropriate thrift hook
+              // we could avoid this copy, too.
+              ret.data = new byte[n];
+              System.arraycopy(buf, 0, ret.data, 0, n);
+            }
+            ret.length = n;
+
+            summer.update(ret.data);
+            ret.crc = (int) summer.getValue();
+            summer.reset();
+            LOG.debug("readBlock(" + block.blockId + ", " + offset + ", " + length + "): CRC32: "
+                + ret.crc);
           } catch (Throwable t) {
-            LOG.warn("readBlock(" + block.blockId + ", " + offset + ", "
-                + length + "): Cannot close block reader", t);
+            LOG.warn("readBlock(" + block.blockId + ", " + offset + ", " + length + "): Failed", t);
+            throw ThriftUtils.toThrift(t);
+          } finally {
+            if (reader != null) {
+              try {
+                reader.close();
+              } catch (Throwable t) {
+                LOG.warn("readBlock(" + block.blockId + ", " + offset + ", " + length
+                    + "): Cannot close block reader", t);
+              }
+            }
           }
+          return ret;
         }
-      }
-      return ret;
+      });
     }
 
     private Socket getSocket() throws java.io.IOException {

+ 232 - 349
desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/NamenodePlugin.java

@@ -19,7 +19,10 @@ package org.apache.hadoop.thriftfs;
 
 import java.io.FileNotFoundException;
 import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -28,37 +31,31 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.thriftfs.api.Block;
 import org.apache.hadoop.thriftfs.api.Constants;
 import org.apache.hadoop.thriftfs.api.ContentSummary;
-import org.apache.hadoop.thriftfs.api.DatanodeInfo;
-import org.apache.hadoop.thriftfs.api.DFSHealthReport;
 import org.apache.hadoop.thriftfs.api.IOException;
 import org.apache.hadoop.thriftfs.api.Namenode;
 import org.apache.hadoop.thriftfs.api.RequestContext;
 import org.apache.hadoop.thriftfs.api.Stat;
-import org.apache.hadoop.thriftfs.api.UpgradeStatusReport;
-import org.apache.hadoop.util.ServicePlugin;
 import org.apache.thrift.TException;
 import org.apache.thrift.TProcessor;
 import org.apache.thrift.TProcessorFactory;
 import org.apache.thrift.transport.TTransport;
 
-public class NamenodePlugin
-  extends org.apache.hadoop.hdfs.server.namenode.NamenodePlugin
-  implements Configurable {
+public class NamenodePlugin extends org.apache.hadoop.hdfs.server.namenode.NamenodePlugin implements
+    Configurable {
 
   /** Name of the configuration property of the Thrift server address */
   public static final String THRIFT_ADDRESS_PROPERTY = "dfs.thrift.address";
@@ -71,15 +68,13 @@ public class NamenodePlugin
 
   private NameNode namenode;
 
-  private static Map<DatanodeID, Integer> thriftPorts =
-      new HashMap<DatanodeID, Integer>();
+  private static Map<DatanodeID, Integer> thriftPorts = Collections.synchronizedMap(new HashMap<DatanodeID, Integer>());
 
   static final Log LOG = LogFactory.getLog(NamenodePlugin.class);
 
   private Configuration conf;
   private ThriftPluginServer thriftServer;
 
-
   /** Java server-side implementation of the 'Namenode' Thrift interface. */
   class ThriftHandler extends ThriftHandlerBase implements Namenode.Iface {
 
@@ -87,284 +82,186 @@ public class NamenodePlugin
       super(context);
     }
 
-    public void chmod(RequestContext ctx, String path, short mode) throws IOException, TException {
-      assumeUserContext(ctx);
+    public void chmod(RequestContext ctx, final String path, final short mode) throws IOException {
       LOG.debug("chmod(" + path + ", " + mode + "): Entering");
-      try {
-        namenode.setPermission(path, new FsPermission(mode));
-      } catch (Throwable t) {
-        LOG.info("chmod(" + path + ", " + mode + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
-    }
-
-    public void chown(RequestContext ctx, String path, String owner, String group)
-        throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("chown(" + path + "," + owner + "," + group + "): Entering");
-      try {
-        // XXX Looks like namenode.setOwner() does not complain about this...
-        if (owner == null && group == null) {
-          throw new IllegalArgumentException(
-              "Both 'owner' and 'group' are null");
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          namenode.setPermission(path, new FsPermission(mode));
+          return null;
         }
-        namenode.setOwner(path, owner, group);
-      } catch (Throwable t) {
-        LOG.info("chown(" + path + "," + owner + "," + group + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      });
     }
 
-    public List<Long> df(RequestContext ctx) throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("Entering df()");
-      try {
-        long[] stats = namenode.getStats();
-        List<Long> ret = new ArrayList<Long>();
-        // capacityTotal
-        ret.add(stats[0]);
-        // capacityUsed
-        ret.add(stats[1]);
-        // capacityRemaining
-        ret.add(stats[2]);
-        LOG.debug("df(): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("df(): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
-    }
-
-    public void enterSafeMode(RequestContext ctx) throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("enterSafeMode(): Entering");
-      try {
-        namenode.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
-      } catch (Throwable t) {
-        LOG.info("enterSafeMode(): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
-    }
-
-    public List<Block> getBlocks(RequestContext ctx, String path, long offset, long length)
-        throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("getBlocks(" + path + "," + offset + "," + length
-          + "): Entering");
-      List<Block> ret = new ArrayList<Block>();
-      try {
-        LocatedBlocks blocks = namenode.getBlockLocations(path, offset, length);
-        if (blocks != null) {
-          // blocks may be null if offset is past the end of the file
-          for (LocatedBlock b : blocks.getLocatedBlocks()) {
-            ret.add(ThriftUtils.toThrift(b, path, thriftPorts));
+    public void chown(RequestContext ctx, final String path, final String owner, final String group)
+        throws IOException {
+      LOG.debug("chown(" + path + "," + owner + "," + group + "): Entering");
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          // XXX Looks like namenode.setOwner() does not complain about
+          // this...
+          if (owner == null && group == null) {
+            throw new IllegalArgumentException("Both 'owner' and 'group' are null");
           }
+          namenode.setOwner(path, owner, group);
+          return null;
         }
-        LOG.debug("getBlocks(" + path + "," + offset + "," + length
-            + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("getBlocks(" + path + "," + offset + "," + length
-            + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      });
     }
 
-    public DFSHealthReport getHealthReport(RequestContext ctx) throws IOException {
-      // Health report can only be run by the superuser,
-      // but is always available.  Therefore, we ignore ctx.
-      assumeSuperuserContext();
-
-      DFSHealthReport hr = new DFSHealthReport();
-      try {
-        long[] fsnStats = namenode.getStats();
-        hr.bytesTotal = fsnStats[0];
-        hr.bytesRemaining = fsnStats[2];
-        hr.bytesUsed = fsnStats[1];
-        hr.bytesNonDfs = hr.bytesTotal - hr.bytesRemaining - hr.bytesUsed;
-
-        ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
-        ArrayList<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
-
-        namenode.getNamesystem().DFSNodesStatus(live, dead);
-
-        hr.numLiveDataNodes = live.size();
-        hr.numDeadDataNodes = dead.size();
-
-        org.apache.hadoop.hdfs.server.common.UpgradeStatusReport usr =
-          namenode.distributedUpgradeProgress(FSConstants.UpgradeAction.DETAILED_STATUS);
-        if (usr != null) {
-          hr.upgradeStatus = new UpgradeStatusReport();
-          hr.upgradeStatus.version = usr.getVersion();
-          hr.upgradeStatus.percentComplete = usr.getUpgradeStatus();
-          hr.upgradeStatus.finalized = usr.isFinalized();
-          hr.upgradeStatus.statusText = usr.getStatusText(true);
+    public List<Long> df(RequestContext ctx) {
+      LOG.debug("Entering df()");
+      return assumeUserContextAndExecute(ctx, new PrivilegedAction<List<Long>>() {
+        public List<Long> run() {
+          long[] stats = namenode.getStats();
+          List<Long> ret = new ArrayList<Long>();
+          // capacityTotal
+          ret.add(stats[0]);
+          // capacityUsed
+          ret.add(stats[1]);
+          // capacityRemaining
+          ret.add(stats[2]);
+          LOG.debug("df(): Returning " + ret);
+          return ret;
         }
-
-        hr.httpPort = namenode.getHttpAddress().getPort();
-      } catch (java.io.IOException ioe) {
-        LOG.info("getHealthReport() failed", ioe);
-        throw ThriftUtils.toThrift(ioe);
-      }
-      return hr;
+      });
     }
 
-    public List<DatanodeInfo> getDatanodeReport(
-      RequestContext ctx, org.apache.hadoop.thriftfs.api.DatanodeReportType type)
-        throws IOException, TException
-    {
-      // Datanode report can only be run by the superuser, therefore, we ignore 
-      // ctx.
-      assumeSuperuserContext();
-
-      LOG.debug("getDatanodeReport(" + type + "): Entering");
-      List<DatanodeInfo> ret = new ArrayList<DatanodeInfo>();
-      try {
-        FSConstants.DatanodeReportType rt;
-        switch (type) {
-        case ALL_DATANODES:
-          rt = FSConstants.DatanodeReportType.ALL;
-          break;
-        case DEAD_DATANODES:
-          rt = FSConstants.DatanodeReportType.DEAD;
-          break;
-        case LIVE_DATANODES:
-          rt = FSConstants.DatanodeReportType.LIVE;
-          break;
-        default:
-          throw new IllegalArgumentException("Invalid report type " + type);
+    public void enterSafeMode(RequestContext ctx) throws IOException {
+      LOG.debug("enterSafeMode(): Entering");
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          namenode.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+          return null;
         }
-        for (org.apache.hadoop.hdfs.protocol.DatanodeInfo node : namenode
-            .getDatanodeReport(rt)) {
-          ret.add(ThriftUtils.toThrift(node, thriftPorts));
+      });
+    }
+
+    public List<Block> getBlocks(RequestContext ctx, final String path, final long offset,
+        final long length) throws IOException {
+      LOG.debug("getBlocks(" + path + "," + offset + "," + length + "): Entering");
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<List<Block>>() {
+        public List<Block> run() throws java.io.IOException {
+          List<Block> ret = new ArrayList<Block>();
+          LocatedBlocks blocks = namenode.getBlockLocations(path, offset, length);
+          if (blocks != null) {
+            // blocks may be null if offset is past the end of the file
+            for (LocatedBlock b : blocks.getLocatedBlocks()) {
+              ret.add(ThriftUtils.toThrift(b, path, thriftPorts));
+            }
+          }
+          LOG.debug("getBlocks(" + path + "," + offset + "," + length + "): Returning " + ret);
+          return ret;
         }
-        LOG.debug("getDatanodeReport(" + type + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("getDatanodeReport(" + type + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      });
     }
 
-    public long getPreferredBlockSize(RequestContext ctx, String path) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public long getPreferredBlockSize(RequestContext ctx, final String path) throws IOException {
       LOG.debug("getPreferredBlockSize(" + path + "): Entering");
-      try {
-        long ret = namenode.getPreferredBlockSize(path);
-        LOG.debug("getPreferredBlockSize(" + path + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("getPreferredBlockSize(" + path + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Long>() {
+        public Long run() throws java.io.IOException {
+          long ret = namenode.getPreferredBlockSize(path);
+          LOG.debug("getPreferredBlockSize(" + path + "): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public boolean isInSafeMode(RequestContext ctx) throws IOException, TException {
-      assumeUserContext(ctx);
+    public boolean isInSafeMode(RequestContext ctx) throws IOException {
       LOG.debug("isInSafeMode(): Entering");
-      try {
-        boolean ret = namenode.setSafeMode(SafeModeAction.SAFEMODE_GET);
-        LOG.debug("isInSafeMode(): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("isInSafeMode(): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Boolean>() {
+        public Boolean run() throws java.io.IOException {
+          boolean ret = namenode.setSafeMode(SafeModeAction.SAFEMODE_GET);
+          LOG.debug("isInSafeMode(): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public void leaveSafeMode(RequestContext ctx) throws IOException, TException {
-      assumeUserContext(ctx);
+    public void leaveSafeMode(RequestContext ctx) throws IOException {
       LOG.debug("leaveSafeMode(): Entering");
-      try {
-        namenode.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
-      } catch (Throwable t) {
-        LOG.info("leaveSafeMode(): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          namenode.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+          return null;
+        }
+      });
     }
 
-    public List<Stat> ls(RequestContext ctx, String path) throws IOException, TException {
-      assumeUserContext(ctx);
+    public List<Stat> ls(RequestContext ctx, final String path) throws IOException {
       LOG.debug("ls(" + path + "):Entering");
-      List<Stat> ret = new ArrayList<Stat>();
-      try {
-        FileStatus[] listing = namenode.getListing(path);
-        if (listing == null) {
-          throw new FileNotFoundException("Not found: " + path);
-        }
-        for (FileStatus f : listing) {
-          ret.add(fileStatusToStat(f));
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<List<Stat>>() {
+        public List<Stat> run() throws java.io.IOException {
+          List<Stat> ret = new ArrayList<Stat>();
+          byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
+          DirectoryListing listing;
+          do {
+            listing = namenode.getListing(path, lastReturnedName);
+            if (listing == null) {
+              throw new FileNotFoundException("Not found: " + path);
+            }
+            for (HdfsFileStatus f : listing.getPartialListing()) {
+              ret.add(fileStatusToStat(f, path));
+            }
+            lastReturnedName = listing.getLastName();
+          } while (listing.hasMore());
+          LOG.debug("ls(" + path + "): Returning " + ret);
+          return ret;
         }
-        LOG.debug("ls(" + path + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("ls(" + path + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      });
     }
 
-    public boolean mkdirhier(RequestContext ctx, String path, short perms) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public boolean mkdirhier(RequestContext ctx, final String path, final short perms)
+        throws IOException {
       LOG.debug("mkdirhier(" + path + ", " + perms + "): Entering");
-      try {
-        boolean ret = namenode.mkdirs(path, new FsPermission(perms));
-        LOG.debug("mkdirhier(" + path + ", " + perms + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("mkdirhier(" + path + ", " + perms + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Boolean>() {
+        public Boolean run() throws java.io.IOException {
+          boolean ret = namenode.mkdirs(path, new FsPermission(perms));
+          LOG.debug("mkdirhier(" + path + ", " + perms + "): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public void refreshNodes(RequestContext ctx) throws IOException, TException {
-      assumeUserContext(ctx);
+    public void refreshNodes(RequestContext ctx) throws IOException {
       LOG.debug("refreshNodes(): Entering");
-      try {
-        namenode.refreshNodes();
-      } catch (Throwable t) {
-        LOG.info("refreshNodes(): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          namenode.refreshNodes();
+          return null;
+        }
+      });
     }
 
-    public boolean rename(RequestContext ctx, String path, String newPath) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public boolean rename(RequestContext ctx, final String path, final String newPath)
+        throws IOException {
       LOG.debug("rename(" + path + ", " + newPath + "): Entering");
-      try {
-        boolean ret = namenode.rename(path, newPath);
-        LOG.debug("rename(" + path + ", " + newPath + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("rename(" + path + ", " + newPath + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Boolean>() {
+        public Boolean run() throws java.io.IOException {
+          boolean ret = namenode.rename(path, newPath);
+          LOG.debug("rename(" + path + ", " + newPath + "): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public void reportBadBlocks(RequestContext ctx, List<Block> blocks) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public void reportBadBlocks(RequestContext ctx, final List<Block> blocks) throws IOException {
       LOG.debug("reportBadBlocks(" + blocks + "): Entering");
-      int n = blocks.size();
-      LocatedBlock[] lb = new LocatedBlock[n];
-      for (int i = 0; i < n; ++i) {
-        lb[i] = ThriftUtils.fromThrift(blocks.get(i));
-      }
-      try {
-        namenode.reportBadBlocks(lb);
-      } catch (Throwable t) {
-        LOG.info("reportBadBlocks(" + blocks + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          int n = blocks.size();
+          LocatedBlock[] lb = new LocatedBlock[n];
+          for (int i = 0; i < n; ++i) {
+            lb[i] = ThriftUtils.fromThrift(blocks.get(i));
+          }
+          namenode.reportBadBlocks(lb);
+          return null;
+        }
+      });
     }
 
-    public void setQuota(RequestContext ctx, String path, long namespaceQuota, long diskspaceQuota)
-        throws IOException, TException {
-      assumeUserContext(ctx);
-      LOG.debug("setQuota(" + path + "," + namespaceQuota + ","
-          + diskspaceQuota + "): Entering");
+    public void setQuota(RequestContext ctx, final String path, long namespaceQuota,
+        long diskspaceQuota) throws IOException {
+      LOG.debug("setQuota(" + path + "," + namespaceQuota + "," + diskspaceQuota + "): Entering");
       if (namespaceQuota == Constants.QUOTA_DONT_SET) {
         namespaceQuota = FSConstants.QUOTA_DONT_SET;
       }
@@ -377,144 +274,131 @@ public class NamenodePlugin
       if (diskspaceQuota == Constants.QUOTA_RESET) {
         diskspaceQuota = FSConstants.QUOTA_RESET;
       }
-      LOG.debug("setQuota(" + path + "," + namespaceQuota + ","
-          + diskspaceQuota + "): Quota values translated");
-      try {
-        namenode.setQuota(path, namespaceQuota, diskspaceQuota);
-      } catch (Throwable t) {
-        LOG.info("setQuota(" + path + "," + namespaceQuota + ","
-            + diskspaceQuota + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      final long finalNamespaceQuota = namespaceQuota;
+      final long finalDiskspaceQuota = diskspaceQuota;
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          LOG.debug("setQuota(" + path + "," + finalNamespaceQuota + "," + finalDiskspaceQuota
+              + "): Quota values translated");
+          namenode.setQuota(path, finalNamespaceQuota, finalDiskspaceQuota);
+          return null;
+        }
+      });
     }
 
-    public boolean setReplication(RequestContext ctx, String path, short repl) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public boolean setReplication(RequestContext ctx, final String path, final short repl)
+        throws IOException {
       LOG.debug("setReplication(" + path + "," + repl + "): Entering");
-      try {
-        return namenode.setReplication(path, repl);
-      } catch (Throwable t) {
-        LOG.info("setReplication(" + path + "," + repl + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Boolean>() {
+        public Boolean run() throws java.io.IOException {
+          return namenode.setReplication(path, repl);
+        }
+      });
     }
 
-    public Stat stat(RequestContext ctx, String path) throws IOException, TException {
-      assumeUserContext(ctx);
+    public Stat stat(RequestContext ctx, final String path) throws IOException {
       LOG.debug("stat(" + path + "): Entering");
-      try {
-        Stat ret = fileStatusToStat(namenode.getFileInfo(path));
-        LOG.debug("stat(" + path + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("stat(" + path + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Stat>() {
+        public Stat run() throws java.io.IOException {
+          Stat ret = fileStatusToStat(namenode.getFileInfo(path), path);
+          LOG.debug("stat(" + path + "): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public ContentSummary getContentSummary(RequestContext ctx, String path)
-      throws IOException, TException {
-      assumeUserContext(ctx);
+    public ContentSummary getContentSummary(RequestContext ctx, final String path)
+        throws IOException {
       LOG.debug("getContentSummary(" + path + "): Entering");
-      try {
-        ContentSummary cs = getContentSummary(path);
-        LOG.debug("getContentSummary(" + path + "): Returning " + cs);
-        return cs;
-      } catch (Throwable t) {
-        LOG.info("getContentSummary(" + path + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<ContentSummary>() {
+        public ContentSummary run() throws java.io.IOException {
+          ContentSummary cs = getContentSummary(path);
+          LOG.debug("getContentSummary(" + path + "): Returning " + cs);
+          return cs;
+        }
+      });
     }
 
-    public List<ContentSummary> multiGetContentSummary(RequestContext ctx, List<String> paths)
-        throws IOException, TException {
-        assumeUserContext(ctx);
-        LOG.debug("multiGetContentSummary(" + paths + "): Entering");
-        List<ContentSummary> ret = new ArrayList<ContentSummary>();
-        try {
-            for (String path : paths) {
+    public List<ContentSummary> multiGetContentSummary(RequestContext ctx, final List<String> paths)
+        throws IOException {
+      LOG.debug("multiGetContentSummary(" + paths + "): Entering");
+      return assumeUserContextAndExecute(ctx,
+          new PrivilegedExceptionAction<List<ContentSummary>>() {
+            public List<ContentSummary> run() throws java.io.IOException {
+              List<ContentSummary> ret = new ArrayList<ContentSummary>();
+              for (String path : paths) {
                 ret.add(getContentSummary(path));
+              }
+              LOG.debug("multiGetContentSummary(" + paths + "): Returning " + ret);
+              return ret;
             }
-        } catch (Throwable t) {
-            LOG.info("multiGetContentSummary(" + paths + "): Failed", t);
-            throw ThriftUtils.toThrift(t);
-        }
-        LOG.debug("multiGetContentSummary(" + paths + "): Returning " + ret);
-        return ret;
+          });
     }
 
     private ContentSummary getContentSummary(String path) throws java.io.IOException {
-        try {
-            return ThriftUtils.toThrift(namenode.getContentSummary(path), path);
-        } catch (java.io.IOException e) {
-            LOG.error(e);
-            throw e;
-        }
+      return ThriftUtils.toThrift(namenode.getContentSummary(path), path);
     }
 
-    public boolean unlink(RequestContext ctx, String path, boolean recursive) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public boolean unlink(RequestContext ctx, final String path, final boolean recursive)
+        throws IOException {
       LOG.debug("unlink(" + path + "," + recursive + "): Entering");
-      try {
-        boolean ret = namenode.delete(path, recursive);
-        LOG.debug("unlink(" + path + "," + recursive + "): Returning " + ret);
-        return ret;
-      } catch (Throwable t) {
-        LOG.info("unlink(" + path + "," + recursive + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Boolean>() {
+        public Boolean run() throws java.io.IOException, TException {
+          boolean ret = namenode.delete(path, recursive);
+          LOG.debug("unlink(" + path + "," + recursive + "): Returning " + ret);
+          return ret;
+        }
+      });
     }
 
-    public void utime(RequestContext ctx, String path, long atime, long mtime) throws IOException,
-        TException {
-      assumeUserContext(ctx);
+    public void utime(RequestContext ctx, final String path, final long atime, final long mtime)
+        throws IOException {
       LOG.debug("utime(" + path + "," + atime + "," + mtime + "): Entering");
-      if (mtime == -1 && atime == -1) {
-        LOG.debug("utime(" + path + "," + atime + "," + mtime
-            + "): Setting mtime and atime to now");
-        mtime = atime = System.currentTimeMillis();
-      }
-      try {
-        namenode.setTimes(path, mtime, atime);
-      } catch (Throwable t) {
-        LOG.info("utime(" + path + "," + atime + "," + mtime + "): Failed", t);
-        throw ThriftUtils.toThrift(t);
-      }
+      assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
+        public Void run() throws java.io.IOException {
+          if (mtime == -1 && atime == -1) {
+            LOG.debug("utime(" + path + "," + atime + "," + mtime
+                + "): Setting mtime and atime to now");
+            long now = System.currentTimeMillis();
+            namenode.setTimes(path, now, now);
+          } else {
+            namenode.setTimes(path, mtime, atime);
+          }
+          return null;
+        }
+      });
     }
 
-    private Stat fileStatusToStat(FileStatus f) throws java.io.IOException {
+    private Stat fileStatusToStat(HdfsFileStatus f, String parentPath) throws java.io.IOException {
       if (f == null) {
         throw new FileNotFoundException();
       }
 
       Stat st = new Stat();
-      st.path = f.getPath().toString();
+      st.path = f.getFullPath(new Path(parentPath)).toString();
       st.isDir = f.isDir();
       st.atime = f.getAccessTime();
       st.mtime = f.getModificationTime();
       st.perms = f.getPermission().toShort();
       st.owner = f.getOwner();
       st.group = f.getGroup();
-      if (! st.isDir) {
+      if (!st.isDir) {
         st.length = f.getLen();
         st.blockSize = f.getBlockSize();
         st.replication = f.getReplication();
       }
       return st;
     }
-    public void datanodeDown(String name, String storage, int thriftPort) throws TException {
+
+    public void datanodeDown(String name, String storage, int thriftPort) {
       DatanodeID dnId = new DatanodeID(name, storage, -1, -1);
-      LOG.info("Datanode " + dnId + ": Thrift port "
-               + thriftPort + " closed");
+      LOG.info("Datanode " + dnId + ": Thrift port " + thriftPort + " closed");
       thriftPorts.remove(dnId);
     }
 
-    public void datanodeUp(String name, String storage, int thriftPort) throws TException {
+    public void datanodeUp(String name, String storage, int thriftPort) {
       DatanodeID dnId = new DatanodeID(name, storage, -1, -1);
-      LOG.info("Datanode " + dnId + ": " +
-               "Thrift port " + thriftPort + " open");
+      LOG.info("Datanode " + dnId + ": " + "Thrift port " + thriftPort + " open");
       thriftPorts.put(dnId, thriftPort);
     }
   }
@@ -524,17 +408,16 @@ public class NamenodePlugin
 
   @Override
   public void start(Object service) {
-    this.namenode = (NameNode)service;
+    this.namenode = (NameNode) service;
     try {
-      InetSocketAddress address = NetUtils.createSocketAddr(
-        conf.get(THRIFT_ADDRESS_PROPERTY, DEFAULT_THRIFT_ADDRESS));
+      InetSocketAddress address = NetUtils.createSocketAddr(conf.get(THRIFT_ADDRESS_PROPERTY,
+          DEFAULT_THRIFT_ADDRESS));
 
       this.thriftServer = new ThriftPluginServer(address, new ProcessorFactory());
       thriftServer.setConf(conf);
       thriftServer.start();
       // The port may have been 0, so we update it.
-      conf.set(THRIFT_ADDRESS_PROPERTY, address.getHostName() + ":" + 
-          thriftServer.getPort());
+      conf.set(THRIFT_ADDRESS_PROPERTY, address.getHostName() + ":" + thriftServer.getPort());
     } catch (java.io.IOException ioe) {
       throw new RuntimeException("Cannot start Thrift namenode plug-in", ioe);
     }

+ 25 - 45
desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftHandlerBase.java

@@ -18,28 +18,23 @@
 
 package org.apache.hadoop.thriftfs;
 
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
-import java.util.StringTokenizer;
 import java.util.Map;
-import javax.security.auth.login.LoginException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.spi.OutputRecord;
 
 import org.apache.hadoop.thriftfs.api.HadoopServiceBase;
+import org.apache.hadoop.thriftfs.api.IOException;
 import org.apache.hadoop.thriftfs.api.MetricsContext;
 import org.apache.hadoop.thriftfs.api.MetricsRecord;
 import org.apache.hadoop.thriftfs.api.RequestContext;
@@ -96,7 +91,7 @@ public abstract class ThriftHandlerBase implements HadoopServiceBase.Iface {
       for (org.apache.hadoop.metrics.MetricsContext ctx : allContexts) {
         ret.add(metricsContextToThrift(ctx));
       }
-    } catch (IOException ioe) {
+    } catch (java.io.IOException ioe) {
       LOG.warn("getAllMetrics() failed", ioe);
       throw ThriftUtils.toThrift(ioe);
     }
@@ -185,48 +180,33 @@ public abstract class ThriftHandlerBase implements HadoopServiceBase.Iface {
     return dump;
   }
 
-
   /**
-   * Should be called by all RPCs on the request context passed in.
-   * This assumes the authentication role of the requester.
+   * The methods below should be called by all RPCs with the request context
+   * passed in, whenever said RPCs are accessing Hadoop-internal methods. These
+   * assume the authentication role of the requester.
+   *
+   * Most of the time you can just wrap the entire contents of the method with
+   * these methods. If, however, your RPC needs to throw an exception not of
+   * type IOException, then you may need to wrap only the portions which
+   * actually touch Hadoop, and then throw your own exception(s) based on the
+   * result of these calls.
    */
-  protected void assumeUserContext(RequestContext ctx) {
-    UserGroupInformation ugi = null;
-    if (ctx != null && ctx.confOptions != null) {
-      Configuration conf = new Configuration(false);
-      
-      for (Map.Entry<String, String> entry : ctx.confOptions.entrySet()) {
-        conf.set(entry.getKey(), entry.getValue());
-      }
-      // UnixUserGroupInformation.readFromConf() caches UGIs based on
-      // on username, so it's not usable if the group list
-      // may be changing.
-      String[] ugiparts = conf.getStrings(UnixUserGroupInformation.UGI_PROPERTY_NAME);
-      ugi = new UnixUserGroupInformation(ugiparts);
-    } else {
-      LOG.warn("Did not receive UGI information with call.");
-      // For backwards-compatibility, continue, though this really
-      // should be an error.
-      ugi = new UnixUserGroupInformation(new String[] { "nobody", "nogroup" });
+  protected <T> T assumeUserContextAndExecute(RequestContext ctx, PrivilegedExceptionAction<T> action) throws IOException {
+    try {
+      return UserGroupInformation.createProxyUser(ctx.confOptions.get("effective_user"), UserGroupInformation.getLoginUser()).doAs(action);
+    } catch (Throwable e) {
+      throw ThriftUtils.toThrift(e);
     }
-    UserGroupInformation.setCurrentUser(ugi);
-    LOG.info("Connection from user " + ugi);
   }
 
-  /**
-   * Take on the credentials of the current process, which,
-   * since this is running inside of a Hadoop daemon, is
-   * the cluster's superuser.  If for some reason this
-   * fails, this will return nobody/nogroup.
-   */
-  protected void assumeSuperuserContext() {
-    UserGroupInformation ugi = null;
+  protected <T> T assumeUserContextAndExecute(RequestContext ctx, PrivilegedAction<T> action) {
     try {
-      ugi = UnixUserGroupInformation.login();
-    } catch (LoginException e) {
-      LOG.info("Cannot get current UNIX user: " + e.getMessage());
-      ugi = new UnixUserGroupInformation(new String[] { "nobody", "nogroup" });
+      return UserGroupInformation.createProxyUser(ctx.confOptions.get("effective_user"),
+          UserGroupInformation.getLoginUser()).doAs(action);
+    } catch (java.io.IOException e) {
+      // This should only be thrown in the event getLoginUser() fails.
+      throw new Error(e);
     }
-    UserGroupInformation.setCurrentUser(ugi);
   }
+
 }

+ 0 - 7
desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftServerContext.java

@@ -17,18 +17,11 @@
  */
 package org.apache.hadoop.thriftfs;
 
-import java.io.OutputStream;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.util.Random;
-import java.util.StringTokenizer;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.LineReader;
 import org.apache.thrift.transport.TTransport;
 import org.apache.thrift.transport.TSocket;
 

+ 2 - 3
desktop/libs/hadoop/java/src/java/org/apache/hadoop/thriftfs/ThriftUtils.java

@@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.thriftfs.api.Block;
 import org.apache.hadoop.thriftfs.api.Constants;
@@ -65,7 +64,7 @@ public class ThriftUtils {
   }
 
   public static Block toThrift(LocatedBlock block, String path,
-      Map<DatanodeID, Integer> thriftPorts) {
+      Map<DatanodeID, Integer> thriftPorts) throws java.io.IOException {
     if (block == null) {
       return new Block();
     }
@@ -80,7 +79,7 @@ public class ThriftUtils {
 
     org.apache.hadoop.hdfs.protocol.Block b = block.getBlock();
     return new Block(b.getBlockId(), path, b.getNumBytes(),
-                     b.getGenerationStamp(), block.getStartOffset(), nodes);
+                     b.getGenerationStamp(), nodes, block.getStartOffset(), block.getBlockToken().encodeToUrlString());
   }
 
   public static ContentSummary toThrift(org.apache.hadoop.fs.ContentSummary cs, String path) {

+ 11 - 16
desktop/libs/hadoop/java/src/test/org/apache/hadoop/thriftfs/Helper.java

@@ -20,10 +20,11 @@ package org.apache.hadoop.thriftfs;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -34,11 +35,11 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
-import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.thriftfs.api.Datanode;
 import org.apache.hadoop.thriftfs.api.DatanodeInfo;
 import org.apache.hadoop.thriftfs.api.RequestContext;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.transport.TSocket;
@@ -72,31 +73,25 @@ public class Helper {
     return conf;
   }
 
-  public static RequestContext createRequestContext(boolean superuser) {
+  public static RequestContext createRequestContext(boolean superuser) throws IOException {
     RequestContext ctx = new RequestContext();
     Configuration conf = new Configuration();
 
-    UnixUserGroupInformation ugi;
+    UserGroupInformation ugi;
     if (superuser) {
-      try {
-        ugi = UnixUserGroupInformation.login();
-      } catch (LoginException le) {
-        // we need to be able to determine the superuser for these
-        // tests to work
-        throw new RuntimeException(le);
-      }
+      ugi = UserGroupInformation.getCurrentUser();
     } else {
-      ugi = new UnixUserGroupInformation(
-        TEST_USER, new String[] { TEST_GROUP });
+      ugi = UserGroupInformation.createUserForTesting(TEST_USER, new String[] { TEST_GROUP });
     }
 
-    UnixUserGroupInformation.saveToConf(
-      conf, UnixUserGroupInformation.UGI_PROPERTY_NAME, ugi);
-
     ctx.confOptions = new HashMap<String, String>();
     for (Map.Entry<String, String> entry : conf) {
       ctx.confOptions.put(entry.getKey(), entry.getValue());
     }
+    List<String> groupList = new ArrayList<String>();
+    for (String group : ugi.getGroupNames())
+      groupList.add(group);
+    ctx.confOptions.put("effective_user", ugi.getUserName());
     return ctx;
   }
 

+ 2 - 19
desktop/libs/hadoop/java/src/test/org/apache/hadoop/thriftfs/TestNamenodePlugin.java

@@ -27,13 +27,10 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.thriftfs.api.Block;
 import org.apache.hadoop.thriftfs.api.Constants;
 import org.apache.hadoop.thriftfs.api.ContentSummary;
-import org.apache.hadoop.thriftfs.api.DatanodeInfo;
-import org.apache.hadoop.thriftfs.api.DatanodeReportType;
 import org.apache.hadoop.thriftfs.api.IOException;
 import org.apache.hadoop.thriftfs.api.Namenode;
 import org.apache.hadoop.thriftfs.api.RequestContext;
@@ -138,20 +135,6 @@ public class TestNamenodePlugin {
     }
   }
 
-  @Test
-  public void testGetDatanodeReport() throws Exception {
-    int numNodes = cluster.getDataNodes().size();
-    List<DatanodeInfo> nodes = namenode
-      .getDatanodeReport(ctx, DatanodeReportType.ALL_DATANODES);
-    assertEquals(numNodes, nodes.size());
-
-    nodes = namenode.getDatanodeReport(ctx, DatanodeReportType.DEAD_DATANODES);
-    assertEquals(0, nodes.size());
-
-    nodes = namenode.getDatanodeReport(ctx, DatanodeReportType.LIVE_DATANODES);
-    assertEquals(numNodes, nodes.size());
-  }
-
   @Test
   public void testGetPreferredBlockSize() throws Exception {
     long bs = 1024;
@@ -461,14 +444,14 @@ public class TestNamenodePlugin {
 
     // Dir made by 'ctx' should be owned by the current user
     namenode.mkdirhier(ctx, "/test-by-current", (short)0755);
-    assertEquals(UserGroupInformation.getCurrentUGI().getUserName(),
+    assertEquals(UserGroupInformation.getCurrentUser().getUserName(),
                  fs.getFileStatus(byCurrentPath).getOwner());
 
     // With a null context (eg clients that don't support this), should be
     // the current user
     assertTrue(fs.delete(byCurrentPath, true));
     namenode.mkdirhier(null, "/test-by-current", (short)0755);
-    assertEquals(UserGroupInformation.getCurrentUGI().getUserName(),
+    assertEquals(UserGroupInformation.getCurrentUser().getUserName(),
                  fs.getFileStatus(byCurrentPath).getOwner());
 
 

+ 6 - 0
desktop/libs/hadoop/src/hadoop/conf.py

@@ -98,6 +98,12 @@ HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
   dynamic_default=find_jar("../../java-lib/hue-plugins-*.jar", root=os.path.dirname(__file__)),
   private=True)
 
+HADOOP_STATIC_GROUP_MAPPING_CLASSPATH = Config("hadoop_static_group_mapping_classpath",
+  help="[Used only in testing code.] Path to the Hadoop static group mapping jar.",
+  type=str,
+  dynamic_default=find_jar("../../static-group-mapping/java-lib/static-group-mapping-*.jar", root=os.path.dirname(__file__)),
+  private=True)
+
 HDFS_CLUSTERS = UnspecifiedConfigSection(
   "hdfs_clusters",
   help="One entry for each HDFS cluster",

+ 13 - 17
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -39,6 +39,7 @@ from desktop.lib.conf import validate_port
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
 from hadoop.api.common.ttypes import RequestContext, IOException
+import hadoop.conf
 from hadoop.fs import normpath
 from hadoop.fs.exceptions import PermissionDeniedException
 
@@ -54,7 +55,6 @@ del _tmp_mod
 LOG = logging.getLogger(__name__)
 
 DEFAULT_USER = "webui"
-DEFAULT_GROUPS = ["webui"]
 
 # The number of bytes to read if not specified
 DEFAULT_READ_SIZE = 1024*1024 # 1MB
@@ -199,7 +199,7 @@ class HadoopFileSystem(object):
     # user information in a thread-local variable so that
     # safety can be preserved there.
     self.thread_local = threading.local()
-    self.setuser(DEFAULT_USER, DEFAULT_GROUPS)
+    self.setuser(DEFAULT_USER)
     LOG.debug("Initialized HadoopFS: %s:%d (%s)", host, thrift_port, hadoop_bin_path)
 
   def _get_hdfs_base(self):
@@ -231,18 +231,13 @@ class HadoopFileSystem(object):
     """
     return self.stats("/")["user"]
 
-  def setuser(self, user, groups=None):
-    # Hadoop UGI *must* have at least one group, so we mirror
-    # the username as a group if not specified
+  def setuser(self, user):
+    # Hadoop determines the groups the user belongs to on the server side.
     self.thread_local.request_context = RequestContext()
-    if not groups:
-      groups = [user]
     if not self.request_context.confOptions:
       self.request_context.confOptions = {}
-    self.thread_local.ugi = ",".join([user] + groups)
-    self.thread_local.request_context.confOptions['hadoop.job.ugi'] = self.thread_local.ugi
+    self.thread_local.request_context.confOptions['effective_user'] = user
     self.thread_local.user = user
-    self.thread_local.groups = groups
 
   @property
   def user(self):
@@ -256,10 +251,6 @@ class HadoopFileSystem(object):
   def request_context(self):
     return self.thread_local.request_context
 
-  @property
-  def ugi(self):
-    return self.thread_local.ugi
-
   @_coerce_exceptions
   def open(self, path, mode="r", *args, **kwargs):
     if mode == "w":
@@ -765,17 +756,22 @@ class FileUpload(object):
       extra_confs.append("-Ddfs.block.size=%d" % block_size)
     self.subprocess_cmd = [self.fs.hadoop_bin_path,
                            "dfs",
-                           "-Dfs.default.name=" + self.fs.uri,
-                           "-Dhadoop.job.ugi=" + self.fs.ugi] + \
+                           "-Dfs.default.name=" + self.fs.uri] + \
                            extra_confs + \
                            ["-put", "-", encode_fs_path(path)]
+    env = i18n.make_utf8_env()
+    if env.has_key('HADOOP_CLASSPATH'):
+      env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()
+    else:
+      env['HADOOP_CLASSPATH'] = hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()
+
     self.path = path
     self.putter = subprocess.Popen(self.subprocess_cmd,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    close_fds=True,
-                                   env=i18n.make_utf8_env(),
+                                   env=env,
                                    bufsize=WRITE_BUFFER_SIZE)
   @require_open
   def write(self, data):

+ 9 - 14
desktop/libs/hadoop/src/hadoop/job_tracker.py

@@ -38,7 +38,6 @@ VALID_TASK_TYPES = set(["map", "reduce", "job_cleanup", "job_setup"])
 JT_THRIFT_TIMEOUT = 15
 
 DEFAULT_USER = "webui"
-DEFAULT_GROUPS = ["webui"]
 
 def test_jt_configuration(cluster):
   """Test FS configuration. Returns list of (confvar, error)."""
@@ -75,7 +74,7 @@ class LiveJobTracker(object):
     # threads by restricting the stateful components to a thread
     # thread-local.
     self.thread_local = threading.local()
-    self.setuser(DEFAULT_USER, DEFAULT_GROUPS)
+    self.setuser(DEFAULT_USER)
 
   def thriftjobid_from_string(self, jobid):
     """The jobid looks like this: job_201001301455_0001"""
@@ -153,21 +152,17 @@ class LiveJobTracker(object):
     for taskstatus in tip.taskStatuses.values():
       self._fixup_taskstatus(taskstatus)
 
-  def setuser(self, user, groups=None):
-    # Hadoop UGI *must* have at least one group, so we mirror
-    # the username as a group if not specified
+  def setuser(self, user):
+    # Hadoop determines the groups the user belongs to on the server side.
     self.thread_local.request_context = RequestContext()
-    if not groups:
-      groups = [user]
     if not self.thread_local.request_context.confOptions:
       self.thread_local.request_context.confOptions = {}
-    self.thread_local.ugi = ",".join([user] + groups)
-    self.thread_local.request_context.confOptions['hadoop.job.ugi'] = self.thread_local.ugi
+    self.thread_local.request_context.confOptions['effective_user'] = user
+    self.thread_local.user = user
 
   @property
-  def ugi(self):
-    # Here for backwards-compatibility.
-    return self.thread_local.ugi
+  def user(self):
+    return self.thread_local.user
 
   @property
   def request_context(self):
@@ -276,7 +271,7 @@ class LiveJobTracker(object):
     """
     Returns a ThriftJobList (does not include task info)
     """
-    joblist = self.client.getKilledJobs(None)
+    joblist = self.client.getKilledJobs(self.thread_local.request_context)
     for job in joblist.jobs:
       self._fixup_job(job)
     return joblist
@@ -326,7 +321,7 @@ class LiveJobTracker(object):
   def get_task(self, jobid, taskid):
     """Return a ThriftTaskInProgress"""
     try:
-      tip = self.client.getTask(jobid, taskid)
+      tip = self.client.getTask(self.thread_local.request_context, taskid)
     except JobNotFoundException, e:
       e.response_data = dict(code="JT_JOB_NOT_FOUND",
                              message="Could not find job %s on JobTracker." % (jobid.asString,),

+ 59 - 16
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -31,6 +31,7 @@
 import atexit
 import subprocess
 import os
+import pwd
 import logging
 import sys
 import signal
@@ -56,6 +57,13 @@ CLEANUP_TMP_DIR=os.environ.get("MINI_CLUSTER_CLEANUP", True)
 # How long to wait for cluster to start up.  (seconds)
 MAX_CLUSTER_STARTUP_TIME = 120.0
 
+# users and their groups which are used in Hue tests.
+TEST_USER_GROUP_MAPPING = {
+   'test': ['test'], 'chown_test': ['chown_test'],
+   'notsuperuser': ['notsuperuser'], 'gamma': ['gamma'],
+   'webui': ['webui']
+}
+
 LOGGER=logging.getLogger(__name__)
 
 def _find_unused_port():
@@ -107,7 +115,22 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
 """)
     finally:
       f.close()
-  
+
+    if self.superuser not in TEST_USER_GROUP_MAPPING:
+      TEST_USER_GROUP_MAPPING[self.superuser] = [self.superuser]
+
+    _write_static_group_mapping(TEST_USER_GROUP_MAPPING,
+      tmppath('ugm.properties'))
+
+    write_config({'hadoop.security.group.mapping': 'org.apache.hadoop.security.StaticUserGroupMapping',
+      'hadoop.security.static.group.mapping.file': tmppath('ugm.properties')}, tmppath('in-conf/core-site.xml'))
+
+    hadoop_policy_keys = ['client', 'client.datanode', 'datanode', 'inter.datanode', 'namenode', 'inter.tracker', 'job.submission', 'task.umbilical', 'refresh.policy', 'admin.operations']
+    hadoop_policy_config = {}
+    for policy in hadoop_policy_keys:
+      hadoop_policy_config['security.' + policy + '.protocol.acl'] = '*'
+    write_config(hadoop_policy_config, tmppath('in-conf/hadoop-policy.xml'))
+
     details_file = file(tmppath("details.json"), "w+")
     try:
       args = [ hadoop.conf.HADOOP_BIN.get(), "jar",
@@ -138,11 +161,13 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
         "-D", "jobtracker.thrift.address=localhost:%d" % _find_unused_port(),
         # Jobs realize they have finished faster with this timeout.
         "-D", "jobclient.completion.poll.interval=50",
+        "-D", "hadoop.security.authorization=true",
+        "-D", "hadoop.policy.file=%s/hadoop-policy.xml" % in_conf_dir,
       ]
       env = {}
       env["HADOOP_CONF_DIR"] = in_conf_dir
       env["HADOOP_OPTS"] = "-Dtest.build.data=%s" % (self.tmpdir, )
-      env["HADOOP_CLASSPATH"] = hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get()
+      env["HADOOP_CLASSPATH"] = ':'.join([hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(), hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()])
       env["HADOOP_HEAPSIZE"] = "128"
       env["HADOOP_HOME"] = hadoop.conf.HADOOP_HOME.get()
       env["HADOOP_LOG_DIR"] = self.log_dir
@@ -199,26 +224,31 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
     self.config_dir = tmppath("conf")
     os.mkdir(self.config_dir)
     write_config(self.config, tmppath("conf/core-site.xml"), 
-      ["fs.default.name", "jobclient.completion.poll.interval", "fs.checkpoint.period", "fs.checkpoint.dir"])
+      ["fs.default.name", "jobclient.completion.poll.interval",
+       "fs.checkpoint.period", "fs.checkpoint.dir",
+       'hadoop.security.group.mapping', 'hadoop.security.static.group.mapping.file'])
     write_config(self.config, tmppath("conf/hdfs-site.xml"), ["fs.default.name", "dfs.http.address", "dfs.secondary.http.address"])
     # mapred.job.tracker isn't written out into self.config, so we fill
     # that one out more manually.
     write_config({ "mapred.job.tracker": "localhost:%d" % self.jobtracker_port },
-      tmppath("conf/mapred-site.xml"), ["mapred.job.tracker"])
+      tmppath("conf/mapred-site.xml"))
+    write_config(hadoop_policy_config, tmppath('conf/hadoop-policy.xml'))
 
     # Once the config is written out, we can start the 2NN.
-    env = os.environ.copy()
     args = [hadoop.conf.HADOOP_BIN.get(), 
       '--config', self.config_dir,
       'secondarynamenode']
+
+    LOGGER.debug("Starting 2NN at: " +
+      self.config['dfs.secondary.http.address'])
+    LOGGER.debug("2NN command: %s env: %s" % (repr(args), repr(env)))
+
     self.secondary_proc = subprocess.Popen(
       args=args,
-      stdout=file(tmppath("stdout"), "w"),
-      stderr=stderr,
+      stdout=file(tmppath("stdout.2nn"), "w"),
+      stderr=file(tmppath("stderr.2nn"), "w"),
       env=env)
 
-    LOGGER.debug("Starting 2NN at: " +
-      self.config['dfs.secondary.http.address'])
     while True:
       try:
         response = urllib2.urlopen(urllib2.Request('http://' +
@@ -284,7 +314,7 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
     This is essentially the user that the cluster was started
     with.
     """
-    return self.config["hadoop.job.ugi"].split(",")[0]
+    return pwd.getpwuid(os.getuid()).pw_name
 
   @property
   def namenode_thrift_port(self):
@@ -363,7 +393,7 @@ def shared_cluster(conf=False):
   cluster.shutdown = finish
   return cluster
 
-def write_config(config, path, variables):
+def write_config(config, path, variables=None):
   """
   Minimal utility to write Hadoop-style configuration
   from a configuration map (config), into a new file
@@ -375,16 +405,29 @@ def write_config(config, path, variables):
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration>
 """)
-    for name in variables:
+    keys = (variables and (variables,) or (config.keys(),))[0]
+    for name in keys:
       value = config[name]
-      f.write("<property>\n")
-      f.write("  <name>%s</name>\n" % name)
-      f.write("  <value>%s</value>\n" % value)
-      f.write("</property>\n")
+      f.write("  <property>\n")
+      f.write("    <name>%s</name>\n" % name)
+      f.write("    <value>%s</value>\n" % value)
+      f.write("  </property>\n")
     f.write("</configuration>\n")
   finally:
     f.close()
 
+def _write_static_group_mapping(user_group_mapping, path):
+  """
+  Create a Java-style .properties file to contain the static user -> group
+  mapping used by tests.
+  """
+  f = file(path, 'w')
+  try:
+    for user, groups in user_group_mapping.iteritems():
+      f.write('%s = %s\n' % (user, ','.join(groups)))
+  finally:
+    f.close()
+
 def shared_cluster_internal():
   """
   Manages _shared_cluster.

+ 75 - 0
desktop/libs/hadoop/static-group-mapping/build.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="static-group-mapping" default="jar">
+
+  <!-- Additional -->
+  <property name="name" value="${ant.project.name}"/>
+  <property name="build.encoding" value="ISO-8859-1"/>
+  <property name="build.classes" location="${build.dir}/classes"/>
+  <property name="src.dir"  location="${basedir}/src"/>
+
+  <property name="lib.dir" location="${build.dir}/lib" />
+
+  <!-- this version number is provided by the invoker -->
+  <property name="static-group-mapping.version" value="DEV" />
+  <property name="test.available" value="true" />
+
+  <property name="javac.debug" value="on"/>
+  <property name="javac.optimize" value="on"/>
+  <property name="javac.deprecation" value="on"/>
+  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+  <property name="javac.args" value=""/>
+
+  <!-- create the list of files to add to the classpath -->
+  <path id="build.classpath">
+    <fileset dir="${hadoop.home}">
+      <include name="hadoop*core*.jar" />
+    </fileset>
+  </path>
+
+  <target name="init">
+    <mkdir dir="${build.classes}"/>
+  </target>
+
+  <target name="compile" depends="init">
+    <echo message="contrib: ${name}"/>
+    <echo message="src dir: ${src.dir}"/>
+    <javac
+     encoding="${build.encoding}"
+     srcdir="${src.dir}"
+     includes="**/*.java"
+     destdir="${build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     deprecation="${javac.deprecation}">
+     <classpath refid="build.classpath"/>
+     <compilerarg line="${javac.args} ${javac.args.warnings}" />
+    </javac>
+  </target>
+
+  <!-- Override jar target to specify main class, and to include default
+       configuration file -->
+  <target name="jar" depends="compile">
+    <jar jarfile="${build.dir}/static-group-mapping-${static-group-mapping.version}.jar"
+         basedir="${build.classes}">
+    </jar>
+  </target>
+
+</project>

+ 76 - 0
desktop/libs/hadoop/static-group-mapping/src/org/apache/hadoop/security/StaticUserGroupMapping.java

@@ -0,0 +1,76 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Implements getGroups() of the GroupMappingServiceProvider interface by
+ * parsing a static, .properties-style file of the form:
+ *
+ * <code>username = comma,separated,list,of,groups</code>
+ */
+public class StaticUserGroupMapping implements GroupMappingServiceProvider {
+
+  private static final String GROUP_MAPPING_CONF_KEY = "hadoop.security.static.group.mapping.file";
+  private static Map<String, List<String>> userGroupMapping = new ConcurrentHashMap<String, List<String>>();
+
+  static {
+    Properties properties = new Properties();
+    try {
+      String groupMappingFile = new Configuration().get(GROUP_MAPPING_CONF_KEY);
+      if (groupMappingFile == null) {
+        throw new RuntimeException("StaticUserGroupMapping class specified, but no static group " + 
+          "mapping file given. Set " + GROUP_MAPPING_CONF_KEY);
+      }
+      properties.load(new FileInputStream(groupMappingFile));
+      for (Entry<Object, Object> user : properties.entrySet()) {
+        userGroupMapping.put((String) user.getKey(), Arrays.asList(((String) user.getValue()).split(",")));
+      }
+    } catch (FileNotFoundException e) {
+      throw new RuntimeException(e);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public List<String> getGroups(String user) throws IOException {
+    return userGroupMapping.get(user);
+  }
+
+  public static void main(String[] args) {
+    for (Entry<String, List<String>> ugi : userGroupMapping.entrySet()) {
+      System.out.println(ugi.getKey() + " : " + ugi.getValue().size() + " : "
+          + StringUtils.join(StringUtils.COMMA_STR, ugi.getValue()));
+    }
+  }
+
+}

Some files were not shown because too many files changed in this diff