浏览代码

HUE-258. Hue file upload should create files as the user doing the upload

Adds a new SudoShell jar which can use proxyuser credentials to do fs
operations as another user.
Todd Lipcon 15 年之前
父节点
当前提交
75143f0a9a

+ 17 - 2
desktop/libs/hadoop/Makefile

@@ -51,10 +51,16 @@ STATIC_GROUP_MAPPING_BUILD := $(STATIC_GROUP_MAPPING_DIR)/build
 STATIC_GROUP_MAPPING_LIB := $(STATIC_GROUP_MAPPING_DIR)/java-lib
 STATIC_GROUP_MAPPING := $(STATIC_GROUP_MAPPING_LIB)/static-group-mapping-$(DESKTOP_VERSION).jar
 
-compile: $(PLUGIN) $(STATIC_GROUP_MAPPING)
+SUDO_SHELL_DIR := $(realpath .)/sudo-shell
+SUDO_SHELL_SRC := $(SUDO_SHELL_DIR)/src
+SUDO_SHELL_BUILD := $(SUDO_SHELL_DIR)/build
+SUDO_SHELL_LIB := $(SUDO_SHELL_DIR)/java-lib
+SUDO_SHELL := $(SUDO_SHELL_LIB)/sudo-shell-$(DESKTOP_VERSION).jar
+
+compile: $(PLUGIN) $(STATIC_GROUP_MAPPING) $(SUDO_SHELL)
 
 clean::
-	rm -Rf $(PLUGIN_JAVA_LIB) $(STATIC_GROUP_MAPPING_LIB)
+	rm -Rf $(PLUGIN_JAVA_LIB) $(STATIC_GROUP_MAPPING_LIB) $(SUDO_SHELL_LIB)
 
 ifneq (,$(wildcard $(PLUGIN_JAVA_DIR)))
 PLUGIN_OPTS := -Dbuild.dir=$(BLD_DIR_PLUGINS) \
@@ -81,6 +87,15 @@ $(STATIC_GROUP_MAPPING): $(shell find $(STATIC_GROUP_MAPPING_SRC))
 	cd $(STATIC_GROUP_MAPPING_DIR) && ant $(ANT_OPTS) $(STATIC_GROUP_MAPPING_OPTS) jar
 	cp $(STATIC_GROUP_MAPPING_BUILD)/$(notdir $(STATIC_GROUP_MAPPING)) $(STATIC_GROUP_MAPPING)
 
+SUDO_SHELL_OPTS := -Dbuild.dir=$(SUDO_SHELL_BUILD) \
+         -Dhadoop.home=$(HADOOP_HOME) \
+         -Dsudo-shell.version=$(DESKTOP_VERSION)
+
+$(SUDO_SHELL): $(shell find $(SUDO_SHELL_SRC))
+	mkdir -p $(SUDO_SHELL_LIB)
+	@echo "--- Building sudo fsshell tool"
+	cd $(SUDO_SHELL_DIR) && ant $(ANT_OPTS) $(SUDO_SHELL_OPTS) jar
+	cp $(SUDO_SHELL_BUILD)/$(notdir $(SUDO_SHELL)) $(SUDO_SHELL)
 #
 # Tell the `bdist' target to exclude our java source.
 #

+ 7 - 0
desktop/libs/hadoop/src/hadoop/conf.py

@@ -104,6 +104,13 @@ HADOOP_STATIC_GROUP_MAPPING_CLASSPATH = Config("hadoop_static_group_mapping_clas
   dynamic_default=find_jar("../../static-group-mapping/java-lib/static-group-mapping-*.jar", root=os.path.dirname(__file__)),
   private=True)
 
+SUDO_SHELL_JAR = Config("hadoop_sudo_shell_jar",
+  help="Tool that allows a proxy user UGI to be used to upload files.",
+  type=str,
+  dynamic_default=find_jar("../../sudo-shell/java-lib/sudo-shell-*.jar",
+                           root=os.path.dirname(__file__)),
+  private=True)
+
 HDFS_CLUSTERS = UnspecifiedConfigSection(
   "hdfs_clusters",
   help="One entry for each HDFS cluster",

+ 3 - 14
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -789,20 +789,15 @@ class FileUpload(object):
     if block_size:
       extra_confs.append("-Ddfs.block.size=%d" % block_size)
     self.subprocess_cmd = [self.fs.hadoop_bin_path,
-                           "dfs",
+                           "jar",
+                           hadoop.conf.SUDO_SHELL_JAR.get(),
+                           self.fs.user,
                            "-Dfs.default.name=" + self.fs.uri] + \
                            extra_confs + \
                            ["-put", "-", encode_fs_path(path)]
 
     self.subprocess_env = i18n.make_utf8_env()
 
-    if self.fs.security_enabled:
-      token = self.fs.get_delegation_token()
-      self.token_file = tempfile.NamedTemporaryFile()
-      self.token_file.write(token.delegationTokenBytes)
-      self.token_file.flush()
-      self.subprocess_env['HADOOP_TOKEN_FILE_LOCATION'] = self.token_file.name
-
     if self.subprocess_env.has_key('HADOOP_CLASSPATH'):
       self.subprocess_env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()
     else:
@@ -832,12 +827,6 @@ class FileUpload(object):
       if ioe.errno == errno.EPIPE:
         stdout, stderr = self.putter.communicate()
 
-    if self.fs.security_enabled and self.token_file:
-      try:
-        self.token_file.close()
-      except:
-        LOG.warn('Failed to close HDFS delegation token file %s' % (self.token_file.name(),))
-
     self.closed = True
     if stderr:
       LOG.warn("HDFS FileUpload (cmd='%s', env='%s') outputted stderr:\n%s" %

+ 78 - 0
desktop/libs/hadoop/sudo-shell/build.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="sudo-shell" default="jar">
+
+  <!-- Additional -->
+  <property name="name" value="${ant.project.name}"/>
+  <property name="build.encoding" value="ISO-8859-1"/>
+  <property name="build.classes" location="${build.dir}/classes"/>
+  <property name="src.dir"  location="${basedir}/src"/>
+
+  <property name="lib.dir" location="${build.dir}/lib" />
+
+  <!-- this version number is provided by the invoker -->
+  <property name="sudo-shell.version" value="DEV" />
+  <property name="test.available" value="true" />
+
+  <property name="javac.debug" value="on"/>
+  <property name="javac.optimize" value="on"/>
+  <property name="javac.deprecation" value="on"/>
+  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+  <property name="javac.args" value=""/>
+
+  <!-- create the list of files to add to the classpath -->
+  <path id="build.classpath">
+    <fileset dir="${hadoop.home}">
+      <include name="hadoop*core*.jar" />
+    </fileset>
+  </path>
+
+  <target name="init">
+    <mkdir dir="${build.classes}"/>
+  </target>
+
+  <target name="compile" depends="init">
+    <echo message="contrib: ${name}"/>
+    <echo message="src dir: ${src.dir}"/>
+    <javac
+     encoding="${build.encoding}"
+     srcdir="${src.dir}"
+     includes="**/*.java"
+     destdir="${build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     deprecation="${javac.deprecation}">
+     <classpath refid="build.classpath"/>
+     <compilerarg line="${javac.args} ${javac.args.warnings}" />
+    </javac>
+  </target>
+
+  <!-- Override jar target to specify main class, and to include default
+       configuration file -->
+  <target name="jar" depends="compile">
+    <jar jarfile="${build.dir}/sudo-shell-${sudo-shell.version}.jar"
+         basedir="${build.classes}">
+      <manifest>
+        <attribute name="Main-Class" value="com.cloudera.hue.SudoFsShell"/>
+      </manifest>
+    </jar>
+  </target>
+
+</project>

+ 52 - 0
desktop/libs/hadoop/sudo-shell/src/com/cloudera/hue/SudoFsShell.java

@@ -0,0 +1,52 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.cloudera.hue;
+
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.security.UserGroupInformation;
+import java.security.PrivilegedExceptionAction;
+
+public class SudoFsShell {
+  private static void usage() {
+    System.err.println("usage: SudoFsShell <username> <shell args ...>");
+  }
+
+  public static void main(String []args) throws Exception {
+    if (args.length < 1) {
+      usage();
+      System.exit(1);
+    }
+
+    String username = args[0];
+    final String shellArgs[] = new String[args.length - 1];
+    System.arraycopy(args, 1, shellArgs, 0, args.length-1);
+
+    UserGroupInformation sudoUgi;
+    if (UserGroupInformation.isSecurityEnabled()) {
+      sudoUgi = UserGroupInformation.createProxyUser(
+        username, UserGroupInformation.getCurrentUser());
+    } else {
+      sudoUgi = UserGroupInformation.createRemoteUser(username);
+    }
+
+    sudoUgi.doAs(new PrivilegedExceptionAction<Void>() {
+        public Void run() throws Exception {
+          FsShell.main(shellArgs);
+          return null;
+        }
+      });
+  }
+}