浏览代码

HUE-698 Shell app does not work with a secured cluster

Romain Rigaux 13 年之前
父节点
当前提交
8f98473ca8

+ 5 - 3
apps/shell/src/shell/shellmanager.py

@@ -41,6 +41,8 @@ from eventlet.green import time
 from hadoop.cluster import all_mrclusters, get_all_hdfs, \
 from hadoop.cluster import all_mrclusters, get_all_hdfs, \
                            get_cluster_conf_for_job_submission
                            get_cluster_conf_for_job_submission
 
 
+from desktop.conf import KERBEROS
+
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
 SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
 SHELL_INPUT_LOGGER = logging.getLogger("shell_input")
 SHELL_INPUT_LOGGER = logging.getLogger("shell_input")
@@ -132,7 +134,7 @@ class Shell(object):
     Returns the NamedTemporaryFile that contains the combined delegation tokens.
     Returns the NamedTemporaryFile that contains the combined delegation tokens.
     """
     """
     merged_token_file = tempfile.NamedTemporaryFile(dir=delegation_token_dir)
     merged_token_file = tempfile.NamedTemporaryFile(dir=delegation_token_dir)
-    merge_tool_args = [hadoop.conf.HADOOP_BIN.get(), 'jar']
+    merge_tool_args = [hadoop.KERBEROS.HDFS_CLUSTERS['default'].HADOOP_BIN.get(), 'jar']
     merge_tool_args += [hadoop.conf.CREDENTIALS_MERGER_JAR.get(), merged_token_file.name]
     merge_tool_args += [hadoop.conf.CREDENTIALS_MERGER_JAR.get(), merged_token_file.name]
     merge_tool_args += [token_file.name for token_file in delegation_token_files]
     merge_tool_args += [token_file.name for token_file in delegation_token_files]
     LOG.debug("Merging credentials files with command: '%s'" % (' '.join(merge_tool_args)))
     LOG.debug("Merging credentials files with command: '%s'" % (' '.join(merge_tool_args)))
@@ -161,9 +163,9 @@ class Shell(object):
         current_user = cluster.user
         current_user = cluster.user
         try:
         try:
           cluster.setuser(username)
           cluster.setuser(username)
-          token = cluster.get_delegation_token()
+          token = cluster.get_delegation_token(KERBEROS.HUE_PRINCIPAL.get())
           token_file = tempfile.NamedTemporaryFile(dir=delegation_token_dir)
           token_file = tempfile.NamedTemporaryFile(dir=delegation_token_dir)
-          token_file.write(token.delegationTokenBytes)
+          token_file.write(token)
           token_file.flush()
           token_file.flush()
           delegation_token_files.append(token_file)
           delegation_token_files.append(token_file)
         finally:
         finally:

+ 6 - 3
desktop/core/ext-py/urllib2_kerberos-0.1.6/urllib2_kerberos.py

@@ -60,9 +60,12 @@ class AbstractKerberosAuthHandler:
         host = req.get_host()
         host = req.get_host()
         LOG.debug("req.get_host() returned %s" % host)
         LOG.debug("req.get_host() returned %s" % host)
 
 
-        tail, sep, head = host.rpartition(':')
-        domain = tail or head
-                
+        # We need Python 2.4 compatibility
+        #tail, sep, head = host.rpartition(':')
+        #domain = tail or head
+        host_parts = host.rsplit(':', 1)
+        domain = host_parts[0]
+
         result, self.context = k.authGSSClientInit("HTTP@%s" % domain)
         result, self.context = k.authGSSClientInit("HTTP@%s" % domain)
 
 
         if result < 1:
         if result < 1:

+ 26 - 6
desktop/libs/hadoop/credentials-merger/src/main/java/com/cloudera/hue/CredentialsMerger.java

@@ -15,23 +15,30 @@
 // limitations under the License.
 // limitations under the License.
 package com.cloudera.hue;
 package com.cloudera.hue;
 
 
+import java.io.BufferedReader;
+import java.io.FileReader;
 import java.io.IOException;
 import java.io.IOException;
 import java.io.File;
 import java.io.File;
 
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
 
 
 /**
 /**
  * A tool to merge the credentials of multiple distinct files containing Hadoop
  * A tool to merge the credentials of multiple distinct files containing Hadoop
  * delegation tokens into a single file.
  * delegation tokens into a single file.
  */
  */
 public class CredentialsMerger {
 public class CredentialsMerger {
-  
+
   /**
   /**
    * Merge several credentials files into one. Give the desired output file
    * Merge several credentials files into one. Give the desired output file
    * first, followed by all of the input files.
    * first, followed by all of the input files.
    *
    *
+   * <p>File formats are tried in this order: TokenStorageFile, urlEncodedString.
+   * </p>
+   *
    * @param args &lt;out&gt; &lt;in1&gt; ...
    * @param args &lt;out&gt; &lt;in1&gt; ...
    * @throws IOException  in the event of an error reading or writing files.
    * @throws IOException  in the event of an error reading or writing files.
    */
    */
@@ -44,15 +51,29 @@ public class CredentialsMerger {
     Path outputFile = new Path("file://" + new File(args[0]).getAbsolutePath());
     Path outputFile = new Path("file://" + new File(args[0]).getAbsolutePath());
     Configuration conf = new Configuration();
     Configuration conf = new Configuration();
     Credentials credentials = new Credentials();
     Credentials credentials = new Credentials();
+
     for (int i = 1; i < args.length; i++) {
     for (int i = 1; i < args.length; i++) {
-      Credentials singleFileCredentials = Credentials.readTokenStorageFile(
-          new Path("file://" + new File(args[i]).getAbsolutePath()), conf);
-      credentials.addAll(singleFileCredentials);
+      try {
+        Credentials singleFileCredentials = Credentials.readTokenStorageFile(
+            new Path("file://" + new File(args[i]).getAbsolutePath()), conf);
+        credentials.addAll(singleFileCredentials);
+      } catch (IOException e) {
+        BufferedReader reader = new BufferedReader(new FileReader(args[i]));
+        try {
+          // Retry to read the token with an encodedUrl format
+          Token<?> token = new Token();
+          String encodedtoken = reader.readLine();
+          token.decodeFromUrlString(encodedtoken);
+          credentials.addToken(new Text(args[i]), token);
+        } finally {
+          reader.close();
+        }
+      }
     }
     }
 
 
     credentials.writeTokenStorageFile(outputFile, conf);
     credentials.writeTokenStorageFile(outputFile, conf);
   }
   }
-  
+
   /**
   /**
    * Show command usage.
    * Show command usage.
    */
    */
@@ -60,5 +81,4 @@ public class CredentialsMerger {
     System.err.println("Usage: " + CredentialsMerger.class.getCanonicalName()
     System.err.println("Usage: " + CredentialsMerger.class.getCanonicalName()
         + " <dst> <src> ...");
         + " <dst> <src> ...");
   }
   }
-
 }
 }

+ 2 - 4
desktop/libs/hadoop/src/hadoop/job_tracker.py

@@ -378,7 +378,5 @@ class LiveJobTracker(object):
     """
     """
     return self.client.setJobPriority(self.thread_local.request_context, jobid, priority)
     return self.client.setJobPriority(self.thread_local.request_context, jobid, priority)
 
 
-  def get_delegation_token(self):
-    # TODO(atm): The second argument here should really be the Hue kerberos
-    # principal, which doesn't exist yet. Todd's working on that.
-    return self.client.getDelegationToken(self.thread_local.request_context, 'hadoop')
+  def get_delegation_token(self, principal):
+    return self.client.getDelegationToken(self.thread_local.request_context, principal).delegationTokenBytes