Browse Source

[webhdfs] Client can now talk to a secured HDFS

* Added security_enabled (commented out) to hue.ini
bc Wong 13 years ago
parent
commit
b4fb5f6750

+ 9 - 2
desktop/conf.dist/hue.ini

@@ -181,12 +181,15 @@
       # The NameNode http port
       ## http_port=50070
 
+      # Change this if your HDFS cluster is Kerberos-secured
+      ## security_enabled=false
+
       # Use WebHdfs/HttpFs as the communication mechanism. To fallback to
       # using the Thrift plugin (used in Hue 1.x), this must be uncommented
       # and explicitly set to the empty value.
       ## webhdfs_url=
 
-  # Configuration for MapReduce JobTracker
+  # Configuration for MapReduce 0.20 JobTracker (MR1)
   # ------------------------------------------------------------------------
   [[mapred_clusters]]
 
@@ -200,7 +203,11 @@
       # Whether to submit jobs to this cluster
       ## submit_to=False
 
-  # Configuration for Yarn
+      # Change this if your MapReduce cluster is Kerberos-secured
+      ## security_enabled=false
+
+
+  # Configuration for Yarn (MR2)
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
 

+ 17 - 7
desktop/core/src/desktop/lib/rest/http_client.py

@@ -21,6 +21,8 @@ import types
 import urllib
 import urllib2
 
+from urllib2_kerberos import HTTPKerberosAuthHandler
+
 __docformat__ = "epytext"
 
 LOG = logging.getLogger(__name__)
@@ -77,17 +79,12 @@ class HttpClient(object):
     self._logger = logger or LOG
     self._headers = { }
 
-    # Make a basic auth handler that does nothing. Set credentials later.
-    self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
-    authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)
-
     # Make a cookie processor
     cookiejar = cookielib.CookieJar()
 
     self._opener = urllib2.build_opener(
         HTTPErrorProcessor(),
-        urllib2.HTTPCookieProcessor(cookiejar),
-        authhandler)
+        urllib2.HTTPCookieProcessor(cookiejar))
 
 
   def set_basic_auth(self, username, password, realm):
@@ -98,9 +95,22 @@ class HttpClient(object):
     @param realm: The authentication realm.
     @return: The current object
     """
-    self._passmgr.add_password(realm, self._base_url, username, password)
+    # Make a basic auth handler that does nothing. Set credentials later.
+    passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+    passmgr.add_password(realm, self._base_url, username, password)
+    authhandler = urllib2.HTTPBasicAuthHandler(passmgr)
+
+    self._opener.add_handler(authhandler)
     return self
 
+
+  def set_kerberos_auth(self):
+    """Set up kerberos auth for the client, based on the current ticket."""
+    authhandler = HTTPKerberosAuthHandler()
+    self._opener.add_handler(authhandler)
+    return self
+
+
   def set_headers(self, headers):
     """
     Add headers to the request

+ 19 - 9
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -33,7 +33,6 @@ from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary
 
-import hadoop.conf
 
 DEFAULT_HDFS_SUPERUSER = 'hdfs'
 
@@ -59,12 +58,11 @@ class WebHdfs(Hdfs):
     self._temp_dir = temp_dir
     self._fs_defaultfs = fs_defaultfs
 
-    self._client = self._make_client(url)
+    self._client = self._make_client(url, security_enabled)
     self._root = resource.Resource(self._client)
 
     # To store user info
     self._thread_local = threading.local()
-    self._thread_local.user = WebHdfs.DEFAULT_USER
 
     LOG.debug("Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" %
               (self._url, self._security_enabled, self._superuser))
@@ -80,9 +78,12 @@ class WebHdfs(Hdfs):
   def __str__(self):
     return "WebHdfs at %s" % (self._url,)
 
-  def _make_client(self, url):
-    return http_client.HttpClient(
+  def _make_client(self, url, security_enabled):
+    client = http_client.HttpClient(
         url, exc_class=WebHdfsException, logger=LOG)
+    if security_enabled:
+      client.set_kerberos_auth()
+    return client
 
   @property
   def uri(self):
@@ -111,14 +112,23 @@ class WebHdfs(Hdfs):
   
   @property
   def user(self):
-    return self._thread_local.user
+    try:
+      return self._thread_local.user
+    except AttributeError:
+      return WebHdfs.DEFAULT_USER
 
   def _getparams(self):
-    return { "user.name" : self._thread_local.user }
+    if self.security_enabled:
+      return {
+        "user.name" : WebHdfs.DEFAULT_USER,
+        "doas" : self.user
+      }
+    else:
+      return { "user.name" : self.user }
 
   def setuser(self, user):
     """Set a new user. Return the current user."""
-    curr = self._thread_local.user
+    curr = self.user
     self._thread_local.user = user
     return curr
 
@@ -427,7 +437,7 @@ class WebHdfs(Hdfs):
         "Failed to create '%s'. HDFS did not return a redirect" % (path,))
 
     # Now talk to the real thing. The redirect url already includes the params.
-    client = self._make_client(next_url)
+    client = self._make_client(next_url, self.security_enabled)
     return resource.Resource(client).invoke(method, data=data)