Преглед изворни кода

PR826 [hdfs] Adding HTTPS options for WebHDFS (#826)

* Update hdfs_site.py

* Update webhdfs.py

* Update Import to restrict to one value

* removed comment after testing

* remove extra space
jkm пре 6 година
родитељ
комит
87ccf2e09e

+ 2 - 3
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -35,7 +35,7 @@ from hadoop.fs import normpath as fs_normpath, SEEK_SET, SEEK_CUR, SEEK_END
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary
-from hadoop.hdfs_site import get_nn_sentry_prefixes, get_umask_mode, get_supergroup
+from hadoop.hdfs_site import get_nn_sentry_prefixes, get_umask_mode, get_supergroup, get_webhdfs_ssl
 
 
 import hadoop.conf
@@ -1006,8 +1006,7 @@ def _get_service_url(hdfs_config):
   fs_defaultfs = hdfs_config.FS_DEFAULTFS.get()
   netloc = Hdfs.urlsplit(fs_defaultfs)[1]
   host = netloc.split(':')[0]
-  port = hadoop.conf.DEFAULT_NN_HTTP_PORT
-  return "http://%s:%s/webhdfs/v1" % (host, port)
+  return "{0}://{1}:{2}/webhdfs/v1".format(get_webhdfs_ssl()["protocol"], host, get_webhdfs_ssl()["port"])
 
 
 def test_fs_configuration(fs_config):

+ 9 - 2
desktop/libs/hadoop/src/hadoop/hdfs_site.py

@@ -21,7 +21,7 @@ import os.path
 
 import conf
 import confparse
-
+from hadoop.conf import DEFAULT_NN_HTTP_PORT
 
 LOG = logging.getLogger(__name__)
 
@@ -31,7 +31,8 @@ _CNF_NN_PERMISSIONS_UMASK_MODE = 'fs.permissions.umask-mode'
 _CNF_NN_SENTRY_PREFIXES = 'sentry.authorization-provider.hdfs-path-prefixes' # Deprecated
 _CNF_NN_SENTRY_PATH_PREFIXES = 'sentry.hdfs.integration.path.prefixes'
 _CNF_NN_PERMISSIONS_SUPERGROUP = 'dfs.permissions.superusergroup'
-
+_CNF_HTTP_POLICY = 'dfs.http.policy'
+_CNF_WEBHDFS_HTTPS_PORT = 'dfs.https.port'
 
 def reset():
   global _HDFS_SITE_DICT
@@ -51,6 +52,12 @@ def get_umask_mode():
 
   return int(umask, 8)
 
+def get_webhdfs_ssl():
+  settings = {"protocol": "http", "port": hadoop.conf.DEFAULT_NN_HTTP_PORT}
+  if get_conf().get(_CNF_HTTP_POLICY, 'http') == "HTTPS_ONLY":
+    settings["protocol"] = 'https'
+    settings["port"] = get_conf().get(_CNF_WEBHDFS_HTTPS_PORT, hadoop.conf.DEFAULT_NN_HTTP_PORT)
+  return settings
 
 def get_nn_sentry_prefixes():
   prefixes = set()