Эх сурвалжийг харах

Option to get the defaultFS for ABFS from core-site else use the config from Hue (#2826)

Jira: CDPD-39592
Mahesh Balakrishnan 3 жил өмнө
parent
commit
c4b866830f

+ 1 - 0
desktop/conf.dist/hue.ini

@@ -1916,6 +1916,7 @@ submit_to=True
 [[abfs_clusters]]
 # Default ABFS cluster
 [[[default]]]
+## enable_defaultfs_from_coresite=true
 ## fs_defaultfs=abfs://<container_name>@<account_name>.dfs.core.windows.net
 ## webhdfs_url=https://<account_name>.dfs.core.windows.net
 

+ 1 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1899,6 +1899,7 @@
   [[abfs_clusters]]
     # Default ABFS cluster
     [[[default]]]
+      ## enable_defaultfs_from_coresite=true
       ## fs_defaultfs=abfs://<container_name>@<account_name>.dfs.core.windows.net
       ## webhdfs_url=https://<account_name>.dfs.core.windows.net
 

+ 7 - 1
desktop/libs/azure/src/azure/conf.py

@@ -77,7 +77,8 @@ def get_default_abfs_url():
 def get_default_abfs_fs():
   default_fs = core_site.get_default_fs()
 
-  return default_fs if default_fs and default_fs.startwith('abfs://') else ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
+  return default_fs if default_fs and default_fs.startwith('abfs://') and \
+                       ABFS_CLUSTERS['default'].ENABLE_DEFAULTFS_FROM_CORESITE.get() else ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
 
 ADLS_CLUSTERS = UnspecifiedConfigSection(
   "adls_clusters",
@@ -143,6 +144,11 @@ ABFS_CLUSTERS = UnspecifiedConfigSection(
   each=ConfigSection(
     help="Information about a single ABFS cluster",
     members=dict(
+      ENABLE_DEFAULTFS_FROM_CORESITE=Config(
+        key="enable_defaultfs_from_coresite",
+        type=bool,
+        default=True,
+        help="Enable this param to use the defaultFS from core-site.xml"),
       FS_DEFAULTFS=Config("fs_defaultfs", help="abfs://<container_name>@<account_name>.dfs.core.windows.net", type=str, default=None),
       WEBHDFS_URL=Config("webhdfs_url",
                          help="https://<account_name>.dfs.core.windows.net",