|
@@ -77,7 +77,8 @@ def get_default_abfs_url():
|
|
|
def get_default_abfs_fs():
|
|
def get_default_abfs_fs():
|
|
|
default_fs = core_site.get_default_fs()
|
|
default_fs = core_site.get_default_fs()
|
|
|
|
|
|
|
|
- return default_fs if default_fs and default_fs.startwith('abfs://') else ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
|
|
|
|
|
|
|
+ return default_fs if default_fs and default_fs.startwith('abfs://') and \
|
|
|
|
|
+ ABFS_CLUSTERS['default'].ENABLE_DEFAULTFS_FROM_CORESITE.get() else ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
|
|
|
|
|
|
|
|
ADLS_CLUSTERS = UnspecifiedConfigSection(
|
|
ADLS_CLUSTERS = UnspecifiedConfigSection(
|
|
|
"adls_clusters",
|
|
"adls_clusters",
|
|
@@ -143,6 +144,11 @@ ABFS_CLUSTERS = UnspecifiedConfigSection(
|
|
|
each=ConfigSection(
|
|
each=ConfigSection(
|
|
|
help="Information about a single ABFS cluster",
|
|
help="Information about a single ABFS cluster",
|
|
|
members=dict(
|
|
members=dict(
|
|
|
|
|
+ ENABLE_DEFAULTFS_FROM_CORESITE=Config(
|
|
|
|
|
+ key="enable_defaultfs_from_coresite",
|
|
|
|
|
+ type=bool,
|
|
|
|
|
+ default=True,
|
|
|
|
|
+ help="Enable this param to use the defaultFS from core-site.xml"),
|
|
|
FS_DEFAULTFS=Config("fs_defaultfs", help="abfs://<container_name>@<account_name>.dfs.core.windows.net", type=str, default=None),
|
|
FS_DEFAULTFS=Config("fs_defaultfs", help="abfs://<container_name>@<account_name>.dfs.core.windows.net", type=str, default=None),
|
|
|
WEBHDFS_URL=Config("webhdfs_url",
|
|
WEBHDFS_URL=Config("webhdfs_url",
|
|
|
help="https://<account_name>.dfs.core.windows.net",
|
|
help="https://<account_name>.dfs.core.windows.net",
|