浏览代码

[raz] Auto-configure default FS from core-site.xml (#2396)

- Default FS is not RAZ specific
- Also, `get_home_dir_for_ABFS` -> `get_home_dir_for_abfs`
Harsh Gupta 4 年之前
父节点
当前提交
d9b127f8d8

+ 2 - 2
apps/filebrowser/src/filebrowser/views.py

@@ -218,8 +218,8 @@ def view(request, path):
 
   # default_abfs_home is set in jquery.filechooser.js
   if 'default_abfs_home' in request.GET:
-    from azure.abfs.__init__ import get_home_dir_for_ABFS
-    home_dir_path = get_home_dir_for_ABFS()
+    from azure.abfs.__init__ import get_home_dir_for_abfs
+    home_dir_path = get_home_dir_for_abfs()
     if request.fs.isdir(home_dir_path):
       return format_preserving_redirect(
           request,

+ 2 - 2
desktop/core/src/desktop/models.py

@@ -2002,8 +2002,8 @@ class ClusterConfig(object):
       })
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
-      from azure.abfs.__init__ import get_home_dir_for_ABFS
-      home_path = remote_home_storage if remote_home_storage else get_home_dir_for_ABFS().encode('utf-8')
+      from azure.abfs.__init__ import get_home_dir_for_abfs
+      home_path = remote_home_storage if remote_home_storage else get_home_dir_for_abfs().encode('utf-8')
       interpreters.append({
         'type': 'abfs',
         'displayName': _('ABFS'),

+ 1 - 1
desktop/libs/azure/src/azure/abfs/__init__.py

@@ -164,7 +164,7 @@ def abfspath(path, fs_defaultfs = None):
   LOG.debug("%s" % path)
   return path
 
-def get_home_dir_for_ABFS():
+def get_home_dir_for_abfs():
   """
   Attempts to go to the directory set by the user in the configuration file. If not defaults to abfs://
   """

+ 1 - 1
desktop/libs/azure/src/azure/abfs/abfs.py

@@ -296,7 +296,7 @@ class ABFS(object):
     """
     Attempts to go to the directory set by the user in the configuration file. If not defaults to abfs://
     """
-    return Init_ABFS.get_home_dir_for_ABFS()
+    return Init_ABFS.get_home_dir_for_abfs()
 
   # Find or alter information about the URI path
   # --------------------------------

+ 6 - 1
desktop/libs/azure/src/azure/conf.py

@@ -75,7 +75,12 @@ def get_default_abfs_url():
   return ABFS_CLUSTERS['default'].WEBHDFS_URL.get()
 
 def get_default_abfs_fs():
-  return ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
+  default_fs = core_site.get_default_fs()
+  
+  if default_fs:
+    return default_fs
+  else:
+    return ABFS_CLUSTERS['default'].FS_DEFAULTFS.get()
 
 ADLS_CLUSTERS = UnspecifiedConfigSection(
   "adls_clusters",

+ 4 - 0
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -50,6 +50,8 @@ _CNF_S3A_RAZ_BUCKET_ENDPOINT = 'fs.s3a.bucket.(?P<bucket>[^.]+).endpoint'
 _CNF_ADLS_RAZ_API_URL = 'fs.azure.ext.raz.rest.host.url'
 _CNF_ADLS_RAZ_CLUSTER_NAME = 'fs.azure.ext.raz.adls.access.cluster.name'
 
+_CNF_DEFAULT_FS = 'fs.defaultFS'
+
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
 _CNF_ADLS_REFRESH_URL = 'dfs.adls.oauth2.refresh.url'
@@ -153,6 +155,8 @@ def get_raz_default_endpoint():
         'bucket': match.group('bucket')
       }
 
+def get_default_fs():
+  return get_conf().get(_CNF_DEFAULT_FS)
 
 def get_adls_client_id():
   """

+ 6 - 0
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -76,6 +76,10 @@ def test_core_site():
     <name>fs.azure.ext.raz.adls.access.cluster.name</name>
     <value>gehue-adls</value>
   </property>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>abfs://data@gethuedevstorage.dfs.core.windows.net/hue-adls</value>
+  </property> 
 </configuration>
     """
     open_file(os.path.join(hadoop_home, 'core-site.xml'), 'w').write(xml)
@@ -90,6 +94,8 @@ def test_core_site():
     assert_equal(core_site.get_raz_cluster_name(), 'gehue-adls')
     assert_equal(core_site.get_raz_default_endpoint(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'gethue-dev'})
 
+    assert_equal(core_site.get_default_fs(), 'abfs://data@gethuedevstorage.dfs.core.windows.net/hue-adls')
+
     assert_equal(get_remote_home_storage(), 's3a://gethue-dev')
     assert_equal(get_region(), 'us-west-2')
   finally: