Browse Source

[raz] Auto detect s3 bucket and region from config file

Romain Rigaux 4 years ago
parent
commit
d552ae0067
1 changed files with 17 additions and 0 deletions
  1. 17 0
      desktop/libs/hadoop/src/hadoop/core_site.py

+ 17 - 0
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -18,6 +18,7 @@
 from __future__ import absolute_import
 from __future__ import absolute_import
 import errno
 import errno
 import logging
 import logging
+import re
 import sys
 import sys
 
 
 from hadoop import conf
 from hadoop import conf
@@ -44,6 +45,7 @@ _CNF_S3A_SESSION_TOKEN = 'fs.s3a.session.token'
 
 
 _CNF_S3A_RAZ_API_URL = 'fs.s3a.ext.raz.rest.host.url'
 _CNF_S3A_RAZ_API_URL = 'fs.s3a.ext.raz.rest.host.url'
 _CNF_S3A_RAZ_CLUSTER_NAME = 'fs.s3a.ext.raz.s3.access.cluster.name'
 _CNF_S3A_RAZ_CLUSTER_NAME = 'fs.s3a.ext.raz.s3.access.cluster.name'
+_CNF_S3A_RAZ_BUCKET_ENDPOINT = 'fs.s3a.bucket.(?P<bucket>[^.]+).endpoint'
 
 
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
@@ -131,6 +133,21 @@ def get_raz_cluster_name():
   """
   """
   return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '')
   return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '')
 
 
+def get_raz_default_endpoint():
+  """
+  Get the name of the default S3 bucket of Raz
+
+  S3 only. Add check for Azure when supported.
+  """
+
+  for key, val in get_conf().items():
+    match = re.search(_CNF_S3A_RAZ_BUCKET_ENDPOINT, key)
+    if match:
+      return {
+        'region': val,
+        'bucket': match.group('bucket')
+      }
+
 
 
 def get_adls_client_id():
 def get_adls_client_id():
   """
   """