瀏覽代碼

[raz] Take S3 bucket region from Hue config instead of from core-site.xml (#3439)

What changes were proposed in this pull request?
- Here the region in the core-site is always set to env. bucket but if someone wants to browse a bucket of different region, if will try to find that bucket in that same region.
- This happens even when we explicitly change the region via configs since core-site value was given preference. Now we are only depending on region values from Hue config.

How was this patch tested?
- Tested in live E2E cluster.
Harsh Gupta 2 年之前
父節點
當前提交
b03bfc571e

+ 1 - 1
desktop/libs/aws/src/aws/client.py

@@ -45,7 +45,7 @@ def _make_client(identifier, user):
   client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
 
   if aws_conf.is_raz_s3():
-    host = aws_conf.get_default_host() or client_conf.HOST.get()
+    host = client_conf.HOST.get()
     s3_client = RazS3Connection(username=user, host=host)  # Note: Remaining AWS configuration is fully skipped
     s3_client_expiration = None
   else:

+ 2 - 10
desktop/libs/aws/src/aws/conf.py

@@ -108,14 +108,6 @@ def get_default_region():
   return get_region(conf=AWS_ACCOUNTS['default']) if 'default' in AWS_ACCOUNTS else get_region()
 
 
-def get_default_host():
-  '''Returns the S3 host when Raz is configured'''
-
-  if get_raz_api_url():
-    endpoint = get_raz_s3_default_bucket()
-    if endpoint:
-      return endpoint.get('host')
-
 def get_region(conf=None):
   global REGION_CACHED
 
@@ -123,9 +115,9 @@ def get_region(conf=None):
     return REGION_CACHED
   region = ''
 
-  if conf or get_default_host():
+  if conf:
     # First check the host/endpoint configuration
-    endpoint = get_default_host() or conf.HOST.get()
+    endpoint = conf.HOST.get()
     if endpoint:
       if re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE):
         region = re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE).group('region')

+ 0 - 2
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -25,7 +25,6 @@ import tempfile
 from nose.tools import assert_true, assert_equal, assert_false, assert_not_equal, assert_raises
 
 from desktop.models import get_remote_home_storage
-from aws.conf import get_region
 
 from hadoop import core_site
 
@@ -97,7 +96,6 @@ def test_core_site():
     assert_equal(core_site.get_default_fs(), 'abfs://data@gethuedevstorage.dfs.core.windows.net/hue-adls')
 
     assert_equal(get_remote_home_storage(), 's3a://gethue-dev')
-    assert_equal(get_region(), 'us-west-2')
   finally:
     core_site.reset()
     for f in finish: