Pārlūkot izejas kodu

HUE-9063 [fb] Lazy load aws client.

Jean-Francois Desjeans Gauthier 6 gadi atpakaļ
vecāks
revīzija
d7e6f0920e

+ 2 - 15
desktop/libs/aws/src/aws/client.py

@@ -55,7 +55,8 @@ def current_ms_from_utc():
 
 def get_client(identifier='default', user=_DEFAULT_USER):
   global CLIENT_CACHE
-  _init_clients()
+  if not CLIENT_CACHE:
+    CLIENT_CACHE = {}
 
   cache_key = _get_cache_key(identifier, user) if conf_idbroker.is_idbroker_enabled('s3a') else _get_cache_key(identifier) # We don't want to cache by username when IDBroker not enabled
   client = CLIENT_CACHE.get(cache_key)
@@ -72,20 +73,6 @@ def get_credential_provider(identifier='default', user=_DEFAULT_USER):
   return CredentialProviderIDBroker(IDBroker.from_core_site('s3a', user)) if conf_idbroker.is_idbroker_enabled('s3a') else CredentialProviderConf(client_conf)
 
 
-def _init_clients():
-  global CLIENT_CACHE
-  if CLIENT_CACHE is not None:
-    return
-  CLIENT_CACHE = {} # Can't convert this to django cache, because S3FileSystem is not pickable
-  if conf_idbroker.is_idbroker_enabled('s3a'):
-    return # No default initializations when IDBroker is enabled
-  for identifier in list(aws_conf.AWS_ACCOUNTS.keys()):
-    CLIENT_CACHE[_get_cache_key(identifier)] = _make_client(identifier)
-  # If default configuration not initialized, initialize client connection with IAM metadata
-  if not CLIENT_CACHE.has_key(_get_cache_key()) and aws_conf.has_iam_metadata():
-    CLIENT_CACHE[_get_cache_key()] = _make_client('default')
-
-
 def _make_client(identifier, user=_DEFAULT_USER):
   client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
 

+ 29 - 5
desktop/libs/aws/src/aws/conf.py

@@ -34,7 +34,15 @@ HYPHEN_ENDPOINT_RE = 's3-(?P<region>[a-z0-9-]+).amazonaws.com'
 DUALSTACK_ENDPOINT_RE = 's3.dualstack.(?P<region>[a-z0-9-]+).amazonaws.com'
 AWS_ACCOUNT_REGION_DEFAULT = 'us-east-1' # Location.USEast
 PERMISSION_ACTION_S3 = "s3_access"
+REGION_CACHED = None
+IS_IAM_CACHED = None
+IS_EC2_CACHED = None
 
+def clear_cache():
+  global REGION_CACHED, IS_IAM_CACHED, IS_EC2_CACHED
+  REGION_CACHED = None
+  IS_IAM_CACHED = None
+  IS_EC2_CACHED = None
 
 def get_locations():
   return ('EU',  # Ireland
@@ -91,6 +99,9 @@ def get_default_region():
 
 
 def get_region(conf=None):
+  global REGION_CACHED
+  if REGION_CACHED is not None:
+    return REGION_CACHED
   region = ''
 
   if conf:
@@ -123,6 +134,8 @@ def get_region(conf=None):
     LOG.warn("Region, %s, not found in the list of supported regions: %s" % (region, ', '.join(get_locations())))
     region = ''
 
+  REGION_CACHED = region
+
   return region
 
 
@@ -239,23 +252,34 @@ def is_ec2_instance():
   # To avoid unnecessary network call, check if Hue is running on EC2 instance
   # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
   # /sys/hypervisor/uuid doesn't work on m5/c5, but /sys/devices/virtual/dmi/id/product_uuid does
+  global IS_EC2_CACHED
+  if IS_EC2_CACHED is not None:
+    return IS_EC2_CACHED
   try:
-    return (os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'r').read()[:3].lower() == 'ec2') or (os.path.exists('/sys/devices/virtual/dmi/id/product_uuid') and open('/sys/devices/virtual/dmi/id/product_uuid', 'r').read()[:3].lower() == 'ec2')
+    IS_EC2_CACHED = (os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'r').read()[:3].lower() == 'ec2') or (os.path.exists('/sys/devices/virtual/dmi/id/product_uuid') and open('/sys/devices/virtual/dmi/id/product_uuid', 'r').read()[:3].lower() == 'ec2') 
   except IOError as e:
-    return 'Permission denied' in str(e) # If permission is denied, assume cost of network call
+    IS_EC2_CACHED = 'Permission denied' in str(e) # If permission is denied, assume cost of network call
   except Exception as e:
+    IS_EC2_CACHED = False
     LOG.exception("Failed to read /sys/hypervisor/uuid or /sys/devices/virtual/dmi/id/product_uuid: %s" % e)
-    return False
+  return IS_EC2_CACHED
+
 
 def has_iam_metadata():
   try:
+    global IS_IAM_CACHED
+    if IS_IAM_CACHED is not None:
+      return IS_IAM_CACHED
     import boto.utils
     if is_ec2_instance():
       metadata = boto.utils.get_instance_metadata(timeout=1, num_retries=1)
-      return 'iam' in metadata
+      IS_IAM_CACHED = 'iam' in metadata
+    else:
+      IS_IAM_CACHED = False
   except Exception as e:
+    IS_IAM_CACHED = False
     LOG.exception("Encountered error when checking IAM metadata: %s" % e)
-  return False
+  return IS_IAM_CACHED
 
 
 def has_s3_access(user):

+ 5 - 0
desktop/libs/aws/src/aws/s3/s3_test.py

@@ -78,6 +78,7 @@ def test_get_default_region():
   try:
     assert_equal('ap-northeast-2', get_default_region())
   finally:
+    conf.clear_cache()
     if finish:
       finish()
 
@@ -86,6 +87,7 @@ def test_get_default_region():
   try:
     assert_equal('ap-south-1', get_default_region())
   finally:
+    conf.clear_cache()
     if finish:
       finish()
 
@@ -94,6 +96,7 @@ def test_get_default_region():
   try:
     assert_equal('ap-southeast-2', get_default_region())
   finally:
+    conf.clear_cache()
     if finish:
       finish()
 
@@ -102,6 +105,7 @@ def test_get_default_region():
   try:
     assert_equal(Location.DEFAULT, get_default_region())
   finally:
+    conf.clear_cache()
     if finish:
       finish()
 
@@ -110,5 +114,6 @@ def test_get_default_region():
   try:
     assert_equal('ca-central-1', get_default_region())
   finally:
+    conf.clear_cache()
     if finish:
       finish()

+ 3 - 0
desktop/libs/aws/src/aws/tests.py

@@ -44,6 +44,7 @@ class TestAWS(unittest.TestCase):
     finally:
       finish()
       clear_cache()
+      conf.clear_cache()
 
   def test_with_idbroker(self):
     try:
@@ -74,6 +75,7 @@ class TestAWS(unittest.TestCase):
     finally:
       finish()
       clear_cache()
+      conf.clear_cache()
 
   def test_with_idbroker_and_config(self):
     try:
@@ -95,3 +97,4 @@ class TestAWS(unittest.TestCase):
     finally:
       finish()
       clear_cache()
+      conf.clear_cache()