Ver Fonte

HUE-9073 [fb] Only enable IDBroker when on cloud.

We currently enable S3 when IDBroker is configured with S3:
https://github.com/cloudera/hue/blob/6db70177fc1c6955340da4d7475cdda5c44c2aef/desktop/libs/aws/src/aws/conf.py#L248,
https://github.com/cloudera/hue/blob/6db70177fc1c6955340da4d7475cdda5c44c2aef/desktop/core/src/desktop/lib/idbroker/conf.py#L60
However, I'm noticing on that IDBroker is always configured with all 3
cloud providers.
Furthermore, the CM cloud connector is not being used in CDP so CM is
currently not configuring Hue with anything. For now only enable cloud
storage is it's specifically configured or if on cloud and IDBroker
enabled.
Jean-Francois Desjeans Gauthier há 6 anos atrás
pai
commit
63bcf03493

+ 1 - 1
desktop/libs/aws/src/aws/conf.py

@@ -245,7 +245,7 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
 
 
 def is_enabled():
-  return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata() or conf_idbroker.is_idbroker_enabled('s3a')
+  return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata()
 
 
 def is_ec2_instance():

+ 44 - 38
desktop/libs/aws/src/aws/tests.py

@@ -54,26 +54,28 @@ class TestAWS(unittest.TestCase):
       with patch('aws.client.conf_idbroker.get_conf') as get_conf:
         with patch('aws.client.Client.get_s3_connection'):
           with patch('aws.client.IDBroker.get_cab') as get_cab:
-            get_conf.return_value = {
-              'fs.s3a.ext.cab.address': 'address'
-            }
-            get_cab.return_value = {
-              'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
-            }
-            provider = get_credential_provider('default', 'hue')
-            assert_equal(provider.get_credentials().get('AccessKeyId'), 'AccessKeyId')
-            client1 = get_client(name='default', fs='s3a', user='hue')
-            client2 = get_client(name='default', fs='s3a', user='hue')
-            assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
+            with patch('aws.client.aws_conf.has_iam_metadata') as has_iam_metadata:
+              get_conf.return_value = {
+                'fs.s3a.ext.cab.address': 'address'
+              }
+              get_cab.return_value = {
+                'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
+              }
+              has_iam_metadata.return_value = True
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('AccessKeyId'), 'AccessKeyId')
+              client1 = get_client(name='default', fs='s3a', user='hue')
+              client2 = get_client(name='default', fs='s3a', user='hue')
+              assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
 
-            get_cab.return_value = {
-              'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': int(current_ms_from_utc()) + 10*1000}
-            }
-            client3 = get_client(name='default', fs='s3a', user='hue')
-            client4 = get_client(name='default', fs='s3a', user='hue')
-            client5 = get_client(name='default', fs='s3a', user='test')
-            assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
-            assert_not_equal(client4, client5) # Test different user have different clients
+              get_cab.return_value = {
+                'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': int(current_ms_from_utc()) + 10*1000}
+              }
+              client3 = get_client(name='default', fs='s3a', user='hue')
+              client4 = get_client(name='default', fs='s3a', user='hue')
+              client5 = get_client(name='default', fs='s3a', user='test')
+              assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
+              assert_not_equal(client4, client5) # Test different user have different clients
     finally:
       finish()
       clear_cache()
@@ -85,17 +87,19 @@ class TestAWS(unittest.TestCase):
       with patch('aws.client.conf_idbroker.get_conf') as get_conf:
         with patch('aws.client.Client.get_s3_connection'):
           with patch('aws.client.IDBroker.get_cab') as get_cab:
-            get_conf.return_value = {
-              'fs.s3a.ext.cab.address': 'address'
-            }
-            get_cab.return_value = {
-              'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
-            }
-            provider = get_credential_provider('default', 'hue')
-            assert_equal(provider.get_credentials().get('AccessKeyId'), 'AccessKeyId')
+            with patch('aws.client.aws_conf.has_iam_metadata') as has_iam_metadata:
+              get_conf.return_value = {
+                'fs.s3a.ext.cab.address': 'address'
+              }
+              get_cab.return_value = {
+                'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
+              }
+              has_iam_metadata.return_value = True
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('AccessKeyId'), 'AccessKeyId')
 
-            client = Client.from_config(conf.AWS_ACCOUNTS['default'], get_credential_provider('default', 'hue'))
-            assert_equal(client._region, 'ap-northeast-1')
+              client = Client.from_config(conf.AWS_ACCOUNTS['default'], get_credential_provider('default', 'hue'))
+              assert_equal(client._region, 'ap-northeast-1')
     finally:
       finish()
       clear_cache()
@@ -108,15 +112,17 @@ class TestAWS(unittest.TestCase):
         with patch('aws.client.conf_idbroker.get_conf') as get_conf:
           with patch('aws.client.Client.get_s3_connection'):
             with patch('aws.client.IDBroker.get_cab') as get_cab:
-              get_region.return_value = 'us-west-1'
-              get_conf.return_value = {
-                'fs.s3a.ext.cab.address': 'address'
-              }
-              get_cab.return_value = {
-                'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
-              }
-              client = Client.from_config(None, get_credential_provider('default', 'hue'))
-              assert_equal(client._region, 'us-west-1') # Test different user have different clients
+              with patch('aws.client.aws_conf.has_iam_metadata') as has_iam_metadata:
+                get_region.return_value = 'us-west-1'
+                get_conf.return_value = {
+                  'fs.s3a.ext.cab.address': 'address'
+                }
+                get_cab.return_value = {
+                  'Credentials': {'AccessKeyId': 'AccessKeyId', 'Expiration': 0}
+                }
+                has_iam_metadata.return_value = True
+                client = Client.from_config(None, get_credential_provider('default', 'hue'))
+                assert_equal(client._region, 'us-west-1') # Test different user have different clients
     finally:
       finish()
       clear_cache()

+ 19 - 2
desktop/libs/azure/src/azure/conf.py

@@ -29,6 +29,8 @@ LOG = logging.getLogger(__name__)
 PERMISSION_ACTION_ABFS = "abfs_access"
 PERMISSION_ACTION_ADLS = "adls_access"
 REFRESH_URL = 'https://login.microsoftonline.com/<tenant_id>/oauth2/<version>token'
+META_DATA_URL = 'http://169.254.169.254/metadata/instance'
+AZURE_METADATA = None
 
 def get_default_client_id():
   """
@@ -144,10 +146,10 @@ ABFS_CLUSTERS = UnspecifiedConfigSection(
 )
 
 def is_adls_enabled():
-  return ('default' in list(AZURE_ACCOUNTS.keys()) and AZURE_ACCOUNTS['default'].get_raw() and AZURE_ACCOUNTS['default'].CLIENT_ID.get() or conf_idbroker.is_idbroker_enabled('azure')) and 'default' in list(ADLS_CLUSTERS.keys())
+  return ('default' in list(AZURE_ACCOUNTS.keys()) and AZURE_ACCOUNTS['default'].get_raw() and AZURE_ACCOUNTS['default'].CLIENT_ID.get() or (conf_idbroker.is_idbroker_enabled('azure') and has_azure_metadata())) and 'default' in list(ADLS_CLUSTERS.keys())
 
 def is_abfs_enabled():
-  return ('default' in list(AZURE_ACCOUNTS.keys()) and AZURE_ACCOUNTS['default'].get_raw() and AZURE_ACCOUNTS['default'].CLIENT_ID.get() or conf_idbroker.is_idbroker_enabled('azure')) and 'default' in list(ABFS_CLUSTERS.keys())
+  return ('default' in list(AZURE_ACCOUNTS.keys()) and AZURE_ACCOUNTS['default'].get_raw() and AZURE_ACCOUNTS['default'].CLIENT_ID.get() or (conf_idbroker.is_idbroker_enabled('azure') and has_azure_metadata())) and 'default' in list(ABFS_CLUSTERS.keys())
 
 def has_adls_access(user):
   from desktop.auth.backend import is_admin
@@ -157,6 +159,21 @@ def has_abfs_access(user):
   from desktop.auth.backend import is_admin
   return user.is_authenticated() and user.is_active and (is_admin(user) or user.has_hue_permission(action="abfs_access", app="filebrowser"))
 
+def azure_metadata():
+  global AZURE_METADATA
+  if AZURE_METADATA is None:
+    from desktop.lib.rest import http_client, resource
+    client = http_client.HttpClient(META_DATA_URL, logger=LOG)
+    root = resource.Resource(client)
+    try:
+      AZURE_METADATA = root.get('/compute', params={'api-version':'2019-06-04', 'format':'json'}, headers={'Metadata': 'true'})
+    except Exception as e:
+      AZURE_METADATA = False
+  return AZURE_METADATA
+
+def has_azure_metadata():
+  return azure_metadata() is not None
+
 def config_validator(user):
   res = []
 

+ 40 - 36
desktop/libs/azure/src/azure/tests.py

@@ -83,24 +83,26 @@ class TestAzureAdl(unittest.TestCase):
       with patch('azure.client.conf_idbroker.get_conf') as get_conf:
         with patch('azure.client.WebHdfs.get_client'):
           with patch('azure.client.IDBroker.get_cab') as get_cab:
-            get_conf.return_value = {
-              'fs.azure.ext.cab.address': 'address'
-            }
-            get_cab.return_value = { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': 0 }
-            provider = get_credential_provider('default', 'hue')
-            assert_equal(provider.get_credentials().get('access_token'), 'access_token')
-            client1 = get_client(name='default', fs='adl', user='hue')
-            client2 = get_client(name='default', fs='adl', user='hue')
-            assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
-
-            get_cab.return_value = {
-              'Credentials': {'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': int(current_ms_from_utc()) + 10*1000}
-            }
-            client3 = get_client(name='default', fs='adl', user='hue')
-            client4 = get_client(name='default', fs='adl', user='hue')
-            client5 = get_client(name='default', fs='adl', user='test')
-            assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
-            assert_not_equal(client4, client5) # Test different user have different clients
+            with patch('azure.client.conf.has_azure_metadata') as has_azure_metadata:
+              get_conf.return_value = {
+                'fs.azure.ext.cab.address': 'address'
+              }
+              has_azure_metadata.return_value = True
+              get_cab.return_value = { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': 0 }
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('access_token'), 'access_token')
+              client1 = get_client(name='default', fs='adl', user='hue')
+              client2 = get_client(name='default', fs='adl', user='hue')
+              assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
+
+              get_cab.return_value = {
+                'Credentials': {'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': int(current_ms_from_utc()) + 10*1000}
+              }
+              client3 = get_client(name='default', fs='adl', user='hue')
+              client4 = get_client(name='default', fs='adl', user='hue')
+              client5 = get_client(name='default', fs='adl', user='test')
+              assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
+              assert_not_equal(client4, client5) # Test different user have different clients
     finally:
       for f in finish:
         f()
@@ -159,24 +161,26 @@ class TestAzureAbfs(unittest.TestCase):
       with patch('azure.client.conf_idbroker.get_conf') as get_conf:
         with patch('azure.client.ABFS.get_client'):
           with patch('azure.client.IDBroker.get_cab') as get_cab:
-            get_conf.return_value = {
-              'fs.azure.ext.cab.address': 'address'
-            }
-            get_cab.return_value = { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': 0 }
-            provider = get_credential_provider('default', 'hue')
-            assert_equal(provider.get_credentials().get('access_token'), 'access_token')
-            client1 = get_client(name='default', fs='abfs', user='hue')
-            client2 = get_client(name='default', fs='abfs', user='hue')
-            assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
-
-            get_cab.return_value = {
-              'Credentials': {'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': int(current_ms_from_utc()) + 10*1000}
-            }
-            client3 = get_client(name='default', fs='abfs', user='hue')
-            client4 = get_client(name='default', fs='abfs', user='hue')
-            client5 = get_client(name='default', fs='abfs', user='test')
-            assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
-            assert_not_equal(client4, client5) # Test different user have different clients
+            with patch('azure.client.conf.has_azure_metadata') as has_azure_metadata:
+              get_conf.return_value = {
+                'fs.azure.ext.cab.address': 'address'
+              }
+              has_azure_metadata.return_value = True
+              get_cab.return_value = { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': 0 }
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('access_token'), 'access_token')
+              client1 = get_client(name='default', fs='abfs', user='hue')
+              client2 = get_client(name='default', fs='abfs', user='hue')
+              assert_not_equal(client1, client2) # Test that with Expiration 0 clients not equal
+
+              get_cab.return_value = {
+                'Credentials': {'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': int(current_ms_from_utc()) + 10*1000}
+              }
+              client3 = get_client(name='default', fs='abfs', user='hue')
+              client4 = get_client(name='default', fs='abfs', user='hue')
+              client5 = get_client(name='default', fs='abfs', user='test')
+              assert_equal(client3, client4) # Test that with 10 sec expiration, clients equal
+              assert_not_equal(client4, client5) # Test different user have different clients
     finally:
       for f in finish:
         f()