Ver código fonte

HUE-9073 [fb] Add ABFS core_site config.

Jean-Francois Desjeans Gauthier 6 anos atrás
pai
commit
981951a43a

+ 5 - 5
desktop/libs/azure/src/azure/active_directory.py

@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from builtins import object
 import logging
 
-from azure.conf import AZURE_ACCOUNTS, get_default_refresh_url
+from azure.conf import get_refresh_url
 from desktop.lib.python_util import current_ms_from_utc
 from desktop.lib.rest import http_client, resource
 
@@ -56,14 +56,14 @@ class ActiveDirectory(object):
 
 
   @classmethod
-  def from_config(cls, conf='default', version=None):
-    access_key_id = AZURE_ACCOUNTS['default'].CLIENT_ID.get()
-    secret_access_key = AZURE_ACCOUNTS['default'].CLIENT_SECRET.get()
+  def from_config(cls, conf=None, version=None):
+    access_key_id = conf.CLIENT_ID.get()
+    secret_access_key = conf.CLIENT_SECRET.get()
 
     if None in (access_key_id, secret_access_key):
       raise ValueError('Can\'t create azure client, credential is not configured')
 
-    url = get_default_refresh_url(version)
+    url = get_refresh_url(conf, version)
 
     return cls(
       url,

+ 9 - 10
desktop/libs/azure/src/azure/conf.py

@@ -18,11 +18,11 @@ from __future__ import absolute_import
 import logging
 
 from django.utils.translation import ugettext_lazy as _, ugettext as _t
-from hadoop.core_site import get_adls_client_id, get_adls_authentication_code, get_adls_refresh_url
 
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_password_from_script
 from desktop.lib.idbroker import conf as conf_idbroker
 
+from hadoop import core_site
 
 LOG = logging.getLogger(__name__)
 
@@ -30,32 +30,31 @@ PERMISSION_ACTION_ABFS = "abfs_access"
 PERMISSION_ACTION_ADLS = "adls_access"
 REFRESH_URL = 'https://login.microsoftonline.com/<tenant_id>/oauth2/<version>token'
 
-
 def get_default_client_id():
   """
   Attempt to set AWS client id from script, else core-site, else None
   """
   client_id_script = AZURE_ACCOUNTS['default'].CLIENT_ID_SCRIPT.get()
-  return client_id_script or get_adls_client_id()
+  return client_id_script or core_site.get_adls_client_id() or core_site.get_azure_client_id()
 
 def get_default_secret_key():
   """
   Attempt to set AWS secret key from script, else core-site, else None
   """
   client_secret_script = AZURE_ACCOUNTS['default'].CLIENT_SECRET_SCRIPT.get()
-  return client_secret_script or get_adls_authentication_code()
+  return client_secret_script or core_site.get_adls_authentication_code() or core_site.get_azure_client_secret()
 
 def get_default_tenant_id():
   """
   Attempt to set AWS tenant id from script, else core-site, else None
   """
-  tenant_id_script = AZURE_ACCOUNTS['default'].TENANT_ID_SCRIPT.get()
-  return tenant_id_script or get_adls_refresh_url()
+  return AZURE_ACCOUNTS['default'].TENANT_ID_SCRIPT.get()
 
-def get_default_refresh_url(version):
-  refresh_url = REFRESH_URL.replace('<tenant_id>', AZURE_ACCOUNTS['default'].TENANT_ID.get()).replace('<version>', version + '/' if version else '')
-  refresh_url = refresh_url if refresh_url else get_adls_refresh_url()
-  return refresh_url or get_adls_refresh_url()
+def get_refresh_url(conf, version):
+  refresh_url = core_site.get_adls_refresh_url() or core_site.get_azure_client_endpoint()
+  if not refresh_url:
+    refresh_url = REFRESH_URL.replace('<tenant_id>', conf.TENANT_ID.get()).replace('<version>', version + '/' if version else '')
+  return refresh_url
 
 def get_default_region():
   return ""

+ 44 - 0
desktop/libs/azure/src/azure/tests.py

@@ -33,6 +33,28 @@ LOG = logging.getLogger(__name__)
 
 class TestAzureAdl(unittest.TestCase):
 
+  def test_with_core_site(self):
+    try:
+      finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {}}),
+                conf.ADLS_CLUSTERS.set_for_testing({'default': {'fs_defaultfs': 'fs_defaultfs', 'webhdfs_url': 'webhdfs_url'}}))
+      with patch('azure.client.conf_idbroker.get_conf') as get_conf:
+        with patch('azure.client.WebHdfs'):
+          with patch('azure.client.ActiveDirectory.get_token') as get_token:
+            with patch('azure.conf.core_site.get_conf') as core_site_get_conf:
+              get_token.return_value = {'access_token': 'access_token', 'token_type': '', 'expires_on': None}
+              get_conf.return_value = {}
+              core_site_get_conf.return_value = {'dfs.adls.oauth2.client.id': 'client_id', 'dfs.adls.oauth2.credential': 'client_secret', 'dfs.adls.oauth2.refresh.url': 'refresh_url'}
+              client1 = get_client(name='default', fs='adl')
+              client2 = get_client(name='default', fs='adl', user='test')
+
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('access_token'), 'access_token')
+              assert_equal(client1, client2) # Should be the same as no support for user based client with credentials & no Expiration
+    finally:
+      for f in finish:
+        f()
+      clear_cache()
+
   def test_with_credentials(self):
     try:
       finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {'client_id':'client_id', 'client_secret': 'client_secret', 'tenant_id': 'tenant_id'}}),
@@ -87,6 +109,28 @@ class TestAzureAdl(unittest.TestCase):
 
 class TestAzureAbfs(unittest.TestCase):
 
+  def test_with_core_site(self):
+    try:
+      finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {}}),
+                conf.ABFS_CLUSTERS.set_for_testing({'default': {'fs_defaultfs': 'fs_defaultfs', 'webhdfs_url': 'webhdfs_url'}}))
+      with patch('azure.client.conf_idbroker.get_conf') as get_conf:
+        with patch('azure.client.ABFS'):
+          with patch('azure.client.ActiveDirectory.get_token') as get_token:
+            with patch('azure.conf.core_site.get_conf') as core_site_get_conf:
+              get_token.return_value = {'access_token': 'access_token', 'token_type': '', 'expires_on': None}
+              get_conf.return_value = {}
+              core_site_get_conf.return_value = {'fs.azure.account.oauth2.client.id': 'client_id', 'fs.azure.account.oauth2.client.secret': 'client_secret', 'fs.azure.account.oauth2.client.endpoint': 'refresh_url'}
+              client1 = get_client(name='default', fs='abfs')
+              client2 = get_client(name='default', fs='abfs', user='test')
+
+              provider = get_credential_provider('default', 'hue')
+              assert_equal(provider.get_credentials().get('access_token'), 'access_token')
+              assert_equal(client1, client2) # Should be the same as no support for user based client with credentials & no Expiration
+    finally:
+      for f in finish:
+        f()
+      clear_cache()
+
   def test_with_credentials(self):
     try:
       finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {'client_id':'client_id', 'client_secret': 'client_secret', 'tenant_id': 'tenant_id'}}),

+ 14 - 2
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -44,10 +44,13 @@ _CNF_S3A_SESSION_TOKEN = 'fs.s3a.session.token'
 
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
-_CNF_ADLS_SECRET_KEY = 'dfs.adls.oauth2.credential'
 _CNF_ADLS_REFRESH_URL = 'dfs.adls.oauth2.refresh.url'
 _CNF_ADLS_GRANT_TYPE = 'dfs.adls.oauth2.access.token.provider.type'
 
+_CNF_AZURE_CLIENT_ID = 'fs.azure.account.oauth2.client.id'
+_CNF_AZURE_CLIENT_SECRET = 'fs.azure.account.oauth2.client.secret'
+_CNF_AZURE_CLIENT_ENDPOINT = 'fs.azure.account.oauth2.client.endpoint'
+
 _CNF_SECURITY = 'hadoop.security.authentication'
 
 def reset():
@@ -137,4 +140,13 @@ def get_adls_grant_type():
   return get_conf().get(_CNF_ADLS_GRANT_TYPE)
 
 def is_kerberos_enabled():
-  return get_conf().get(_CNF_SECURITY) == 'kerberos'
+  return get_conf().get(_CNF_SECURITY) == 'kerberos'
+
+def get_azure_client_id():
+  return get_conf().get(_CNF_AZURE_CLIENT_ID)
+
+def get_azure_client_secret():
+  return get_conf().get(_CNF_AZURE_CLIENT_SECRET)
+
+def get_azure_client_endpoint():
+  return get_conf().get(_CNF_AZURE_CLIENT_ENDPOINT)