Преглед изворни кода

HUE-4567 [aws] Read AWS credentials from configured script files

Jenny Kim пре 9 година
родитељ
комит
1842de1ca9

+ 2 - 2
desktop/libs/aws/src/aws/client.py

@@ -42,8 +42,8 @@ class Client(object):
       raise ValueError('Can\'t create AWS client, credential is not configured')
 
     return cls(
-      aws_access_key_id=conf.ACCESS_KEY_ID.get(),
-      aws_secret_access_key=conf.SECRET_ACCESS_KEY.get(),
+      aws_access_key_id=access_key_id,
+      aws_secret_access_key=secret_access_key,
       region=conf.REGION.get()
     )
 

+ 37 - 3
desktop/libs/aws/src/aws/conf.py

@@ -17,7 +17,27 @@ from __future__ import absolute_import
 
 from boto.regioninfo import get_regions
 
-from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool
+from django.utils.translation import ugettext_lazy as _
+
+from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool, coerce_password_from_script
+
+from hadoop.core_site import get_s3a_access_key, get_s3a_secret_key
+
+
+def get_default_access_key_id():
+  """
+  Attempt to set AWS access key ID from script, else core-site, else None
+  """
+  access_key_id_script = AWS_ACCOUNTS['default'].ACCESS_KEY_ID_SCRIPT.get()
+  return access_key_id_script or get_s3a_access_key()
+
+
+def get_default_secret_key():
+  """
+  Attempt to set AWS secret key from script, else core-site, else None
+  """
+  secret_access_key_script = AWS_ACCOUNTS['default'].SECRET_ACCESS_KEY_SCRIPT.get()
+  return secret_access_key_script or get_s3a_secret_key()
 
 
 AWS_ACCOUNTS = UnspecifiedConfigSection(
@@ -29,13 +49,27 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
       ACCESS_KEY_ID=Config(
         key='access_key_id',
         type=str,
-        private=True
+        private=True,
+        dynamic_default=get_default_access_key_id
       ),
+      ACCESS_KEY_ID_SCRIPT=Config(
+        key='access_key_id_script',
+        default=None,
+        private=True,
+        type=coerce_password_from_script,
+        help=_("Execute this script to produce the AWS access key ID.")),
       SECRET_ACCESS_KEY=Config(
         key='secret_access_key',
         type=str,
-        private=True
+        private=True,
+        dynamic_default=get_default_secret_key
       ),
+      SECRET_ACCESS_KEY_SCRIPT=Config(
+        key='secret_access_key_script',
+        default=None,
+        private=True,
+        type=coerce_password_from_script,
+        help=_("Execute this script to produce the AWS secret access key.")),
       ALLOW_ENVIRONMENT_CREDENTIALS=Config(
         help='Allow to use environment sources of credentials (environment variables, EC2 profile).',
         key='allow_environment_credentials',

+ 17 - 1
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -26,7 +26,7 @@ import os.path
 import conf
 import confparse
 
-__all = ['get_conf', 'get_trash_interval']
+__all = ['get_conf', 'get_trash_interval', 'get_s3a_access_key', 'get_s3a_secret_key']
 
 LOG = logging.getLogger(__name__)
 
@@ -34,6 +34,8 @@ _CORE_SITE_PATH = None                  # Path to core-site.xml
 _CORE_SITE_DICT = None                  # A dictionary of name/value config options
 
 _CNF_TRASH_INTERVAL = 'fs.trash.interval'
+_CNF_S3A_ACCESS_KEY = 'fs.s3a.access.key'
+_CNF_S3A_SECRET_KEY = 'fs.s3a.secret.key'
 
 def reset():
   """Reset the cached conf"""
@@ -79,3 +81,17 @@ def get_trash_interval():
   Also indicates whether trash is enabled or not.
   """
   return get_conf().get(_CNF_TRASH_INTERVAL)
+
+def get_s3a_access_key():
+  """
+  Get S3A AWS access key ID
+  https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html
+  """
+  return get_conf().get(_CNF_S3A_ACCESS_KEY)
+
+def get_s3a_secret_key():
+  """
+  Get S3A AWS secret key
+  https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html
+  """
+  return get_conf().get(_CNF_S3A_SECRET_KEY)