Browse Source

HUE-5420 [aws] Support alternative s3 endpoints (#455)

* [HUE-5420] Support alternative s3 endpoints.

This is done by exposing the following configuration options:
  * proxy_address - address of the endpoint
  * proxy_port - port to use on the endpoint
  * is_secure - determine whether we're using https or http
  * calling_format - allow OrdinaryCallingFormat since not all endpoints have DNS
setup to support `<bucket-name>.domain/<file>`.

* [HUE-5420] Use _() around help strings and document options in conf.tmpl.
Ewan Higgs 9 years ago
parent
commit
51e26240ac

+ 12 - 0
desktop/conf.dist/hue.ini

@@ -1277,6 +1277,18 @@
       # AWS region to use
       # AWS region to use
       ## region=us-east-1
       ## region=us-east-1
 
 
+      # Endpoint overrides
+      ## proxy_address=
+      ## proxy_port=
+
+      # Secure connections are the default, but this can be explicitly overridden:
+      ## is_secure=true
+
+      # The default calling format uses https://<bucket-name>.s3.amazonaws.com but 
+      # this may not make sense if DNS is not configured in this way for custom endpoints.
+      # e.g. Use boto.s3.connection.OrdinaryCallingFormat for https://s3.amazonaws.com/<bucket-name>
+      ## calling_format=boto.s3.connection.S3Connection.DefaultCallingFormat
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings for the Sentry lib
 # Settings for the Sentry lib

+ 12 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1284,6 +1284,18 @@
       # AWS region to use
       # AWS region to use
       ## region=us-east-1
       ## region=us-east-1
 
 
+      # Endpoint overrides
+      ## proxy_address=
+      ## proxy_port=
+
+      # Secure connections are the default, but this can be explicitly overridden:
+      ## is_secure=true
+
+      # The default calling format uses https://<bucket-name>.s3.amazonaws.com but
+      # this may not make sense if DNS is not configured in this way for custom endpoints.
+      # e.g. Use boto.s3.connection.OrdinaryCallingFormat for https://s3.amazonaws.com/<bucket-name>
+      ## calling_format=boto.s3.connection.S3Connection.DefaultCallingFormat
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings for the Sentry lib
 # Settings for the Sentry lib

+ 25 - 7
desktop/libs/aws/src/aws/client.py

@@ -17,6 +17,7 @@ from __future__ import absolute_import
 
 
 import boto
 import boto
 import boto.s3
 import boto.s3
+import boto.s3.connection
 import boto.utils
 import boto.utils
 
 
 from aws.conf import get_default_region, has_iam_metadata
 from aws.conf import get_default_region, has_iam_metadata
@@ -27,12 +28,17 @@ HTTP_SOCKET_TIMEOUT_S = 60
 
 
 class Client(object):
 class Client(object):
   def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
   def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
-               timeout=HTTP_SOCKET_TIMEOUT_S):
+               timeout=HTTP_SOCKET_TIMEOUT_S, proxy_address=None,
+               proxy_port=None, calling_format=None, is_secure=True):
     self._access_key_id = aws_access_key_id
     self._access_key_id = aws_access_key_id
     self._secret_access_key = aws_secret_access_key
     self._secret_access_key = aws_secret_access_key
     self._security_token = aws_security_token
     self._security_token = aws_security_token
     self._region = region.lower() if region else get_default_region()
     self._region = region.lower() if region else get_default_region()
     self._timeout = timeout
     self._timeout = timeout
+    self._proxy_address = proxy_address
+    self._proxy_port = proxy_port
+    self._calling_format = conf.DEFAULT_CALLING_FORMAT if calling_format is None else calling_format
+    self._is_secure = is_secure
 
 
     boto.config.add_section('Boto')
     boto.config.add_section('Boto')
     boto.config.set('Boto', 'http_socket_timeout', str(self._timeout))
     boto.config.set('Boto', 'http_socket_timeout', str(self._timeout))
@@ -51,15 +57,27 @@ class Client(object):
       aws_access_key_id=access_key_id,
       aws_access_key_id=access_key_id,
       aws_secret_access_key=secret_access_key,
       aws_secret_access_key=secret_access_key,
       aws_security_token=security_token,
       aws_security_token=security_token,
-      region=conf.REGION.get()
+      region=conf.REGION.get(),
+      proxy_address=conf.PROXY_ADDRESS.get(),
+      proxy_port=conf.PROXY_PORT.get(),
+      calling_format=conf.CALLING_FORMAT.get(),
+      is_secure=conf.IS_SECURE.get()
     )
     )
 
 
   def get_s3_connection(self):
   def get_s3_connection(self):
     # First attempt to connect via specified credentials
     # First attempt to connect via specified credentials
-    connection = boto.s3.connect_to_region(self._region,
-                                           aws_access_key_id=self._access_key_id,
-                                           aws_secret_access_key=self._secret_access_key,
-                                           security_token=self._security_token)
+    if self._proxy_address is not None and self._proxy_port is not None:
+      connection = boto.s3.connection.S3Connection(aws_access_key_id=self._access_key_id,
+                                        aws_secret_access_key=self._secret_access_key,
+                                        security_token=self._security_token,
+                                        is_secure=self._is_secure,
+                                        calling_format=self._calling_format,
+                                        proxy=self._proxy_address, proxy_port=self._proxy_port)
+    else:
+      connection = boto.s3.connect_to_region(self._region,
+                                             aws_access_key_id=self._access_key_id,
+                                             aws_secret_access_key=self._secret_access_key,
+                                             security_token=self._security_token)
 
 
     if connection is None:
     if connection is None:
       # If no connection, attemt to fallback to IAM instance metadata
       # If no connection, attemt to fallback to IAM instance metadata
@@ -68,4 +86,4 @@ class Client(object):
       if connection is None:
       if connection is None:
         raise ValueError('Can not construct S3 Connection for region %s' % self._region)
         raise ValueError('Can not construct S3 Connection for region %s' % self._region)
 
 
-    return connection
+    return connection

+ 27 - 4
desktop/libs/aws/src/aws/conf.py

@@ -44,12 +44,13 @@ def get_default_secret_key():
 def get_default_region():
 def get_default_region():
   return AWS_ACCOUNTS['default'].REGION.get()
   return AWS_ACCOUNTS['default'].REGION.get()
 
 
+DEFAULT_CALLING_FORMAT='boto.s3.connection.S3Connection.DefaultCallingFormat'
 
 
 AWS_ACCOUNTS = UnspecifiedConfigSection(
 AWS_ACCOUNTS = UnspecifiedConfigSection(
   'aws_accounts',
   'aws_accounts',
-  help='One entry for each AWS account',
+  help=_('One entry for each AWS account'),
   each=ConfigSection(
   each=ConfigSection(
-    help='Information about single AWS account',
+    help=_('Information about single AWS account'),
     members=dict(
     members=dict(
       ACCESS_KEY_ID=Config(
       ACCESS_KEY_ID=Config(
         key='access_key_id',
         key='access_key_id',
@@ -82,7 +83,7 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
         private=True,
         private=True,
       ),
       ),
       ALLOW_ENVIRONMENT_CREDENTIALS=Config(
       ALLOW_ENVIRONMENT_CREDENTIALS=Config(
-        help='Allow to use environment sources of credentials (environment variables, EC2 profile).',
+        help=_('Allow to use environment sources of credentials (environment variables, EC2 profile).'),
         key='allow_environment_credentials',
         key='allow_environment_credentials',
         default=True,
         default=True,
         type=coerce_bool
         type=coerce_bool
@@ -91,6 +92,28 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
         key='region',
         key='region',
         default='us-east-1',
         default='us-east-1',
         type=str
         type=str
+      ),
+      PROXY_ADDRESS=Config(
+        help=_('Alternate address for endpoint.'),
+        key='proxy_address',
+        default=None,
+        type=str
+      ),
+      PROXY_PORT=Config(
+        help=_('Alternate port for endpoint.'),
+        key='proxy_port',
+        default=None,
+        type=int
+      ),
+      CALLING_FORMAT=Config(
+        key='calling_format',
+        default=DEFAULT_CALLING_FORMAT,
+        type=str
+      ),
+      IS_SECURE=Config(
+        key='is_secure',
+        default=True,
+        type=coerce_bool
       )
       )
     )
     )
   )
   )
@@ -122,4 +145,4 @@ def config_validator(user):
       if region_name not in region_names:
       if region_name not in region_names:
         res.append(('aws.aws_accounts.%s.region' % name, 'Unknown region %s' % region_name))
         res.append(('aws.aws_accounts.%s.region' % name, 'Unknown region %s' % region_name))
 
 
-  return res
+  return res