|
|
@@ -15,12 +15,19 @@
|
|
|
# limitations under the License.
|
|
|
from __future__ import absolute_import
|
|
|
|
|
|
+import logging
|
|
|
+import os
|
|
|
+
|
|
|
import boto
|
|
|
import boto.s3
|
|
|
import boto.s3.connection
|
|
|
import boto.utils
|
|
|
|
|
|
from aws.conf import get_default_region, has_iam_metadata, DEFAULT_CALLING_FORMAT
|
|
|
+from aws.s3.s3fs import S3FileSystemException
|
|
|
+
|
|
|
+
|
|
|
+LOG = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
HTTP_SOCKET_TIMEOUT_S = 60
|
|
|
@@ -28,15 +35,18 @@ HTTP_SOCKET_TIMEOUT_S = 60
|
|
|
|
|
|
class Client(object):
|
|
|
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
|
|
|
- timeout=HTTP_SOCKET_TIMEOUT_S, proxy_address=None,
|
|
|
- proxy_port=None, calling_format=None, is_secure=True):
|
|
|
+ timeout=HTTP_SOCKET_TIMEOUT_S, host=None, proxy_address=None, proxy_port=None, proxy_user=None,
|
|
|
+ proxy_pass=None, calling_format=None, is_secure=True):
|
|
|
self._access_key_id = aws_access_key_id
|
|
|
self._secret_access_key = aws_secret_access_key
|
|
|
self._security_token = aws_security_token
|
|
|
self._region = region.lower() if region else get_default_region()
|
|
|
self._timeout = timeout
|
|
|
+ self._host = host
|
|
|
self._proxy_address = proxy_address
|
|
|
self._proxy_port = proxy_port
|
|
|
+ self._proxy_user = proxy_user
|
|
|
+ self._proxy_pass = proxy_pass
|
|
|
self._calling_format = DEFAULT_CALLING_FORMAT if calling_format is None else calling_format
|
|
|
self._is_secure = is_secure
|
|
|
|
|
|
@@ -61,37 +71,55 @@ class Client(object):
|
|
|
aws_secret_access_key=secret_access_key,
|
|
|
aws_security_token=security_token,
|
|
|
region=conf.REGION.get(),
|
|
|
+ host=conf.HOST.get(),
|
|
|
proxy_address=conf.PROXY_ADDRESS.get(),
|
|
|
proxy_port=conf.PROXY_PORT.get(),
|
|
|
+ proxy_user=conf.PROXY_USER.get(),
|
|
|
+ proxy_pass=conf.PROXY_PASS.get(),
|
|
|
calling_format=conf.CALLING_FORMAT.get(),
|
|
|
is_secure=conf.IS_SECURE.get()
|
|
|
)
|
|
|
|
|
|
def get_s3_connection(self):
|
|
|
- # First attempt to connect via specified credentials
|
|
|
- if self._proxy_address is not None and self._proxy_port is not None:
|
|
|
- connection = boto.s3.connection.S3Connection(aws_access_key_id=self._access_key_id,
|
|
|
- aws_secret_access_key=self._secret_access_key,
|
|
|
- security_token=self._security_token,
|
|
|
- is_secure=self._is_secure,
|
|
|
- calling_format=self._calling_format,
|
|
|
- proxy=self._proxy_address,
|
|
|
- proxy_port=self._proxy_port)
|
|
|
- elif self._region:
|
|
|
- connection = boto.s3.connect_to_region(self._region,
|
|
|
- aws_access_key_id=self._access_key_id,
|
|
|
- aws_secret_access_key=self._secret_access_key,
|
|
|
- security_token=self._security_token)
|
|
|
- else:
|
|
|
- connection = boto.s3.connection.S3Connection(aws_access_key_id=self._access_key_id,
|
|
|
- aws_secret_access_key=self._secret_access_key,
|
|
|
- security_token=self._security_token)
|
|
|
+ # Use V4 signature support by default
|
|
|
+ os.environ['S3_USE_SIGV4'] = 'True'
|
|
|
+
|
|
|
+ kwargs = {
|
|
|
+ 'aws_access_key_id': self._access_key_id,
|
|
|
+ 'aws_secret_access_key': self._secret_access_key,
|
|
|
+ 'security_token': self._security_token,
|
|
|
+ 'is_secure': self._is_secure,
|
|
|
+ 'calling_format': self._calling_format
|
|
|
+ }
|
|
|
+
|
|
|
+ # Add proxy if configured
|
|
|
+ if self._proxy_address is not None:
|
|
|
+ kwargs.update({'proxy': self._proxy_address})
|
|
|
+ if self._proxy_port is not None:
|
|
|
+ kwargs.update({'proxy_port': self._proxy_port})
|
|
|
+ if self._proxy_user is not None:
|
|
|
+ kwargs.update({'proxy_user': self._proxy_user})
|
|
|
+ if self._proxy_pass is not None:
|
|
|
+ kwargs.update({'proxy_pass': self._proxy_pass})
|
|
|
+
|
|
|
+ # Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM
|
|
|
+ try:
|
|
|
+ if self._host is not None:
|
|
|
+ kwargs.update({'host': self._host})
|
|
|
+ connection = boto.s3.connection.S3Connection(**kwargs)
|
|
|
+ elif self._region:
|
|
|
+ connection = boto.s3.connect_to_region(self._region, **kwargs)
|
|
|
+ else:
|
|
|
+ connection = boto.s3.connection.S3Connection(**kwargs)
|
|
|
+ except Exception, e:
|
|
|
+ LOG.exception(e)
|
|
|
+ raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
|
|
|
|
|
|
if connection is None:
|
|
|
# If no connection, attemt to fallback to IAM instance metadata
|
|
|
connection = boto.connect_s3()
|
|
|
|
|
|
if connection is None:
|
|
|
- raise ValueError('Can not construct S3 Connection for region %s' % self._region)
|
|
|
+ raise S3FileSystemException('Can not construct S3 Connection for region %s' % self._region)
|
|
|
|
|
|
return connection
|