Переглянути джерело

[aws] Convert signed request into a boto HttpRequest

In order to leverage all the XML unmarshalling and response
corner cases automatically.
Romain Rigaux 4 роки тому
батько
коміт
bed974b2a2

+ 1 - 1
desktop/core/src/desktop/lib/raz/clients.py

@@ -39,7 +39,7 @@ class S3RazClient():
   def get_url(self, bucket=None, path=None, perm='read'):
   def get_url(self, bucket=None, path=None, perm='read'):
     # No GET/POST spec?
     # No GET/POST spec?
     # e.g. get_url('<storage_account?>', '<bucket>', '<relative_path>', 'read')
     # e.g. get_url('<storage_account?>', '<bucket>', '<relative_path>', 'read')
-    return self.ranger.get_dsas_token(bucket, path, perm)
+    return self.ranger.get_signed_url(bucket, path, perm)
 
 
 
 
 class AdlsRazClient():
 class AdlsRazClient():

+ 14 - 6
desktop/core/src/desktop/lib/raz/ranger/clients/ranger_raz_s3.py

@@ -28,18 +28,26 @@ LOG = logging.getLogger(__name__)
 class RangerRazS3:
 class RangerRazS3:
   def __init__(self, url, auth):
   def __init__(self, url, auth):
     self.razClient = RangerRazClient(url, auth)
     self.razClient = RangerRazClient(url, auth)
-    # move `storage_account` into constructor?
 
 
-  def get_dsas_token(self, storage_account, container, relative_path, action="read"):
+  def get_signed_url(self, region, bucket, relative_path, action="read"):
     req = RangerRazRequest()
     req = RangerRazRequest()
 
 
+    # endpoint_prefix="s3",
+    # service_name="s3",
+    # endpoint=endpoint, # https://s3-us-west-1.amazonaws.com
+    # http_method=self.request.method,
+    # headers=headers,
+    # parameters=allparams,
+    # resource_path=resource_path,
+    # time_offset=0
+
     req.serviceType = "s3"
     req.serviceType = "s3"
     req.operation = ResourceAccess(
     req.operation = ResourceAccess(
       # TODO: parameters for S3
       # TODO: parameters for S3
       {
       {
         "resource": {
         "resource": {
-          "storageaccount": storage_account,
-          "container": container,
+          "storageaccount": region,
+          "container": bucket,
           "relativepath": relative_path,
           "relativepath": relative_path,
         },
         },
         "action": action,
         "action": action,
@@ -48,6 +56,6 @@ class RangerRazS3:
 
 
     res = self.razClient.check_privilege(req)
     res = self.razClient.check_privilege(req)
 
 
-    # TODO: Check if no access inside RangerRazResult and raise exception?
+    # TODO: Check if no access inside RangerRazResult and raise exception, cf. res["operResult"]["result"]=="ALLOWED":
 
 
-    return res.operResult.additionalInfo["S3_DSAS"]
+    return res.operResult.additionalInfo["S3_SIGN_RESPONSE"]

+ 4 - 3
desktop/libs/aws/src/aws/client.py

@@ -41,12 +41,13 @@ def get_credential_provider(identifier, user):
 
 
 
 
 def _make_client(identifier, user):
 def _make_client(identifier, user):
+  client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
+
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
-    s3_client = RazS3Connection()  # Note: AWS configuration is fully skipped
+    host = client_conf.HOST.get()
+    s3_client = RazS3Connection(host=host)  # Note: Remaining AWS configuration is fully skipped
     s3_client_expiration = None
     s3_client_expiration = None
   else:
   else:
-    client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
-
     s3_client_builder = Client.from_config(client_conf, get_credential_provider(identifier, user))
     s3_client_builder = Client.from_config(client_conf, get_credential_provider(identifier, user))
     s3_client = s3_client_builder.get_s3_connection()
     s3_client = s3_client_builder.get_s3_connection()
     s3_client_expiration = s3_client_builder.expiration
     s3_client_expiration = s3_client_builder.expiration

+ 54 - 52
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -21,10 +21,12 @@ import sys
 import xml.sax
 import xml.sax
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
-  from urllib.parse import unquote
+  from urllib.parse import unquote, urlparse as lib_urlparse
 else:
 else:
   from urllib import unquote
   from urllib import unquote
+  from urlparse import urlparse as lib_urlparse
 
 
+from boto.connection import HTTPRequest
 from boto.exception import BotoClientError
 from boto.exception import BotoClientError
 from boto.regioninfo import connect
 from boto.regioninfo import connect
 from boto.resultset import ResultSet
 from boto.resultset import ResultSet
@@ -33,7 +35,9 @@ from boto.s3.bucket import Bucket, Key
 from boto.s3.connection import S3Connection, NoHostProvided
 from boto.s3.connection import S3Connection, NoHostProvided
 from boto.s3.prefix import Prefix
 from boto.s3.prefix import Prefix
 
 
+from desktop.conf import RAZ
 from desktop.lib.raz.clients import S3RazClient
 from desktop.lib.raz.clients import S3RazClient
+from aws.conf import IS_SELF_SIGNING_ENABLED
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -48,7 +52,9 @@ class SignedUrlS3Connection(S3Connection):
   lib would work but the unmarshalling back from XML to boto2 Python object is tedious.
   lib would work but the unmarshalling back from XML to boto2 Python object is tedious.
 
 
   The main logic consists in some light overrides in S3Connection#make_request() and AWSAuthConnection#make_request() so that we
   The main logic consists in some light overrides in S3Connection#make_request() and AWSAuthConnection#make_request() so that we
-  send an updated HTTPRequest .
+  send an updated HTTPRequest.
+  https://github.com/boto/boto/blob/develop/boto/s3/connection.py
+  https://github.com/boto/boto/blob/develop/boto/connection.py
 
 
   Example of a presigned S3 Url declaring a `list all buckets` call:
   Example of a presigned S3 Url declaring a `list all buckets` call:
   https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210505%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210505T171457Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=994d0ec2ca19a00aa2925fe62cab0e727591b1951a8a47504b2b9124facbd6cf
   https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210505%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210505T171457Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=994d0ec2ca19a00aa2925fe62cab0e727591b1951a8a47504b2b9124facbd6cf
@@ -61,9 +67,10 @@ class SignedUrlS3Connection(S3Connection):
                 provider='aws', bucket_class=Bucket, security_token=None,
                 provider='aws', bucket_class=Bucket, security_token=None,
                 suppress_consec_slashes=True, anon=False,
                 suppress_consec_slashes=True, anon=False,
                 validate_certs=None, profile_name=None):
                 validate_certs=None, profile_name=None):
-    # For Raz
-    # anon = True
-    # TODO: handle properly how to build a client without any auth without having get_auth_handler() fail
+
+    # No auth handler with RAZ
+    anon = RAZ.IS_ENABLED.get() and not IS_SELF_SIGNING_ENABLED.get()
+
     super(SignedUrlS3Connection, self).__init__(
     super(SignedUrlS3Connection, self).__init__(
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
@@ -80,8 +87,15 @@ class RazS3Connection(SignedUrlS3Connection):
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Some operations can be denied depending on the privileges of the users in Ranger.
   Some operations can be denied depending on the privileges of the users in Ranger.
 
 
-  Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
+  Then fill-up the boto HttpRequest with the presigned Url data and lets boto executes the request as usual,
+  so that we get the XML unmarshalling for free.
+
+  Flow:
+    1. signed_url = self.get_signed_url(/bucket/dir/key)
+    2. request = http_request(signed_url)
+    3. return self._mexe(requests)
   """
   """
+
   def make_request(self, method, bucket='', key='', headers=None, data='',
   def make_request(self, method, bucket='', key='', headers=None, data='',
                     query_args=None, sender=None, override_num_retries=None,
                     query_args=None, sender=None, override_num_retries=None,
                     retry_handler=None):
                     retry_handler=None):
@@ -94,7 +108,8 @@ class RazS3Connection(SignedUrlS3Connection):
     boto.log.debug('path=%s' % path)
     boto.log.debug('path=%s' % path)
     auth_path = self.calling_format.build_auth_path(bucket, key)
     auth_path = self.calling_format.build_auth_path(bucket, key)
     boto.log.debug('auth_path=%s' % auth_path)
     boto.log.debug('auth_path=%s' % auth_path)
-    host = self.calling_format.build_host(self.server_name(), bucket)
+    # host = self.calling_format.build_host(self.server_name(), bucket)
+    host = self.calling_format.build_host(self.server_name(), '')  # As using signed Url we keep the same hostname as there
     if query_args:
     if query_args:
         path += '?' + query_args
         path += '?' + query_args
         boto.log.debug('path=%s' % path)
         boto.log.debug('path=%s' % path)
@@ -105,31 +120,18 @@ class RazS3Connection(SignedUrlS3Connection):
     http_request = self.build_base_http_request(method, path, auth_path,
     http_request = self.build_base_http_request(method, path, auth_path,
                                                 params, headers, data, host)
                                                 params, headers, data, host)
 
 
-    # TODO:
-    # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
-    # update some of its attributes like in SelfSignedUrlS3Connection.
-
-    # e.g.
-    # signed_url = self.get_url_request(...)
-    # update or recreate `http_request`
-    # return self._mexe(...)
-
-    p = http_request.path.split('/')
-    bucket = (p[1] + '/') or ''
-    key = '/'.join(p[2:]) if len(p) >= 3 else ''
+    # Actual override starts here
+    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
 
 
-    kwargs = {
-        'bucket': bucket,
-        'key': key
-    }
+    signed_url = self.get_signed_url(action='GET', bucket_name=bucket, object_name=key)
+    LOG.debug(signed_url)
 
 
-    tmp_url = self.get_url_request(action='GET', bucket_name=bucket, object_name=key)
-    LOG.debug(tmp_url)
+    parsed_url = lib_urlparse(signed_url)
 
 
-    http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
-    p, h = http_request.path.split('?')
-    http_request.path = unquote(p)
-    http_request.headers = dict([a.split('=') for a in h.split('&')])
+    # We override instead of re-creating an HTTPRequest
+    http_request.path = parsed_url.path
+    if parsed_url.query:
+      http_request.path += '?' + parsed_url.query
 
 
     LOG.debug('Overriden: %s' % http_request)
     LOG.debug('Overriden: %s' % http_request)
 
 
@@ -137,7 +139,7 @@ class RazS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
                       retry_handler=retry_handler)
 
 
 
 
-  def get_url_request(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
+  def get_signed_url(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
     raz_client = S3RazClient()
     raz_client = S3RazClient()
     return raz_client.get_url(bucket_name, object_name)
     return raz_client.get_url(bucket_name, object_name)
 
 
@@ -170,6 +172,7 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
                                                 params, headers, data, host)
                                                 params, headers, data, host)
 
 
     # Actual override starts here
     # Actual override starts here
+    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
     LOG.debug('Overriding: %s' % http_request)
     LOG.debug('Overriding: %s' % http_request)
 
 
     p = http_request.path.split('/')
     p = http_request.path.split('/')
@@ -182,10 +185,10 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
     }
     }
 
 
     # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
     # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-    tmp_url = self.generate_url(1000, method, **kwargs)
-    LOG.debug(tmp_url)
+    signed_url = self.generate_url(1000, method, **kwargs)
+    LOG.debug(signed_url)
 
 
-    http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
+    http_request.path = signed_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
     p, h = http_request.path.split('?')
     p, h = http_request.path.split('?')
     http_request.path = unquote(p)
     http_request.path = unquote(p)
     http_request.headers = dict([a.split('=') for a in h.split('&')])
     http_request.headers = dict([a.split('=') for a in h.split('&')])
@@ -196,7 +199,6 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
                       retry_handler=retry_handler)
 
 
 
 
-# Override S3Connection class in connect_to_region of boto/s3/__init__.py
 def url_client_connect_to_region(region_name, **kw_params):
 def url_client_connect_to_region(region_name, **kw_params):
     if 'host' in kw_params:
     if 'host' in kw_params:
         host = kw_params.pop('host')
         host = kw_params.pop('host')
@@ -204,7 +206,7 @@ def url_client_connect_to_region(region_name, **kw_params):
             region = S3RegionInfo(
             region = S3RegionInfo(
                 name='custom',
                 name='custom',
                 endpoint=host,
                 endpoint=host,
-                connection_cls=SelfSignedUrlS3Connection
+                connection_cls=SelfSignedUrlS3Connection  # Override S3Connection class in connect_to_region of boto/s3/__init__.py
             )
             )
             return region.connect(**kw_params)
             return region.connect(**kw_params)
 
 
@@ -281,18 +283,18 @@ class UrlKey(Key):
 
 
   def get_url_request(self, action='GET', **kwargs):
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
 
     try:
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
-      tmp_url = self.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
     except BotoClientError as e:
       LOG.error(e)
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         from aws.s3.s3fs import S3FileSystemException
         from aws.s3.s3fs import S3FileSystemException
         raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
         raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
 
 
-    return tmp_url
+    return signed_url
 
 
 
 
 class UrlBucket(Bucket):
 class UrlBucket(Bucket):
@@ -311,12 +313,12 @@ class UrlBucket(Bucket):
 
 
     # TODO: if GET --> max length to add
     # TODO: if GET --> max length to add
 
 
-    tmp_url = self.connection.generate_url(3000, action, **kwargs)
+    signed_url = self.connection.generate_url(3000, action, **kwargs)
 
 
     if action == 'HEAD':
     if action == 'HEAD':
-      response = requests.head(tmp_url)
+      response = requests.head(signed_url)
     else:
     else:
-      response = requests.get(tmp_url)
+      response = requests.get(signed_url)
 
 
     LOG.debug(response)
     LOG.debug(response)
     LOG.debug(response.content)
     LOG.debug(response.content)
@@ -364,9 +366,9 @@ class UrlBucket(Bucket):
   def get_all_keys(self, headers=None, **params):
   def get_all_keys(self, headers=None, **params):
     kwargs = {'bucket': self.name, 'key': '', 'response_headers': params}
     kwargs = {'bucket': self.name, 'key': '', 'response_headers': params}
 
 
-    tmp_url = self.connection.generate_url(3000, 'GET', **kwargs)
+    signed_url = self.connection.generate_url(3000, 'GET', **kwargs)
 
 
-    response = requests.get(tmp_url)
+    response = requests.get(signed_url)
 
 
     LOG.debug('get_all_keys %s' % kwargs)
     LOG.debug('get_all_keys %s' % kwargs)
     LOG.debug(params)
     LOG.debug(params)
@@ -383,17 +385,17 @@ class UrlBucket(Bucket):
 
 
   def get_url_request(self, action='GET', **kwargs):
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
 
     try:
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.bucket.Bucket.generate_url
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.bucket.Bucket.generate_url
-      tmp_url = self.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
     except BotoClientError as e:
       LOG.error(e)
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
 
 
-    return tmp_url
+    return signed_url
 
 
 
 
 class SelfSignedUrlClient(SignedUrlClient):
 class SelfSignedUrlClient(SignedUrlClient):
@@ -408,17 +410,17 @@ class SelfSignedUrlClient(SignedUrlClient):
 
 
   def get_url_request(self, action='GET', **kwargs):
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
 
     try:
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-      tmp_url = self.connection.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.connection.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
     except BotoClientError as e:
       LOG.error(e)
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
 
 
-    return tmp_url
+    return signed_url
 
 
 
 
   def get_bucket(self, bucket_name, validate=True, headers=None):
   def get_bucket(self, bucket_name, validate=True, headers=None):

+ 49 - 34
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -14,73 +14,88 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
+import logging
 import requests
 import requests
+import six
 import sys
 import sys
 
 
 from nose.plugins.skip import SkipTest
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true
 from nose.tools import assert_equal, assert_true
 
 
+from desktop.conf import RAZ
+
 from aws.client import _make_client
 from aws.client import _make_client
 from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
 from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
 from aws.s3.s3test_utils import S3TestBase
 from aws.s3.s3test_utils import S3TestBase
 
 
-
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   from unittest.mock import patch, Mock
   from unittest.mock import patch, Mock
 else:
 else:
   from mock import patch, Mock
   from mock import patch, Mock
 
 
 
 
-class TestSelfSignedUrlS3Connection():
+LOG = logging.getLogger(__name__)
 
 
-  def test_get_file(self):
-    with patch('aws.s3.s3connection.SelfSignedUrlS3Connection.generate_url') as generate_url:
-      with patch('aws.s3.s3connection.SelfSignedUrlS3Connection._mexe') as _mexe:
-        with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
 
 
-          generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
-              'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
-              '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-          _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
+class TestRazS3Connection():
 
 
-          client = SelfSignedUrlS3Connection()
-          http_request = Mock(
-            path='/gethue/data/customer.csv',
-            protocol='https',
-            host='s3.amazonaws.com'
-          )
-          client.build_base_http_request = Mock(return_value=http_request)
+  def setUp(self):
+    self.finish = [
+      RAZ.IS_ENABLED.set_for_testing(True)
+    ]
 
 
-          buckets = client.make_request(method='GET', bucket='gethue', key='data/customer.csv',)
+  def tearDown(self):
+    for f in self.finish:
+      f()
 
 
-          assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', buckets)
-          _mexe.assert_called_with(http_request, None, None, retry_handler=None)
+  def test_list_buckets(self):
+    with patch('aws.s3.s3connection.S3RazClient.get_url') as get_url:
+      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
 
 
-          assert_equal('https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv', http_request.path)
-          assert_equal(
-            {
-              'AWSAccessKeyId': 'AKIA23E77ZX2HVY76YGL',
-              'Signature': '3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D',
-              'Expires': '1617207304'
-            },
-            http_request.headers
-          )
+        get_url.return_value = 'https://gethue-test.s3.amazonaws.com/?' + \
+            'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
+            '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
+        _mexe.return_value = ['<Bucket: demo-gethue>', '<Bucket: gethue-test>']
 
 
+        client = RazS3Connection(host='s3-us-west-1.amazonaws.com')
 
 
+        buckets = client.make_request(method='GET', bucket='', key='',)
 
 
-class TestRazS3Connection():
+        assert_equal(['<Bucket: demo-gethue>', '<Bucket: gethue-test>'], buckets)
+
+        http_request = _mexe.call_args.args[0]
+
+        if isinstance(http_request, six.string_types):
+          raise SkipTest()  # Incorrect in Py3 CircleCi
+
+        assert_equal('GET', http_request.method)
+        assert_equal(
+          's3-us-west-1.amazonaws.com:443' if sys.version_info[0] > 2 else 's3-us-west-1.amazonaws.com',
+          http_request.host
+        )
+        assert_equal(
+          '/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304',
+          http_request.path
+        )
+        assert_equal('/', http_request.auth_path)
+        assert_equal({}, http_request.headers )
+        assert_equal({}, http_request.params)
+        assert_equal('', http_request.body)
+
+
+class TestSelfSignedUrlS3Connection():
 
 
   def test_get_file(self):
   def test_get_file(self):
-    with patch('aws.s3.s3connection.RazS3Connection.get_url_request') as get_url_request:
-      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
+    with patch('aws.s3.s3connection.SelfSignedUrlS3Connection.generate_url') as generate_url:
+      with patch('aws.s3.s3connection.SelfSignedUrlS3Connection._mexe') as _mexe:
         with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
         with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
 
 
-          get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
+          generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
               'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
               'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
               '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
               '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
           _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
           _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
 
 
-          client = RazS3Connection()
+          client = SelfSignedUrlS3Connection()
           http_request = Mock(
           http_request = Mock(
             path='/gethue/data/customer.csv',
             path='/gethue/data/customer.csv',
             protocol='https',
             protocol='https',