Explorar el Código

[aws] Adding get key test for RazS3Connection

Romain Rigaux hace 4 años
padre
commit
27d8ce8dd5

+ 2 - 2
desktop/libs/aws/src/aws/client.py

@@ -21,7 +21,7 @@ import os
 import boto
 
 from aws import conf as aws_conf
-from aws.s3.s3connection import url_client_connect_to_region, RazSignedUrlS3Connection
+from aws.s3.s3connection import url_client_connect_to_region, RazS3Connection
 from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 
 from desktop.conf import RAZ
@@ -42,7 +42,7 @@ def get_credential_provider(identifier, user):
 
 def _make_client(identifier, user):
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
-    s3_client = RazSignedUrlS3Connection()  # Note: AWS configuration is fully skipped
+    s3_client = RazS3Connection()  # Note: AWS configuration is fully skipped
     s3_client_expiration = None
   else:
     client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None

+ 6 - 8
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -61,7 +61,9 @@ class SignedUrlS3Connection(S3Connection):
                 provider='aws', bucket_class=Bucket, security_token=None,
                 suppress_consec_slashes=True, anon=False,
                 validate_certs=None, profile_name=None):
-    # anon = True # For Raz
+    # For Raz
+    # anon = True
+    # TODO: handle properly how to build a client without any auth without having get_auth_handler() fail
     super(SignedUrlS3Connection, self).__init__(
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
@@ -73,15 +75,13 @@ class SignedUrlS3Connection(S3Connection):
                 validate_certs=validate_certs, profile_name=profile_name)
 
 
-class RazSignedUrlS3Connection(SignedUrlS3Connection):
+class RazS3Connection(SignedUrlS3Connection):
   """
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Some operations can be denied depending on the privileges of the users in Ranger.
 
   Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
   """
-
-
   def make_request(self, method, bucket='', key='', headers=None, data='',
                     query_args=None, sender=None, override_num_retries=None,
                     retry_handler=None):
@@ -108,7 +108,7 @@ class RazSignedUrlS3Connection(SignedUrlS3Connection):
     # TODO:
     # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
     # update some of its attributes like in SelfSignedUrlS3Connection.
-    #
+
     # e.g.
     # signed_url = self.get_url_request(...)
     # update or recreate `http_request`
@@ -123,9 +123,7 @@ class RazSignedUrlS3Connection(SignedUrlS3Connection):
         'key': key
     }
 
-    # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-    tmp_url = 'https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210506%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210506T191959Z&X-Amz-Expires=1000&X-Amz-SignedHeaders=host&X-Amz-Signature=6d83f8ed913bd3316a7185df7c49480300032c381e93356610a07ed474fc21d3'
-    # tmp_url = self.generate_url(1000, method, **kwargs)
+    tmp_url = self.get_url_request(action='GET', bucket_name=bucket, object_name=key)
     LOG.debug(tmp_url)
 
     http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')

+ 39 - 1
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -21,7 +21,7 @@ from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true
 
 from aws.client import _make_client
-from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection
+from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
 from aws.s3.s3test_utils import S3TestBase
 
 
@@ -68,6 +68,44 @@ class TestSelfSignedUrlS3Connection():
 
 
 
+class TestRazS3Connection():
+
+  def test_get_file(self):
+    with patch('aws.s3.s3connection.RazS3Connection.get_url_request') as get_url_request:
+      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
+        with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
+
+          get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
+              'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
+              '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
+          _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
+
+          client = RazS3Connection()
+          http_request = Mock(
+            path='/gethue/data/customer.csv',
+            protocol='https',
+            host='s3.amazonaws.com'
+          )
+          client.build_base_http_request = Mock(return_value=http_request)
+
+          buckets = client.make_request(method='GET', bucket='gethue', key='data/customer.csv',)
+
+          assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', buckets)
+          _mexe.assert_called_with(http_request, None, None, retry_handler=None)
+
+          assert_equal('https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv', http_request.path)
+          assert_equal(
+            {
+              'AWSAccessKeyId': 'AKIA23E77ZX2HVY76YGL',
+              'Signature': '3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D',
+              'Expires': '1617207304'
+            },
+            http_request.headers
+          )
+
+
+# -----------------------------------------------------------------------------------------------------------
+
 class TestSelfSignedUrlClient():
 
   def setUp(self):