ソースを参照

[aws] Adding get key test for RazS3Connection

Romain Rigaux 4 年 前
コミット
27d8ce8dd5

+ 2 - 2
desktop/libs/aws/src/aws/client.py

@@ -21,7 +21,7 @@ import os
 import boto
 import boto
 
 
 from aws import conf as aws_conf
 from aws import conf as aws_conf
-from aws.s3.s3connection import url_client_connect_to_region, RazSignedUrlS3Connection
+from aws.s3.s3connection import url_client_connect_to_region, RazS3Connection
 from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 
 
 from desktop.conf import RAZ
 from desktop.conf import RAZ
@@ -42,7 +42,7 @@ def get_credential_provider(identifier, user):
 
 
 def _make_client(identifier, user):
 def _make_client(identifier, user):
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
-    s3_client = RazSignedUrlS3Connection()  # Note: AWS configuration is fully skipped
+    s3_client = RazS3Connection()  # Note: AWS configuration is fully skipped
     s3_client_expiration = None
     s3_client_expiration = None
   else:
   else:
     client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
     client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None

+ 6 - 8
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -61,7 +61,9 @@ class SignedUrlS3Connection(S3Connection):
                 provider='aws', bucket_class=Bucket, security_token=None,
                 provider='aws', bucket_class=Bucket, security_token=None,
                 suppress_consec_slashes=True, anon=False,
                 suppress_consec_slashes=True, anon=False,
                 validate_certs=None, profile_name=None):
                 validate_certs=None, profile_name=None):
-    # anon = True # For Raz
+    # For Raz
+    # anon = True
+    # TODO: handle properly how to build a client without any auth without having get_auth_handler() fail
     super(SignedUrlS3Connection, self).__init__(
     super(SignedUrlS3Connection, self).__init__(
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
@@ -73,15 +75,13 @@ class SignedUrlS3Connection(S3Connection):
                 validate_certs=validate_certs, profile_name=profile_name)
                 validate_certs=validate_certs, profile_name=profile_name)
 
 
 
 
-class RazSignedUrlS3Connection(SignedUrlS3Connection):
+class RazS3Connection(SignedUrlS3Connection):
   """
   """
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Some operations can be denied depending on the privileges of the users in Ranger.
   Some operations can be denied depending on the privileges of the users in Ranger.
 
 
   Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
   Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
   """
   """
-
-
   def make_request(self, method, bucket='', key='', headers=None, data='',
   def make_request(self, method, bucket='', key='', headers=None, data='',
                     query_args=None, sender=None, override_num_retries=None,
                     query_args=None, sender=None, override_num_retries=None,
                     retry_handler=None):
                     retry_handler=None):
@@ -108,7 +108,7 @@ class RazSignedUrlS3Connection(SignedUrlS3Connection):
     # TODO:
     # TODO:
     # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
     # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
     # update some of its attributes like in SelfSignedUrlS3Connection.
     # update some of its attributes like in SelfSignedUrlS3Connection.
-    #
+
     # e.g.
     # e.g.
     # signed_url = self.get_url_request(...)
     # signed_url = self.get_url_request(...)
     # update or recreate `http_request`
     # update or recreate `http_request`
@@ -123,9 +123,7 @@ class RazSignedUrlS3Connection(SignedUrlS3Connection):
         'key': key
         'key': key
     }
     }
 
 
-    # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-    tmp_url = 'https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210506%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210506T191959Z&X-Amz-Expires=1000&X-Amz-SignedHeaders=host&X-Amz-Signature=6d83f8ed913bd3316a7185df7c49480300032c381e93356610a07ed474fc21d3'
-    # tmp_url = self.generate_url(1000, method, **kwargs)
+    tmp_url = self.get_url_request(action='GET', bucket_name=bucket, object_name=key)
     LOG.debug(tmp_url)
     LOG.debug(tmp_url)
 
 
     http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
     http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')

+ 39 - 1
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -21,7 +21,7 @@ from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true
 from nose.tools import assert_equal, assert_true
 
 
 from aws.client import _make_client
 from aws.client import _make_client
-from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection
+from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
 from aws.s3.s3test_utils import S3TestBase
 from aws.s3.s3test_utils import S3TestBase
 
 
 
 
@@ -68,6 +68,44 @@ class TestSelfSignedUrlS3Connection():
 
 
 
 
 
 
+class TestRazS3Connection():
+
+  def test_get_file(self):
+    with patch('aws.s3.s3connection.RazS3Connection.get_url_request') as get_url_request:
+      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
+        with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
+
+          get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
+              'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
+              '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
+          _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
+
+          client = RazS3Connection()
+          http_request = Mock(
+            path='/gethue/data/customer.csv',
+            protocol='https',
+            host='s3.amazonaws.com'
+          )
+          client.build_base_http_request = Mock(return_value=http_request)
+
+          buckets = client.make_request(method='GET', bucket='gethue', key='data/customer.csv',)
+
+          assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', buckets)
+          _mexe.assert_called_with(http_request, None, None, retry_handler=None)
+
+          assert_equal('https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv', http_request.path)
+          assert_equal(
+            {
+              'AWSAccessKeyId': 'AKIA23E77ZX2HVY76YGL',
+              'Signature': '3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D',
+              'Expires': '1617207304'
+            },
+            http_request.headers
+          )
+
+
+# -----------------------------------------------------------------------------------------------------------
+
 class TestSelfSignedUrlClient():
 class TestSelfSignedUrlClient():
 
 
   def setUp(self):
   def setUp(self):