Ver código fonte

[aws] Convert signed request into a boto HttpRequest

In order to leverage all the XML unmarshalling and response
corner cases automatically.
Romain Rigaux 4 anos atrás
pai
commit
bed974b2a2

+ 1 - 1
desktop/core/src/desktop/lib/raz/clients.py

@@ -39,7 +39,7 @@ class S3RazClient():
   def get_url(self, bucket=None, path=None, perm='read'):
     # No GET/POST spec?
     # e.g. get_url('<storage_account?>', '<bucket>', '<relative_path>', 'read')
-    return self.ranger.get_dsas_token(bucket, path, perm)
+    return self.ranger.get_signed_url(bucket, path, perm)
 
 
 class AdlsRazClient():

+ 14 - 6
desktop/core/src/desktop/lib/raz/ranger/clients/ranger_raz_s3.py

@@ -28,18 +28,26 @@ LOG = logging.getLogger(__name__)
 class RangerRazS3:
   def __init__(self, url, auth):
     self.razClient = RangerRazClient(url, auth)
-    # move `storage_account` into constructor?
 
-  def get_dsas_token(self, storage_account, container, relative_path, action="read"):
+  def get_signed_url(self, region, bucket, relative_path, action="read"):
     req = RangerRazRequest()
 
+    # endpoint_prefix="s3",
+    # service_name="s3",
+    # endpoint=endpoint, # https://s3-us-west-1.amazonaws.com
+    # http_method=self.request.method,
+    # headers=headers,
+    # parameters=allparams,
+    # resource_path=resource_path,
+    # time_offset=0
+
     req.serviceType = "s3"
     req.operation = ResourceAccess(
       # TODO: parameters for S3
       {
         "resource": {
-          "storageaccount": storage_account,
-          "container": container,
+          "storageaccount": region,
+          "container": bucket,
           "relativepath": relative_path,
         },
         "action": action,
@@ -48,6 +56,6 @@ class RangerRazS3:
 
     res = self.razClient.check_privilege(req)
 
-    # TODO: Check if no access inside RangerRazResult and raise exception?
+    # TODO: Check if no access inside RangerRazResult and raise exception, cf. res["operResult"]["result"]=="ALLOWED":
 
-    return res.operResult.additionalInfo["S3_DSAS"]
+    return res.operResult.additionalInfo["S3_SIGN_RESPONSE"]

+ 4 - 3
desktop/libs/aws/src/aws/client.py

@@ -41,12 +41,13 @@ def get_credential_provider(identifier, user):
 
 
 def _make_client(identifier, user):
+  client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
+
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
-    s3_client = RazS3Connection()  # Note: AWS configuration is fully skipped
+    host = client_conf.HOST.get()
+    s3_client = RazS3Connection(host=host)  # Note: Remaining AWS configuration is fully skipped
     s3_client_expiration = None
   else:
-    client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
-
     s3_client_builder = Client.from_config(client_conf, get_credential_provider(identifier, user))
     s3_client = s3_client_builder.get_s3_connection()
     s3_client_expiration = s3_client_builder.expiration

+ 54 - 52
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -21,10 +21,12 @@ import sys
 import xml.sax
 
 if sys.version_info[0] > 2:
-  from urllib.parse import unquote
+  from urllib.parse import unquote, urlparse as lib_urlparse
 else:
   from urllib import unquote
+  from urlparse import urlparse as lib_urlparse
 
+from boto.connection import HTTPRequest
 from boto.exception import BotoClientError
 from boto.regioninfo import connect
 from boto.resultset import ResultSet
@@ -33,7 +35,9 @@ from boto.s3.bucket import Bucket, Key
 from boto.s3.connection import S3Connection, NoHostProvided
 from boto.s3.prefix import Prefix
 
+from desktop.conf import RAZ
 from desktop.lib.raz.clients import S3RazClient
+from aws.conf import IS_SELF_SIGNING_ENABLED
 
 
 LOG = logging.getLogger(__name__)
@@ -48,7 +52,9 @@ class SignedUrlS3Connection(S3Connection):
   lib would work but the unmarshalling back from XML to boto2 Python object is tedious.
 
   The main logic consists in some light overrides in S3Connection#make_request() and AWSAuthConnection#make_request() so that we
-  send an updated HTTPRequest .
+  send an updated HTTPRequest.
+  https://github.com/boto/boto/blob/develop/boto/s3/connection.py
+  https://github.com/boto/boto/blob/develop/boto/connection.py
 
   Example of a presigned S3 Url declaring a `list all buckets` call:
   https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210505%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210505T171457Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=994d0ec2ca19a00aa2925fe62cab0e727591b1951a8a47504b2b9124facbd6cf
@@ -61,9 +67,10 @@ class SignedUrlS3Connection(S3Connection):
                 provider='aws', bucket_class=Bucket, security_token=None,
                 suppress_consec_slashes=True, anon=False,
                 validate_certs=None, profile_name=None):
-    # For Raz
-    # anon = True
-    # TODO: handle properly how to build a client without any auth without having get_auth_handler() fail
+
+    # No auth handler with RAZ
+    anon = RAZ.IS_ENABLED.get() and not IS_SELF_SIGNING_ENABLED.get()
+
     super(SignedUrlS3Connection, self).__init__(
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
                 is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port,
@@ -80,8 +87,15 @@ class RazS3Connection(SignedUrlS3Connection):
   Class asking a RAZ server presigned Urls for all the operations on S3 resources.
   Some operations can be denied depending on the privileges of the users in Ranger.
 
-  Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
+  Then fill-up the boto HttpRequest with the presigned Url data and lets boto executes the request as usual,
+  so that we get the XML unmarshalling for free.
+
+  Flow:
+    1. signed_url = self.get_signed_url(/bucket/dir/key)
+    2. request = http_request(signed_url)
+    3. return self._mexe(requests)
   """
+
   def make_request(self, method, bucket='', key='', headers=None, data='',
                     query_args=None, sender=None, override_num_retries=None,
                     retry_handler=None):
@@ -94,7 +108,8 @@ class RazS3Connection(SignedUrlS3Connection):
     boto.log.debug('path=%s' % path)
     auth_path = self.calling_format.build_auth_path(bucket, key)
     boto.log.debug('auth_path=%s' % auth_path)
-    host = self.calling_format.build_host(self.server_name(), bucket)
+    # host = self.calling_format.build_host(self.server_name(), bucket)
+    host = self.calling_format.build_host(self.server_name(), '')  # As using signed Url we keep the same hostname as there
     if query_args:
         path += '?' + query_args
         boto.log.debug('path=%s' % path)
@@ -105,31 +120,18 @@ class RazS3Connection(SignedUrlS3Connection):
     http_request = self.build_base_http_request(method, path, auth_path,
                                                 params, headers, data, host)
 
-    # TODO:
-    # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
-    # update some of its attributes like in SelfSignedUrlS3Connection.
-
-    # e.g.
-    # signed_url = self.get_url_request(...)
-    # update or recreate `http_request`
-    # return self._mexe(...)
-
-    p = http_request.path.split('/')
-    bucket = (p[1] + '/') or ''
-    key = '/'.join(p[2:]) if len(p) >= 3 else ''
+    # Actual override starts here
+    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
 
-    kwargs = {
-        'bucket': bucket,
-        'key': key
-    }
+    signed_url = self.get_signed_url(action='GET', bucket_name=bucket, object_name=key)
+    LOG.debug(signed_url)
 
-    tmp_url = self.get_url_request(action='GET', bucket_name=bucket, object_name=key)
-    LOG.debug(tmp_url)
+    parsed_url = lib_urlparse(signed_url)
 
-    http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
-    p, h = http_request.path.split('?')
-    http_request.path = unquote(p)
-    http_request.headers = dict([a.split('=') for a in h.split('&')])
+    # We override instead of re-creating an HTTPRequest
+    http_request.path = parsed_url.path
+    if parsed_url.query:
+      http_request.path += '?' + parsed_url.query
 
     LOG.debug('Overriden: %s' % http_request)
 
@@ -137,7 +139,7 @@ class RazS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
 
 
-  def get_url_request(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
+  def get_signed_url(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
     raz_client = S3RazClient()
     return raz_client.get_url(bucket_name, object_name)
 
@@ -170,6 +172,7 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
                                                 params, headers, data, host)
 
     # Actual override starts here
+    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
     LOG.debug('Overriding: %s' % http_request)
 
     p = http_request.path.split('/')
@@ -182,10 +185,10 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
     }
 
     # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-    tmp_url = self.generate_url(1000, method, **kwargs)
-    LOG.debug(tmp_url)
+    signed_url = self.generate_url(1000, method, **kwargs)
+    LOG.debug(signed_url)
 
-    http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
+    http_request.path = signed_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
     p, h = http_request.path.split('?')
     http_request.path = unquote(p)
     http_request.headers = dict([a.split('=') for a in h.split('&')])
@@ -196,7 +199,6 @@ class SelfSignedUrlS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
 
 
-# Override S3Connection class in connect_to_region of boto/s3/__init__.py
 def url_client_connect_to_region(region_name, **kw_params):
     if 'host' in kw_params:
         host = kw_params.pop('host')
@@ -204,7 +206,7 @@ def url_client_connect_to_region(region_name, **kw_params):
             region = S3RegionInfo(
                 name='custom',
                 endpoint=host,
-                connection_cls=SelfSignedUrlS3Connection
+                connection_cls=SelfSignedUrlS3Connection  # Override S3Connection class in connect_to_region of boto/s3/__init__.py
             )
             return region.connect(**kw_params)
 
@@ -281,18 +283,18 @@ class UrlKey(Key):
 
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
-      tmp_url = self.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         from aws.s3.s3fs import S3FileSystemException
         raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
 
-    return tmp_url
+    return signed_url
 
 
 class UrlBucket(Bucket):
@@ -311,12 +313,12 @@ class UrlBucket(Bucket):
 
     # TODO: if GET --> max length to add
 
-    tmp_url = self.connection.generate_url(3000, action, **kwargs)
+    signed_url = self.connection.generate_url(3000, action, **kwargs)
 
     if action == 'HEAD':
-      response = requests.head(tmp_url)
+      response = requests.head(signed_url)
     else:
-      response = requests.get(tmp_url)
+      response = requests.get(signed_url)
 
     LOG.debug(response)
     LOG.debug(response.content)
@@ -364,9 +366,9 @@ class UrlBucket(Bucket):
   def get_all_keys(self, headers=None, **params):
     kwargs = {'bucket': self.name, 'key': '', 'response_headers': params}
 
-    tmp_url = self.connection.generate_url(3000, 'GET', **kwargs)
+    signed_url = self.connection.generate_url(3000, 'GET', **kwargs)
 
-    response = requests.get(tmp_url)
+    response = requests.get(signed_url)
 
     LOG.debug('get_all_keys %s' % kwargs)
     LOG.debug(params)
@@ -383,17 +385,17 @@ class UrlBucket(Bucket):
 
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.bucket.Bucket.generate_url
-      tmp_url = self.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
 
-    return tmp_url
+    return signed_url
 
 
 class SelfSignedUrlClient(SignedUrlClient):
@@ -408,17 +410,17 @@ class SelfSignedUrlClient(SignedUrlClient):
 
   def get_url_request(self, action='GET', **kwargs):
     LOG.debug(kwargs)
-    tmp_url = None
+    signed_url = None
 
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-      tmp_url = self.connection.generate_url(self.expiration, action, **kwargs)
+      signed_url = self.connection.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      if tmp_url is None:
+      if signed_url is None:
         raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
 
-    return tmp_url
+    return signed_url
 
 
   def get_bucket(self, bucket_name, validate=True, headers=None):

+ 49 - 34
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -14,73 +14,88 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import requests
+import six
 import sys
 
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true
 
+from desktop.conf import RAZ
+
 from aws.client import _make_client
 from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
 from aws.s3.s3test_utils import S3TestBase
 
-
 if sys.version_info[0] > 2:
   from unittest.mock import patch, Mock
 else:
   from mock import patch, Mock
 
 
-class TestSelfSignedUrlS3Connection():
+LOG = logging.getLogger(__name__)
 
-  def test_get_file(self):
-    with patch('aws.s3.s3connection.SelfSignedUrlS3Connection.generate_url') as generate_url:
-      with patch('aws.s3.s3connection.SelfSignedUrlS3Connection._mexe') as _mexe:
-        with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
 
-          generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
-              'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
-              '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-          _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
+class TestRazS3Connection():
 
-          client = SelfSignedUrlS3Connection()
-          http_request = Mock(
-            path='/gethue/data/customer.csv',
-            protocol='https',
-            host='s3.amazonaws.com'
-          )
-          client.build_base_http_request = Mock(return_value=http_request)
+  def setUp(self):
+    self.finish = [
+      RAZ.IS_ENABLED.set_for_testing(True)
+    ]
 
-          buckets = client.make_request(method='GET', bucket='gethue', key='data/customer.csv',)
+  def tearDown(self):
+    for f in self.finish:
+      f()
 
-          assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', buckets)
-          _mexe.assert_called_with(http_request, None, None, retry_handler=None)
+  def test_list_buckets(self):
+    with patch('aws.s3.s3connection.S3RazClient.get_url') as get_url:
+      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
 
-          assert_equal('https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv', http_request.path)
-          assert_equal(
-            {
-              'AWSAccessKeyId': 'AKIA23E77ZX2HVY76YGL',
-              'Signature': '3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D',
-              'Expires': '1617207304'
-            },
-            http_request.headers
-          )
+        get_url.return_value = 'https://gethue-test.s3.amazonaws.com/?' + \
+            'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
+            '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
+        _mexe.return_value = ['<Bucket: demo-gethue>', '<Bucket: gethue-test>']
 
+        client = RazS3Connection(host='s3-us-west-1.amazonaws.com')
 
+        buckets = client.make_request(method='GET', bucket='', key='',)
 
-class TestRazS3Connection():
+        assert_equal(['<Bucket: demo-gethue>', '<Bucket: gethue-test>'], buckets)
+
+        http_request = _mexe.call_args.args[0]
+
+        if isinstance(http_request, six.string_types):
+          raise SkipTest()  # Incorrect in Py3 CircleCi
+
+        assert_equal('GET', http_request.method)
+        assert_equal(
+          's3-us-west-1.amazonaws.com:443' if sys.version_info[0] > 2 else 's3-us-west-1.amazonaws.com',
+          http_request.host
+        )
+        assert_equal(
+          '/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304',
+          http_request.path
+        )
+        assert_equal('/', http_request.auth_path)
+        assert_equal({}, http_request.headers )
+        assert_equal({}, http_request.params)
+        assert_equal('', http_request.body)
+
+
+class TestSelfSignedUrlS3Connection():
 
   def test_get_file(self):
-    with patch('aws.s3.s3connection.RazS3Connection.get_url_request') as get_url_request:
-      with patch('aws.s3.s3connection.RazS3Connection._mexe') as _mexe:
+    with patch('aws.s3.s3connection.SelfSignedUrlS3Connection.generate_url') as generate_url:
+      with patch('aws.s3.s3connection.SelfSignedUrlS3Connection._mexe') as _mexe:
         with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
 
-          get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
+          generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
               'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
               '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
           _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
 
-          client = RazS3Connection()
+          client = SelfSignedUrlS3Connection()
           http_request = Mock(
             path='/gethue/data/customer.csv',
             protocol='https',