瀏覽代碼

[aws] MVP of simpler Connection

Romain Rigaux 4 年之前
父節點
當前提交
807a92b32f

+ 6 - 6
desktop/libs/aws/src/aws/client.py

@@ -21,7 +21,7 @@ import os
 import boto
 
 from aws import conf as aws_conf
-from aws.s3.s3connection import RazUrlConnection, BotoUrlConnection
+from aws.s3.s3connection import url_client_connect_to_region, RazSignedUrlS3Connection
 from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 
 from desktop.conf import RAZ
@@ -42,7 +42,7 @@ def get_credential_provider(identifier, user):
 
 def _make_client(identifier, user):
   if RAZ.IS_ENABLED.get() and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
-    s3_client = RazUrlConnection()  # Note: AWS configuration is fully skipped
+    s3_client = RazSignedUrlS3Connection()  # Note: AWS configuration is fully skipped
     s3_client_expiration = None
   else:
     client_conf = aws_conf.AWS_ACCOUNTS[identifier] if identifier in aws_conf.AWS_ACCOUNTS else None
@@ -175,7 +175,10 @@ class Client(object):
         kwargs.update({'host': self._host})
         connection = boto.s3.connection.S3Connection(**kwargs)
       elif self._region:
-        connection = boto.s3.connect_to_region(self._region, **kwargs)
+        if aws_conf.IS_SELF_SIGNING_ENABLED.get():
+          connection = url_client_connect_to_region(self._region, **kwargs)
+        else:
+          connection = boto.s3.connect_to_region(self._region, **kwargs)
       else:
         kwargs.update({'host': 's3.amazonaws.com'})
         connection = boto.s3.connection.S3Connection(**kwargs)
@@ -190,7 +193,4 @@ class Client(object):
       if connection is None:
         raise S3FileSystemException('Can not construct S3 Connection for region %s' % self._region)
 
-    if aws_conf.IS_SELF_SIGNING_ENABLED.get():
-      connection = BotoUrlConnection(connection)
-
     return connection

+ 163 - 5
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -17,11 +17,20 @@
 import boto
 import logging
 import requests
+import sys
 import xml.sax
 
+if sys.version_info[0] > 2:
+  from urllib.parse import unquote
+else:
+  from urllib import unquote
+
 from boto.exception import BotoClientError
+from boto.regioninfo import connect
 from boto.resultset import ResultSet
+from boto.s3 import S3RegionInfo
 from boto.s3.bucket import Bucket, Key
+from boto.s3.connection import S3Connection
 from boto.s3.prefix import Prefix
 
 from desktop.lib.raz.clients import S3RazClient
@@ -30,11 +39,160 @@ from desktop.lib.raz.clients import S3RazClient
 LOG = logging.getLogger(__name__)
 
 
-# Note: Connection means more "Client" but we currently follow boto2 terminology
-# To split in 3 modules at some point s3_url_client, s3_raz_client, s3_self_signing_client,
+class SignedUrlS3Connection(S3Connection):
+  """
+  Contact S3 via a presigned Url of the resource hence not requiring any S3 credentials.
+
+  This is a client replacing the building of the Http Request of the S3 resource via asking a third party providing for a presigned Urls.
+  The request information is then injected into the regular boto HTTPRequest as the format is the same. Raw calls via the requests
+  lib would work but the unmarshalling back from XML to boto2 Python object is tedious.
+
+  The main logic consists in some light overrides in S3Connection#make_request() and AWSAuthConnection#make_request() so that we
+  send an updated HTTPRequest .
+
+  Example of a presigned S3 Url declaring a `list all buckets` call:
+  https://s3-us-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA23E77ZX2HVY76YGL%2F20210505%2Fus-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210505T171457Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=994d0ec2ca19a00aa2925fe62cab0e727591b1951a8a47504b2b9124facbd6cf
+  """
+  pass
+
+
+class RazSignedUrlS3Connection(S3Connection):
+  """
+  Class asking a RAZ server presigned Urls for all the operations on S3 resources.
+  Some operations can be denied depending on the privileges of the users in Ranger.
+
+  Then fill-up the boto Http request with the presigned Url data and let boto executes the request as usual.
+  """
+  def make_request(self, method, bucket='', key='', headers=None, data='',
+                    query_args=None, sender=None, override_num_retries=None,
+                    retry_handler=None):
+
+    if isinstance(bucket, self.bucket_class):
+        bucket = bucket.name
+    if isinstance(key, Key):
+        key = key.name
+    path = self.calling_format.build_path_base(bucket, key)
+    boto.log.debug('path=%s' % path)
+    auth_path = self.calling_format.build_auth_path(bucket, key)
+    boto.log.debug('auth_path=%s' % auth_path)
+    host = self.calling_format.build_host(self.server_name(), bucket)
+    if query_args:
+        path += '?' + query_args
+        boto.log.debug('path=%s' % path)
+        auth_path += '?' + query_args
+        boto.log.debug('auth_path=%s' % auth_path)
+
+    params = {}
+    http_request = self.build_base_http_request(method, path, auth_path,
+                                                params, headers, data, host)
+
+    # TODO:
+    # Build a check_access call to Raz, get back presigned Url data and either create a new Boto http_request or
+    # update some of its attributes like in SelfSignedUrlS3Connection.
+    #
+    # e.g.
+    # signed_url = self.get_url_request(...)
+    # update or recreate `http_request`
+    # return self._mexe(...)
+
+    return super(RazSignedUrlS3Connection, self).make_request(
+        method, path, headers,
+        data, host, auth_path, sender,
+        override_num_retries=override_num_retries,
+        retry_handler=retry_handler
+    )
+
+
+  def get_url_request(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
+    raz_client = S3RazClient()
+    return raz_client.get_url(bucket_name, object_name)
+
+
+class SelfSignedUrlS3Connection(S3Connection):
+  """
+  Test class self generating presigned Urls so that the Http Client using signed Urls instead
+  of direct boto calls to S3 can be tested.
+  """
+  def make_request(self, method, bucket='', key='', headers=None, data='',
+                    query_args=None, sender=None, override_num_retries=None,
+                    retry_handler=None):
+    if isinstance(bucket, self.bucket_class):
+        bucket = bucket.name
+    if isinstance(key, Key):
+        key = key.name
+    path = self.calling_format.build_path_base(bucket, key)
+    boto.log.debug('path=%s' % path)
+    auth_path = self.calling_format.build_auth_path(bucket, key)
+    boto.log.debug('auth_path=%s' % auth_path)
+    host = self.calling_format.build_host(self.server_name(), bucket)
+    if query_args:
+        path += '?' + query_args
+        boto.log.debug('path=%s' % path)
+        auth_path += '?' + query_args
+        boto.log.debug('auth_path=%s' % auth_path)
+
+    params = {}
+    http_request = self.build_base_http_request(method, path, auth_path,
+                                                params, headers, data, host)
+
+    # Actual override starts here
+    LOG.debug('Overriding: %s' % http_request)
+
+    p = http_request.path.split('/')
+    bucket = (p[1] + '/') or ''
+    key = '/'.join(p[2:]) if len(p) >= 3 else ''
+
+    kwargs = {
+        'bucket': bucket,
+        'key': key
+    }
+
+    # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
+    tmp_url = self.generate_url(1000, method, **kwargs)
+    LOG.debug(tmp_url)
+
+    http_request.path = tmp_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
+    p, h = http_request.path.split('?')
+    http_request.path = unquote(p)
+    http_request.headers = dict([a.split('=') for a in h.split('&')])
+
+    LOG.debug('Overriden: %s' % http_request)
+
+    return self._mexe(http_request, sender, override_num_retries,
+                      retry_handler=retry_handler)
 
 
-class UrlConnection():
+# Override S3Connection class in connect_to_region of boto/s3/__init__.py
+def url_client_connect_to_region(region_name, **kw_params):
+    if 'host' in kw_params:
+        host = kw_params.pop('host')
+        if host not in ['', None]:
+            region = S3RegionInfo(
+                name='custom',
+                endpoint=host,
+                connection_cls=SelfSignedUrlS3Connection
+            )
+            return region.connect(**kw_params)
+
+    return connect('s3', region_name, region_cls=S3RegionInfo,
+                   connection_cls=SelfSignedUrlS3Connection, **kw_params)
+
+
+
+# --------------------------------------------------------------------------------
+# Deprecated Client: to remove at v1
+#
+# This clients re-implement S3Connection methods via a PreSignedUrl either
+# provided by a RAZ server or another Boto lib. Request to S3 are then made via
+# requests and the raw XML is unmarshalling back to boto2 Python objects.
+#
+# Note: hooking-in the get/generate URL directly into S3Connection#make_request()
+# was found to be simpler and possible as boto itself sends signed Urls.
+# Handling various operations is relatively simple as defined by HTTP action and
+# paths. Most of the security is handled via header parameters.
+# --------------------------------------------------------------------------------
+
+class SignedUrlClient():
   """
   Share the unmarshalling from XML to boto Python objects from the requests calls.
   """
@@ -60,7 +218,7 @@ class UrlConnection():
 
 
 
-class RazUrlConnection(UrlConnection):
+class RazSignedUrlClient(SignedUrlClient):
 
   def __init__(self):
     self.raz = S3RazClient()
@@ -204,7 +362,7 @@ class UrlBucket(Bucket):
     return tmp_url
 
 
-class BotoUrlConnection(UrlConnection):
+class SelfSignedUrlClient(SignedUrlClient):
 
   def __init__(self, connection):
     self.connection = connection

+ 11 - 11
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -20,7 +20,7 @@ import sys
 from nose.tools import assert_equal, assert_false, assert_true, assert_raises
 
 from aws.client import _make_client
-from aws.s3.s3connection import BotoUrlConnection, UrlBucket, RazUrlConnection
+from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient
 from aws.s3.s3test_utils import S3TestBase
 
 
@@ -30,10 +30,10 @@ else:
   from mock import patch, Mock
 
 
-class TestBotoUrlConnection():
+class TestSelfSignedUrlClient():
 
   def test_get_buckets(self):
-    with patch('aws.s3.s3connection.BotoUrlConnection.get_url_request') as get_url_request:
+    with patch('aws.s3.s3connection.SelfSignedUrlClient.get_url_request') as get_url_request:
       with patch('aws.s3.s3connection.requests.get') as requests_get:
 
         get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
@@ -47,15 +47,15 @@ class TestBotoUrlConnection():
         )
 
         connection = Mock()
-        buckets = BotoUrlConnection(connection=connection).get_all_buckets()
+        buckets = SelfSignedUrlClient(connection=connection).get_all_buckets()
 
         assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
 
 
-class TestRazUrlConnection():
+class TestRazSignedUrlClient():
 
   def test_get_buckets(self):
-    with patch('aws.s3.s3connection.RazUrlConnection.get_url_request') as get_url_request:
+    with patch('aws.s3.s3connection.RazSignedUrlClient.get_url_request') as get_url_request:
       with patch('aws.s3.s3connection.requests.get') as requests_get:
 
         # TODO: update with potentially slightly different URL/headers
@@ -70,12 +70,12 @@ class TestRazUrlConnection():
             b'</Buckets></ListAllMyBucketsResult>'
         )
 
-        buckets = RazUrlConnection().get_all_buckets()
+        buckets = RazSignedUrlClient().get_all_buckets()
 
         assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
 
 
-class TestBotoUrlConnectionIntegration(S3TestBase):
+class TestSelfSignedUrlClientIntegration(S3TestBase):
   #
   # To trigger:
   # TEST_S3_BUCKET=gethue-test ./build/env/bin/hue test specific aws.s3.s3connection_test
@@ -86,21 +86,21 @@ class TestBotoUrlConnectionIntegration(S3TestBase):
 
 
   def setUp(self):
-    super(TestBotoUrlConnectionIntegration, self).setUp()
+    super(TestSelfSignedUrlClientIntegration, self).setUp()
 
     self.c = _make_client(identifier='default', user=None)
     self.connection = self.c._s3_connection.connection
 
 
   def test_list_buckets(self):
-    buckets = BotoUrlConnection(self.connection).get_all_buckets()
+    buckets = SelfSignedUrlClient(self.connection).get_all_buckets()
 
     assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
 
 
   def test_list_file(self):
     kwargs = {'action': 'GET', 'bucket': 'gethue-test', 'key': 'data/query-hive-weblogs.csv'}
-    url = BotoUrlConnection(self.connection).generate_url(**kwargs)
+    url = SelfSignedUrlClient(self.connection).generate_url(**kwargs)
 
     url = 'https://gethue-test.s3.amazonaws.com/data/query-hive-weblogs.csv?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'