Browse Source

[raz] Refactoring to share common XML unmarshalling

New test suite
Romain Rigaux 4 years ago
parent
commit
6061cf98b0

+ 28 - 25
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -26,6 +26,7 @@ from boto.s3.bucketlistresultset import BucketListResultSet
 from boto.s3.prefix import Prefix
 
 from desktop.lib.raz.clients import S3RazClient
+from aws.s3.s3fs import S3FileSystemException
 
 
 LOG = logging.getLogger(__name__)
@@ -39,27 +40,30 @@ class UrlConnection():
   """
   Share the unmarshalling from XML to boto Python objects from the requests calls.
   """
+  def _get_all_buckets(self, signed_url):
+    LOG.debug(signed_url)
+
+    response = requests.get(signed_url)
 
-  def get_all_buckets(self, response):
-    LOG.debug('get_all_buckets')
     LOG.debug(response)
     LOG.debug(response.content)
 
     rs = ResultSet([('Bucket', self.connection.bucket_class)])
-    h = boto.handler.XmlHandler(rs, self.connection)
+    h = boto.handler.XmlHandler(rs, None)
     xml.sax.parseString(response.content, h)
     LOG.debug(rs)
 
+    return rs
 
-class RazUrlConnection():
+
+class RazUrlConnection(UrlConnection):
 
   def __init__(self):
     self.raz = S3RazClient()
 
   def get_all_buckets(self, headers=None):
     url = self._generate_url()
-    # call
-    # unmarshall via UrlConnection
+    return self._get_all_buckets(url)
 
   def get_bucket(self, bucket_name, validate=True, headers=None):
     pass
@@ -86,12 +90,15 @@ class UrlKey(Key):
 
   def _generate_url(self, action='GET', **kwargs):
     LOG.debug(kwargs)
+    tmp_url = None
+
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
       tmp_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      return None
+      if tmp_url is None:
+        raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
 
     return tmp_url
 
@@ -172,38 +179,40 @@ class UrlBucket(Bucket):
 
   def _generate_url(self, action='GET', **kwargs):
     LOG.debug(kwargs)
+    tmp_url = None
+
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.bucket.Bucket.generate_url
       tmp_url = self.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      return None
+      if tmp_url is None:
+        raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
 
     return tmp_url
 
 
-class BotoUrlConnection():
+class BotoUrlConnection(UrlConnection):
 
   def __init__(self, connection):
     self.connection = connection
     self.expiration = 3600
 
     self.connection.make_request = None  # We make sure we never call via regular boto connection directly
-    self.connection.set_bucket_class(UrlBucket)  # We use our bucket class to override any direct call to S3
+    self.connection.set_bucket_class(UrlBucket)  # We use our bucket class to keep overriding any direct call to S3
 
 
   def get_all_buckets(self, headers=None):
     kwargs = {'action': 'GET'}
+    LOG.debug('get_all_buckets: %s' % kwargs)
     try:
       tmp_url = self._generate_url(**kwargs)
     except BotoClientError as e:
       LOG.error(e)
       return None
 
+    LOG.debug(tmp_url)
     response = requests.get(tmp_url)
-
-    LOG.debug('get_all_buckets')
-    print(tmp_url)
     LOG.debug(response)
     LOG.debug(response.content)
 
@@ -218,27 +227,21 @@ class BotoUrlConnection():
   def get_bucket(self, bucket_name, validate=True, headers=None):
     kwargs = {'action': 'GET', 'bucket': bucket_name}
 
-    tmp_url = self._generate_url(**kwargs)
+    signed_url = self._generate_url(**kwargs)
 
-    response = requests.get(tmp_url)
-
-    LOG.debug('get_bucket')
-    LOG.debug(response)
-    LOG.debug(response.content)
-
-    rs = self.connection.bucket_class(self.connection, bucket_name, key_class=UrlKey)  # Using content?
-    LOG.debug(rs)
-
-    return rs
+    return self._get_all_buckets(signed_url)
 
 
   def _generate_url(self, action='GET', **kwargs):
     LOG.debug(kwargs)
+    tmp_url = None
+
     try:
       # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
       tmp_url = self.connection.generate_url(self.expiration, action, **kwargs)
     except BotoClientError as e:
       LOG.error(e)
-      return None
+      if tmp_url is None:
+        raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
 
     return tmp_url

+ 35 - 4
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -15,17 +15,48 @@
 # limitations under the License.
 
 import requests
+import sys
 
 from nose.tools import assert_equal, assert_false, assert_true, assert_raises
 
 from aws.client import _make_client
-from aws.s3.s3connection import BotoUrlConnection
+from aws.s3.s3connection import BotoUrlConnection, UrlBucket
 from aws.s3.s3test_utils import S3TestBase
 
-# TEST_S3_BUCKET=gethue-test ./build/env/bin/hue test specific aws.s3.s3connection_test
 
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock, MagicMock
+else:
+  from mock import patch, Mock, MagicMock
 
-class BotoUrlConnectionIntegrationTest(S3TestBase):
+
+class TestBotoUrlConnection():
+
+  def test_get_buckets(self):
+    with patch('aws.s3.s3connection.BotoUrlConnection._generate_url') as _generate_url:
+      with patch('aws.s3.s3connection.requests.get') as requests_get:
+
+        _generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
+            '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
+        requests_get.return_value = Mock(
+          content=b'<?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult '
+            b'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>0429b0aed2900f450655928a09e06e7aaac9939bc9141fc5aeeccd8b93b9778f'
+            b'</ID><DisplayName>team</DisplayName></Owner><Buckets><Bucket><Name>demo-gethue</Name><CreationDate>2020-08-22T08:03:18.000Z'
+            b'</CreationDate></Bucket><Bucket><Name>gethue-test</Name><CreationDate>2021-03-31T14:47:14.000Z</CreationDate></Bucket>'
+            b'</Buckets></ListAllMyBucketsResult>'
+        )
+
+        connection = Mock()
+        connection.bucket_class = UrlBucket
+        buckets = BotoUrlConnection(connection=connection).get_all_buckets()
+
+        assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
+
+
+class TestBotoUrlConnectionIntegration(S3TestBase):
+  #
+  # To trigger:
+  # TEST_S3_BUCKET=gethue-test ./build/env/bin/hue test specific aws.s3.s3connection_test
 
   @classmethod
   def setUpClass(cls):
@@ -33,7 +64,7 @@ class BotoUrlConnectionIntegrationTest(S3TestBase):
 
 
   def setUp(self):
-    super(BotoUrlConnectionIntegrationTest, self).setUp()
+    super(TestBotoUrlConnectionIntegration, self).setUp()
 
     self.c = _make_client(identifier='default', user=None)
     self.connection = self.c._s3_connection.connection