Просмотр исходного кода

[raz] Fix S3 delete operation

- Cleanup path to be signed and sent to RAZ for signed headers
- Send xml data for /?delete calls for signing to RAZ for signed headers
Harshg999 3 лет назад
Родитель
Сommit
667deac151

+ 2 - 2
desktop/core/src/desktop/lib/raz/clients.py

@@ -28,7 +28,7 @@ class S3RazClient():
   def __init__(self, username):
     self.username = username
 
-  def get_url(self, action='GET', path=None, headers=None):
+  def get_url(self, action='GET', path=None, headers=None, data=None):
     '''
     Example of headers:
     {
@@ -49,7 +49,7 @@ class S3RazClient():
       service='s3',
     )
 
-    return c.check_access(method=action, url=path, headers=headers)
+    return c.check_access(method=action, url=path, headers=headers, data=data)
 
 
 class AdlsRazClient():

+ 6 - 5
desktop/core/src/desktop/lib/raz/raz_client.py

@@ -101,7 +101,7 @@ class RazClient(object):
     self.requestid = str(uuid.uuid4())
 
 
-  def check_access(self, method, url, params=None, headers=None):
+  def check_access(self, method, url, params=None, headers=None, data=None):
     LOG.debug("Check access: method {%s}, url {%s}, params {%s}, headers {%s}" % (method, url, params, headers))
 
     path = lib_urlparse(url)
@@ -132,7 +132,7 @@ class RazClient(object):
     if self.service == 'adls':
       self._make_adls_request(request_data, method, path, url_params, resource_path)
     elif self.service == 's3':
-      self._make_s3_request(request_data, request_headers, method, params, headers, url_params, endpoint, resource_path)
+      self._make_s3_request(request_data, request_headers, method, params, headers, url_params, endpoint, resource_path, data=data)
 
     LOG.debug('Raz url: %s' % raz_url)
     LOG.debug("Sending access check headers: {%s} request_data: {%s}" % (request_headers, request_data))
@@ -240,14 +240,14 @@ class RazClient(object):
     return access_type
 
 
-  def _make_s3_request(self, request_data, request_headers, method, params, headers, url_params, endpoint, resource_path):
+  def _make_s3_request(self, request_data, request_headers, method, params, headers, url_params, endpoint, resource_path, data=None):
     allparams = [raz_signer.StringListStringMapProto(key=key, value=[val]) for key, val in url_params.items()]
     allparams.extend([raz_signer.StringListStringMapProto(key=key, value=[val]) for key, val in params.items()])
     headers = [raz_signer.StringStringMapProto(key=key, value=val) for key, val in headers.items()]
 
     LOG.debug(
-      "Preparing sign request with http_method: {%s}, headers: {%s}, parameters: {%s}, endpoint: {%s}, resource_path: {%s}" %
-      (method, headers, allparams, endpoint, resource_path)
+      "Preparing sign request with http_method: {%s}, headers: {%s}, parameters: {%s}, endpoint: {%s}, resource_path: {%s}, content_to_sign: {%s}" %
+      (method, headers, allparams, endpoint, resource_path, data)
     )
     raz_req = raz_signer.SignRequestProto(
         endpoint_prefix=self.service_params['endpoint_prefix'],
@@ -257,6 +257,7 @@ class RazClient(object):
         headers=headers,
         parameters=allparams,
         resource_path=resource_path,
+        content_to_sign=data,
         time_offset=0
     )
     raz_req_serialized = raz_req.SerializeToString()

+ 28 - 14
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -21,10 +21,10 @@ import sys
 import xml.sax
 
 if sys.version_info[0] > 2:
-  from urllib.parse import unquote, urlparse as lib_urlparse
+  from urllib.parse import unquote, urlparse as lib_urlparse, parse_qs, urlencode
 else:
-  from urllib import unquote
-  from urlparse import urlparse as lib_urlparse
+  from urllib import unquote, urlencode
+  from urlparse import urlparse as lib_urlparse, parse_qs
 
 from boto.connection import HTTPRequest
 from boto.exception import BotoClientError
@@ -103,23 +103,34 @@ class RazS3Connection(SignedUrlS3Connection):
                     retry_handler=None):
 
     if isinstance(bucket, self.bucket_class):
-        bucket = bucket.name
+      bucket = bucket.name
     if isinstance(key, Key):
-        key = key.name
+      key = key.name
+
     path = self.calling_format.build_path_base(bucket, key)
     LOG.debug('path=%s' % path)
+
     auth_path = self.calling_format.build_auth_path(bucket, key)
     LOG.debug('auth_path=%s' % auth_path)
+
     host = self.calling_format.build_host(self.server_name(), bucket)
+
     if query_args:
-        path += '?' + query_args
-        LOG.debug('path=%s' % path)
-        auth_path += '?' + query_args
-        LOG.debug('auth_path=%s' % auth_path)
+      # Clean prefix to remove s3a%3A//[S3_BUCKET]/ for sending correct relative path to RAZ
+      if 'prefix=s3a%3A//' in query_args:
+        qs_parsed = parse_qs(query_args) # all strings will be unquoted
+        prefix_relative_path = qs_parsed['prefix'][0].partition(bucket + '/')[2]
+        qs_parsed['prefix'][0] = prefix_relative_path
+
+        query_args = unquote(urlencode(qs_parsed, doseq=True))
+
+      path += '?' + query_args
+      LOG.debug('path=%s' % path)
+      auth_path += '?' + query_args
+      LOG.debug('auth_path=%s' % auth_path)
 
     params = {}
-    http_request = self.build_base_http_request(method, path, auth_path,
-                                                params, headers, data, host)
+    http_request = self.build_base_http_request(method, path, auth_path, params, headers, data, host)
 
     # Actual override starts here
     LOG.debug('http_request: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
@@ -127,7 +138,10 @@ class RazS3Connection(SignedUrlS3Connection):
 
     url = 'https://%(host)s%(path)s' % {'host': host, 'path': path}
 
-    raz_headers = self.get_signed_url(action=method, url=url, headers=headers)
+    # Do not send the xml data for signing for upload operation
+    xml_data = '' if query_args and 'uploadId=' in query_args else data
+
+    raz_headers = self.get_signed_url(action=method, url=url, headers=headers, data=xml_data)
     LOG.debug('Raz returned those headers: %s' % raz_headers)
 
     if raz_headers is not None:
@@ -141,10 +155,10 @@ class RazS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
 
 
-  def get_signed_url(self, action='GET', url=None, headers=None):
+  def get_signed_url(self, action='GET', url=None, headers=None, data=None):
     raz_client = S3RazClient(username=self.username)
 
-    return raz_client.get_url(action, url, headers)
+    return raz_client.get_url(action, url, headers, data)
 
 
 class SelfSignedUrlS3Connection(SignedUrlS3Connection):