Browse Source

[raz] Integrating client with boto2

Romain Rigaux 4 years ago
parent
commit
db2ecc8a2b

+ 12 - 12
desktop/core/src/desktop/lib/raz/clients.py

@@ -19,6 +19,7 @@ import logging
 from requests_kerberos import HTTPKerberosAuth
 from requests_kerberos import HTTPKerberosAuth
 
 
 from desktop.conf import RAZ
 from desktop.conf import RAZ
+from desktop.lib.raz.raz_client import get_raz_client
 from desktop.lib.raz.ranger.clients.ranger_raz_adls import RangerRazAdls
 from desktop.lib.raz.ranger.clients.ranger_raz_adls import RangerRazAdls
 from desktop.lib.raz.ranger.clients.ranger_raz_s3 import RangerRazS3
 from desktop.lib.raz.ranger.clients.ranger_raz_s3 import RangerRazS3
 
 
@@ -28,18 +29,17 @@ LOG = logging.getLogger(__name__)
 
 
 class S3RazClient():
 class S3RazClient():
 
 
-  def __init__(self, **kwargs):
-    if RAZ.API_AUTHENTICATION.get() == 'kerberos':
-      auth = HTTPKerberosAuth()
-    else:
-      auth = None
-
-    self.ranger = RangerRazS3(RAZ.API_URL.get(), auth)
-
-  def get_url(self, bucket=None, path=None, perm='read'):
-    # No GET/POST spec?
-    # e.g. get_url('<storage_account?>', '<bucket>', '<relative_path>', 'read')
-    return self.ranger.get_signed_url(bucket, path, perm)
+  def get_url(self, action='GET', path=None, perm='read'):
+    c = get_raz_client(
+      raz_url=RAZ.API_URL.get(),
+      username='csso_romain',
+      auth=RAZ.API_AUTHENTICATION.get(),
+      service='s3',
+      service_name='cm_s3',
+      cluster_name='prakashdh62'
+    )
+
+    return c.check_access(method=action, url=path)
 
 
 
 
 class AdlsRazClient():
 class AdlsRazClient():

+ 5 - 2
desktop/core/src/desktop/lib/raz/raz_client.py

@@ -83,6 +83,8 @@ class RazClient(object):
     self.requestid = str(uuid.uuid4())
     self.requestid = str(uuid.uuid4())
 
 
   def check_access(self, method, url, params=None, headers=None):
   def check_access(self, method, url, params=None, headers=None):
+    LOG.debug("Check access: method {%s}, header: {%s}" % (method, headers))
+
     path = lib_urlparse(url)
     path = lib_urlparse(url)
     url_params = dict([p.split('=') for p in path.query.split('&') if path.query])
     url_params = dict([p.split('=') for p in path.query.split('&') if path.query])
     params = params if params is not None else {}
     params = params if params is not None else {}
@@ -128,10 +130,11 @@ class RazClient(object):
       }
       }
     }
     }
     headers = {"Content-Type":"application/json", "Accept-Encoding":"gzip,deflate"}
     headers = {"Content-Type":"application/json", "Accept-Encoding":"gzip,deflate"}
-    rurl = "%s/api/authz/s3/access?delegation=%s" % (self.raz_url, self.raz_token)
+    raz_url = "%s/api/authz/s3/access?delegation=%s" % (self.raz_url, self.raz_token)
+    LOG.debug('Raz url: %s' % raz_url)
 
 
     LOG.debug("Sending access check headers: {%s} request_data: {%s}" % (headers, request_data))
     LOG.debug("Sending access check headers: {%s} request_data: {%s}" % (headers, request_data))
-    raz_req = requests.post(rurl, headers=headers, json=request_data, verify=False)
+    raz_req = requests.post(raz_url, headers=headers, json=request_data, verify=False)
 
 
     s3_sign_response = None
     s3_sign_response = None
     signed_response = None
     signed_response = None

+ 15 - 12
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -108,8 +108,8 @@ class RazS3Connection(SignedUrlS3Connection):
     boto.log.debug('path=%s' % path)
     boto.log.debug('path=%s' % path)
     auth_path = self.calling_format.build_auth_path(bucket, key)
     auth_path = self.calling_format.build_auth_path(bucket, key)
     boto.log.debug('auth_path=%s' % auth_path)
     boto.log.debug('auth_path=%s' % auth_path)
-    # host = self.calling_format.build_host(self.server_name(), bucket)
-    host = self.calling_format.build_host(self.server_name(), '')  # As using signed Url we keep the same hostname as there
+    host = self.calling_format.build_host(self.server_name(), bucket)
+    #host = self.calling_format.build_host(self.server_name(), '')  # As using signed Url we keep the same hostname as there
     if query_args:
     if query_args:
         path += '?' + query_args
         path += '?' + query_args
         boto.log.debug('path=%s' % path)
         boto.log.debug('path=%s' % path)
@@ -121,17 +121,19 @@ class RazS3Connection(SignedUrlS3Connection):
                                                 params, headers, data, host)
                                                 params, headers, data, host)
 
 
     # Actual override starts here
     # Actual override starts here
-    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
+    LOG.debug('http_request: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
+    LOG.debug('http_request object: %s' % http_request)
 
 
-    signed_url = self.get_signed_url(action=method, bucket_name=bucket, object_name=key)
-    LOG.debug(signed_url)
+    url = 'https://%(host)s%(path)s' % {'host': host, 'path': path}
 
 
-    parsed_url = lib_urlparse(signed_url)
+    headers = self.get_signed_url(action=method, url=url)
+    LOG.debug('Raz returned those headers: %s' % headers)
 
 
-    # We override instead of re-creating an HTTPRequest
-    http_request.path = parsed_url.path
-    if parsed_url.query:
-      http_request.path += '?' + parsed_url.query
+    if headers is not None:
+      # We override instead of re-creating an HTTPRequest
+      http_request.headers.update(headers)
+    else:
+      LOG.error('We got back empty header from Raz for the request %s' % http_request)
 
 
     LOG.debug('Overriden: %s' % http_request)
     LOG.debug('Overriden: %s' % http_request)
 
 
@@ -139,9 +141,10 @@ class RazS3Connection(SignedUrlS3Connection):
                       retry_handler=retry_handler)
                       retry_handler=retry_handler)
 
 
 
 
-  def get_signed_url(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
+  def get_signed_url(self, action='GET', url=None):
     raz_client = S3RazClient()
     raz_client = S3RazClient()
-    return raz_client.get_url(bucket_name, object_name)
+
+    return raz_client.get_url(action, url)
 
 
 
 
 class SelfSignedUrlS3Connection(SignedUrlS3Connection):
 class SelfSignedUrlS3Connection(SignedUrlS3Connection):