瀏覽代碼

[raz] Remove old unused Raz client code (#3611)

* [raz] Remove old unused Raz client code

- This code were initial prototype implementations of Ranger Raz client in Hue which were refined after each interations to current design.
- These client are not reference anywhere now.

* [raz] Remove old raz code
Harsh Gupta 1 年之前
父節點
當前提交
667ae0679c

+ 0 - 141
desktop/core/src/desktop/lib/raz/ranger/__init__.py

@@ -1,141 +0,0 @@
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import absolute_import
-
-from builtins import map
-from future.utils import raise_
-import calendar
-import errno
-import logging
-import posixpath
-import re
-import sys
-import time
-
-from functools import wraps
-
-from boto.exception import S3ResponseError
-from hadoop.fs import normpath as fs_normpath
-
-
-ERRNO_MAP = {
-  403: errno.EACCES,
-  404: errno.ENOENT
-}
-DEFAULT_ERRNO = errno.EINVAL
-
-S3_PATH_RE = re.compile('^/*[sS]3[aA]?://([^/]+)(/(.*?([^/]+)?/?))?$')
-S3_ROOT = 's3://'
-S3A_ROOT = 's3a://'
-
-
-def lookup_s3error(error):
-  err_no = ERRNO_MAP.get(error.status, DEFAULT_ERRNO)
-  return IOError(err_no, error.reason)
-
-
-def translate_s3_error(fn):
-  @wraps(fn)
-  def wrapped(*args, **kwargs):
-    try:
-      return fn(*args, **kwargs)
-    except S3ResponseError:
-      _, exc, tb = sys.exc_info()
-      logging.error('S3 error: %s' % exc)
-      lookup = lookup_s3error(exc)
-      raise_(lookup.__class__, lookup, tb)
-  return wrapped
-
-
-def parse_uri(uri):
-  """
-  Returns tuple (bucket_name, key_name, key_basename).
-  Raises ValueError if invalid S3 URI is passed.
-  """
-  match = S3_PATH_RE.match(uri)
-  if not match:
-    raise ValueError("Invalid S3 URI: %s" % uri)
-  key = match.group(3) or ''
-  basename = match.group(4) or ''
-  return match.group(1), key, basename
-
-
-def is_root(uri):
-  """
-  Check if URI is S3 root (S3A://)
-  """
-  return uri.lower() == S3A_ROOT
-
-
-def abspath(cd, uri):
-  """
-  Returns absolute URI, examples:
-
-  abspath('s3a://bucket/key', key2') == 's3a://bucket/key/key2'
-  abspath('s3a://bucket/key', 's3a://bucket2/key2') == 'sa://bucket2/key2'
-  """
-  if cd.lower().startswith(S3A_ROOT):
-    uri = join(cd, uri)
-  else:
-    uri = normpath(join(cd, uri))
-  return uri
-
-
-def join(*comp_list):
-  def _prep(uri):
-    try:
-      return '/%s/%s' % parse_uri(uri)[:2]
-    except ValueError:
-      return '/' if is_root(uri) else uri
-  joined = posixpath.join(*list(map(_prep, comp_list)))
-  if joined and joined[0] == '/':
-    joined = 's3a:/%s' % joined
-  return joined
-
-
-def normpath(path):
-  """
-  Return normalized path but ignore leading S3A_ROOT prefix if it exists
-  """
-  if path.lower().startswith(S3A_ROOT):
-    if is_root(path):
-      normalized = path
-    else:
-      normalized = '%s%s' % (S3A_ROOT, fs_normpath(path[len(S3A_ROOT):]))
-  else:
-    normalized = fs_normpath(path)
-  return normalized
-
-
-def s3datetime_to_timestamp(datetime):
-  """
-  Returns timestamp (seconds) by datetime string from S3 API responses.
-  S3 REST API returns two types of datetime strings:
-  * `Thu, 26 Feb 2015 20:42:07 GMT` for Object HEAD requests
-    (see http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html);
-  * `2015-02-26T20:42:07.000Z` for Bucket GET requests
-    (see http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html).
-  """
-  # There is chance (depends on platform) to get
-  # `'z' is a bad directive in format ...` error (see https://bugs.python.org/issue6641),
-  # but S3 always returns time in GMT, so `GMT` and `.000Z` can be pruned.
-  try:
-    stripped = time.strptime(datetime[:-4], '%a, %d %b %Y %H:%M:%S')
-    assert datetime[-4:] == ' GMT', 'Time [%s] is not in GMT.' % datetime
-  except ValueError:
-    stripped = time.strptime(datetime[:-5], '%Y-%m-%dT%H:%M:%S')
-    assert datetime[-1:] == 'Z' and datetime[-5:-4] == '.', 'Time [%s] is not in GMT.' % datetime
-  return int(calendar.timegm(stripped))

+ 0 - 16
desktop/core/src/desktop/lib/raz/ranger/clients/__init__.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

+ 0 - 55
desktop/core/src/desktop/lib/raz/ranger/clients/ranger_raz_adls.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from desktop.lib.raz.ranger.model.ranger_raz import RangerRazRequest, ResourceAccess
-from desktop.lib.raz.ranger.clients.ranger_raz_client import RangerRazClient
-
-
-LOG = logging.getLogger()
-
-
-class RangerRazAdls:
-  def __init__(self, url, auth):
-    self.razClient = RangerRazClient(url, auth)
-    # move `storage_account` into constructor?
-
-  def get_raz_client(self):
-    return self.razClient
-
-  def get_dsas_token(self, storage_account, container, relative_path, action="read"):
-    req = RangerRazRequest()
-
-    req.serviceType = "adls"
-    req.operation = ResourceAccess(
-      {
-        "resource": {
-          "storageaccount": storage_account,
-          "container": container,
-          "relativepath": relative_path,
-        },
-        "action": action,
-      }
-    )
-
-    res = self.razClient.check_privilege(req)
-
-    # TODO: Check if no access inside RangerRazResult and raise exception?
-
-    return res.operResult.additionalInfo["ADLS_DSAS"]

+ 0 - 261
desktop/core/src/desktop/lib/raz/ranger/clients/ranger_raz_client.py

@@ -1,261 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import logging
-import os
-
-from requests import Session, Response
-from desktop.lib.raz.ranger.model.ranger_raz import RangerRazResult
-from apache_ranger.utils import *
-
-
-LOG = logging.getLogger()
-
-
-class RangerRazClient:
-  def __init__(self, url, auth):
-    self.url = url
-    self.session = Session()
-    self.session.auth = auth
-
-    logging.getLogger("requests").setLevel(logging.WARNING)
-
-  def get_delegation_token(self, renewer, dtServiceName=None, doAsUser=None):
-    resp = None
-
-    if self.__is_kerberos_authenticated():
-      resp = self.__call_api(
-        RangerRazClient.GET_DELEGATION_TOKEN,
-        self.__get_query_params(
-          {
-            RangerRazClient.PARAM_OP: RangerRazClient.GET_DELEGATION_TOKEN,
-            RangerRazClient.PARAM_RENEWER: renewer,
-            RangerRazClient.PARAM_DT_SERVICENAME: dtServiceName,
-          },
-          doAsUser,
-        ),
-      )
-    else:
-      LOG.error("Kerberos Authentication is required to get RAZ delegation token")
-
-    return resp
-
-  def renew_delegation_token(self, delegation_token, doAsUser=None):
-    resp = None
-
-    if self.__is_kerberos_authenticated():
-      resp = self.__call_api(
-        RangerRazClient.RENEW_DELEGATION_TOKEN,
-        self.__get_query_params(
-          {
-            RangerRazClient.PARAM_OP: RangerRazClient.RENEW_DELEGATION_TOKEN,
-            RangerRazClient.PARAM_TOKEN: delegation_token,
-          },
-          doAsUser,
-        ),
-      )
-    else:
-      LOG.error(
-        "Kerberos Authentication is required to renew RAZ delegation token"
-      )
-
-    return resp
-
-  def cancel_delegation_token(self, delegation_token, doAsUser=None):
-    resp = None
-
-    if self.__is_kerberos_authenticated():
-      resp = self.__call_api(
-        RangerRazClient.CANCEL_DELEGATION_TOKEN,
-        self.__get_query_params(
-          {
-            RangerRazClient.PARAM_OP: RangerRazClient.CANCEL_DELEGATION_TOKEN,
-            RangerRazClient.PARAM_TOKEN: delegation_token,
-          },
-          doAsUser,
-        ),
-      )
-    else:
-      LOG.error(
-        "Kerberos Authentication is required to cancel RAZ delegation token"
-      )
-
-    return resp
-
-  def check_privilege(self, raz_request, doAsUser=None):
-    resp = self.__call_api(
-      RangerRazClient.CHECK_PRIVILEGE.format_path(
-        {"serviceType": raz_request.serviceType}
-      ),
-      query_params=self.__get_query_params(None, doAsUser),
-      request_data=raz_request,
-    )
-
-    return type_coerce(resp, RangerRazResult)
-
-  def check_privileges(self, raz_requests, doAsUser=None):
-    resp = self.__call_api(
-      RangerRazClient.CHECK_PRIVILEGES.format_path(
-        {"serviceType": raz_requests.serviceType}
-      ),
-      query_params=self.__get_query_params(None, doAsUser),
-      request_data=raz_requests,
-    )
-
-    return type_coerce_list(resp, RangerRazResult)
-
-  def __is_kerberos_authenticated(self):
-    from requests_kerberos import HTTPKerberosAuth
-
-    return isinstance(self.session.auth, HTTPKerberosAuth)
-
-  def __get_query_params(self, query_params, doAsUser=None):
-    if doAsUser is not None:
-      query_params = query_params or {}
-
-      query_params[RangerRazClient.PARAM_DOAS] = doAsUser
-
-    return query_params
-
-  def __call_api(self, api, query_params=None, request_data=None):
-    ret = None
-    params = {"headers": {"Accept": api.consumes, "Content-type": api.produces}}
-
-    if query_params:
-      params["params"] = query_params
-
-    if request_data:
-      params["data"] = json.dumps(request_data)
-
-    path = os.path.join(self.url, api.path)
-
-    if LOG.isEnabledFor(logging.DEBUG):
-      LOG.debug("------------------------------------------------------")
-      LOG.debug("Call         : %s %s", api.method, path)
-      LOG.debug("Content-type : %s", api.consumes)
-      LOG.debug("Accept       : %s", api.produces)
-
-    response = None
-
-    if api.method == HttpMethod.GET:
-      response = self.session.get(path, **params)
-    elif api.method == HttpMethod.POST:
-      response = self.session.post(path, **params)
-    elif api.method == HttpMethod.PUT:
-      response = self.session.put(path, **params)
-    elif api.method == HttpMethod.DELETE:
-      response = self.session.delete(path, **params)
-
-    if LOG.isEnabledFor(logging.DEBUG):
-      LOG.debug("HTTP Status: %s", response.status_code if response else "None")
-
-    if response is None:
-      ret = None
-    elif response.status_code == api.expected_status:
-      try:
-        if response.content is not None:
-          if LOG.isEnabledFor(logging.DEBUG):
-            LOG.debug(
-              "<== __call_api(%s, %s, %s), result=%s",
-              vars(api),
-              params,
-              request_data,
-              response,
-            )
-
-            LOG.debug(response.json())
-
-          ret = response.json()
-        else:
-          ret = None
-      except Exception as e:
-        print(e)
-
-        LOG.exception(
-          "Exception occurred while parsing response with msg: %s", e
-        )
-
-        raise RangerRazException(api, response)
-    elif response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
-      LOG.error(
-        "Ranger Raz server unavailable. HTTP Status: %s",
-        HTTPStatus.SERVICE_UNAVAILABLE,
-      )
-
-      ret = None
-    else:
-      raise RangerRazException(api, response)
-
-    return ret
-
-  # URIs
-  PARAM_OP = "op"
-  PARAM_RENEWER = "renewer"
-  PARAM_TOKEN = "token"
-  PARAM_DELEGATION = "delegation"
-  PARAM_DOAS = "doAs"
-  PARAM_DT_SERVICENAME = "service"
-  OP_GETDELEGATIONTOKEN = "GETDELEGATIONTOKEN"
-  OP_RENEWDELEGATIONTOKEN = "RENEWDELEGATIONTOKEN"
-  OP_CANCELDELEGATIONTOKEN = "CANCELDELEGATIONTOKEN"
-  URI_DELEGATION_TOKEN = ""
-  URI_CHECK_PRIVILEGE = "api/authz/{serviceType}/access"
-  URI_CHECK_PRIVILEGES = "api/authz/{serviceType}/accesses"
-
-  # APIs
-  GET_DELEGATION_TOKEN = API(URI_DELEGATION_TOKEN, HttpMethod.GET, HTTPStatus.OK)
-  RENEW_DELEGATION_TOKEN = API(URI_DELEGATION_TOKEN, HttpMethod.PUT, HTTPStatus.OK)
-  CANCEL_DELEGATION_TOKEN = API(URI_DELEGATION_TOKEN, HttpMethod.PUT, HTTPStatus.OK)
-  CHECK_PRIVILEGE = API(URI_CHECK_PRIVILEGE, HttpMethod.POST, HTTPStatus.OK)
-  CHECK_PRIVILEGES = API(URI_CHECK_PRIVILEGES, HttpMethod.POST, HTTPStatus.OK)
-
-
-class RangerRazException(Exception):
-  """Exception raised for errors in API calls.
-
-  Attributes:
-    api      -- api endpoint which caused the error
-    response -- response from the server
-  """
-
-  def __init__(self, api, response):
-    self.method = api.method.name
-    self.path = api.path
-    self.expected_status = api.expected_status
-    self.statusCode = -1
-    self.msgDesc = None
-    self.messageList = None
-
-    print(response)
-
-    if api is not None and response is not None:
-      self.statusCode = response.status_code
-      self.message = response.content
-
-    Exception.__init__(
-      self,
-      "{} {} failed: expected_status={}, status={}, message={}".format(
-        self.method,
-        self.path,
-        self.expected_status,
-        self.statusCode,
-        self.message,
-      ),
-    )

+ 0 - 61
desktop/core/src/desktop/lib/raz/ranger/clients/ranger_raz_s3.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from desktop.lib.raz.ranger.model.ranger_raz import RangerRazRequest, ResourceAccess
-from desktop.lib.raz.ranger.clients.ranger_raz_client import RangerRazClient
-
-
-LOG = logging.getLogger()
-
-
-class RangerRazS3:
-  def __init__(self, url, auth):
-    self.razClient = RangerRazClient(url, auth)
-
-  def get_signed_url(self, region, bucket, relative_path, action="read"):
-    req = RangerRazRequest()
-
-    # endpoint_prefix="s3",
-    # service_name="s3",
-    # endpoint=endpoint, # https://s3-us-west-1.amazonaws.com
-    # http_method=self.request.method,
-    # headers=headers,
-    # parameters=allparams,
-    # resource_path=resource_path,
-    # time_offset=0
-
-    req.serviceType = "s3"
-    req.operation = ResourceAccess(
-      # TODO: parameters for S3
-      {
-        "resource": {
-          "storageaccount": region,
-          "container": bucket,
-          "relativepath": relative_path,
-        },
-        "action": action,
-      }
-    )
-
-    res = self.razClient.check_privilege(req)
-
-    # TODO: Check if no access inside RangerRazResult and raise exception, cf. res["operResult"]["result"]=="ALLOWED":
-
-    return res.operResult.additionalInfo["S3_SIGN_RESPONSE"]

+ 0 - 16
desktop/core/src/desktop/lib/raz/ranger/model/__init__.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

+ 0 - 163
desktop/core/src/desktop/lib/raz/ranger/model/ranger_raz.py

@@ -1,163 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from apache_ranger.model.ranger_base import *
-from apache_ranger.utils import *
-
-
-class RangerRazRequestBase(RangerBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerBase.__init__(self, attrs)
-
-    self.requestId = attrs.get("requestId")
-    self.serviceType = attrs.get("serviceType")
-    self.serviceName = attrs.get("serviceName")
-    self.user = attrs.get("user")
-    self.userGroups = attrs.get("userGroups")
-    self.accessTime = attrs.get("accessTime")
-    self.clientIpAddress = attrs.get("clientIpAddress")
-    self.clientType = attrs.get("clientType")
-    self.clusterName = attrs.get("clusterName")
-    self.clusterType = attrs.get("clusterType")
-    self.sessionId = attrs.get("sessionId")
-    self.context = attrs.get("context")
-
-  def type_coerce_attrs(self):
-    super(RangerRazRequestBase, self).type_coerce_attrs()
-
-
-class ResourceAccess(RangerBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerBase.__init__(self, attrs)
-
-    self.resource = attrs.get("resource")
-    self.resourceOwner = attrs.get("resourceOwner")
-    self.action = attrs.get("action")
-    self.accessTypes = attrs.get("accessTypes")
-
-  def type_coerce_attrs(self):
-    super(ResourceAccess, self).type_coerce_attrs()
-
-
-class RangerRazRequest(RangerRazRequestBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerRazRequestBase.__init__(self, attrs)
-
-    self.operation = attrs.get("operation")
-
-  def type_coerce_attrs(self):
-    super(RangerRazRequest, self).type_coerce_attrs()
-
-    self.operation = type_coerce(self.operation, ResourceAccess)
-
-
-class RangerRazMultiOperationRequest(RangerRazRequestBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerRazRequestBase.__init__(self, attrs)
-
-    self.operations = attrs.get("operations")
-
-  def type_coerce_attrs(self):
-    super(RangerRazMultiOperationRequest, self).type_coerce_attrs()
-
-    self.operation = type_coerce_list(self.operation, ResourceAccess)
-
-
-class RangerRazResultBase(RangerBase):
-  ALLOWED = 0
-  DENIED = 1
-  NOT_DETERMINED = 2
-
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerBase.__init__(self, attrs)
-
-    self.requestId = attrs.get("requestId")
-
-  def type_coerce_attrs(self):
-    super(RangerRazResultBase, self).type_coerce_attrs()
-
-
-class AuditInfo(RangerBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerBase.__init__(self, attrs)
-
-    self.auditId = attrs.get("auditId")
-    self.accessType = attrs.get("accessType")
-    self.result = attrs.get("result")
-    self.policyId = attrs.get("policyId")
-    self.policyVersion = attrs.get("policyVersion")
-
-  def type_coerce_attrs(self):
-    super(AuditInfo, self).type_coerce_attrs()
-
-
-class ResourceAccessResult(RangerBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerBase.__init__(self, attrs)
-
-    self.result = attrs.get("result")
-    self.isAudited = attrs.get("isAudited")
-    self.auditLogs = attrs.get("auditLogs")
-    self.additionalInfo = attrs.get("additionalInfo")
-
-  def type_coerce_attrs(self):
-    super(ResourceAccessResult, self).type_coerce_attrs()
-
-    self.auditLogs = type_coerce_list(self.operation, AuditInfo)
-
-
-class RangerRazResult(RangerRazResultBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerRazResultBase.__init__(self, attrs)
-
-    self.operResult = attrs.get("operResult")
-
-  def type_coerce_attrs(self):
-    super(RangerRazResult, self).type_coerce_attrs()
-
-    self.operResult = type_coerce(self.operResult, ResourceAccessResult)
-
-
-class RangerRazMultiOperationResult(RangerRazResultBase):
-  def __init__(self, attrs=None):
-    attrs = attrs or {}
-
-    RangerRazRequestBase.__init__(self, attrs)
-
-    self.operResults = attrs.get("operResults")
-
-  def type_coerce_attrs(self):
-    super(RangerRazMultiOperationResult, self).type_coerce_attrs()
-
-    self.operResults = type_coerce_list(self.operResults, ResourceAccessResult)

+ 3 - 6
desktop/libs/aws/src/aws/client.py

@@ -21,7 +21,7 @@ import os
 import boto
 
 from aws import conf as aws_conf
-from aws.s3.s3connection import url_client_connect_to_region, RazS3Connection
+from aws.s3.s3connection import RazS3Connection
 from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 
 from desktop.lib.idbroker import conf as conf_idbroker
@@ -173,14 +173,11 @@ class Client(object):
     try:
       # Use V4 signature support by default
       os.environ['S3_USE_SIGV4'] = 'True'
-      if self._host is not None and not aws_conf.IS_SELF_SIGNING_ENABLED.get():
+      if self._host is not None:
         kwargs.update({'host': self._host})
         connection = boto.s3.connection.S3Connection(**kwargs)
       elif self._region:
-        if aws_conf.IS_SELF_SIGNING_ENABLED.get():
-          connection = url_client_connect_to_region(self._region, **kwargs)
-        else:
-          connection = boto.s3.connect_to_region(self._region, **kwargs)
+        connection = boto.s3.connect_to_region(self._region, **kwargs)
       else:
         kwargs.update({'host': 's3.amazonaws.com'})
         connection = boto.s3.connection.S3Connection(**kwargs)

+ 1 - 8
desktop/libs/aws/src/aws/conf.py

@@ -164,13 +164,6 @@ HAS_IAM_DETECTION = Config(
   type=coerce_bool
 )
 
-IS_SELF_SIGNING_ENABLED = Config(
-  key='is_self_signing_enabled',
-  help=_('Skip boto and use self signed URL and requests to make the calls to S3. Used for testing the RAZ integration.'),
-  type=coerce_bool,
-  private=True,
-  default=False,
-)
 
 def get_default_get_environment_credentials():
   '''Allow to check if environment credentials are present or not'''
@@ -348,7 +341,7 @@ def is_raz_s3():
   from desktop.conf import RAZ  # Must be imported dynamically in order to have proper value
 
   return (RAZ.IS_ENABLED.get() and 'default' in list(AWS_ACCOUNTS.keys()) and \
-          AWS_ACCOUNTS['default'].HOST.get() and AWS_ACCOUNTS['default'].get_raw() and not IS_SELF_SIGNING_ENABLED.get())
+          AWS_ACCOUNTS['default'].HOST.get() and AWS_ACCOUNTS['default'].get_raw())
 
 
 def config_validator(user):

+ 1 - 303
desktop/libs/aws/src/aws/s3/s3connection.py

@@ -37,7 +37,6 @@ from boto.s3.prefix import Prefix
 
 from desktop.conf import RAZ
 from desktop.lib.raz.clients import S3RazClient
-from aws.conf import IS_SELF_SIGNING_ENABLED
 
 
 LOG = logging.getLogger()
@@ -71,7 +70,7 @@ class SignedUrlS3Connection(S3Connection):
     self.username = username
 
     # No auth handler with RAZ
-    anon = RAZ.IS_ENABLED.get() and not IS_SELF_SIGNING_ENABLED.get()
+    anon = RAZ.IS_ENABLED.get()
 
     super(SignedUrlS3Connection, self).__init__(
       aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
@@ -176,304 +175,3 @@ class RazS3Connection(SignedUrlS3Connection):
   '''
   def _required_auth_capability(self):
     return ['anon']
-
-
-class SelfSignedUrlS3Connection(SignedUrlS3Connection):
-  """
-  Test class self generating presigned Urls so that the Http Client using signed Urls instead
-  of direct boto calls to S3 can be tested.
-  """
-  def make_request(self, method, bucket='', key='', headers=None, data='',
-                    query_args=None, sender=None, override_num_retries=None,
-                    retry_handler=None):
-    if isinstance(bucket, self.bucket_class):
-      bucket = bucket.name
-    if isinstance(key, Key):
-      key = key.name
-    path = self.calling_format.build_path_base(bucket, key)
-    boto.log.debug('path=%s' % path)
-    auth_path = self.calling_format.build_auth_path(bucket, key)
-    boto.log.debug('auth_path=%s' % auth_path)
-    host = self.calling_format.build_host(self.server_name(), bucket)
-    if query_args:
-      path += '?' + query_args
-      boto.log.debug('path=%s' % path)
-      auth_path += '?' + query_args
-      boto.log.debug('auth_path=%s' % auth_path)
-
-    params = {}
-    http_request = self.build_base_http_request(method, path, auth_path,
-                                                params, headers, data, host)
-
-    # Actual override starts here
-    LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
-    LOG.debug('Overriding: %s' % http_request)
-
-    p = http_request.path.split('/')
-    bucket = (p[1] + '/') or ''
-    key = '/'.join(p[2:]) if len(p) >= 3 else ''
-
-    kwargs = {
-        'bucket': bucket,
-        'key': key
-    }
-
-    # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-    signed_url = self.generate_url(1000, method, **kwargs)
-    LOG.debug('Generated url: %s' % signed_url)
-
-    http_request.path = signed_url.replace(http_request.protocol + '://' + http_request.host.split(':')[0], '')
-    p, h = http_request.path.split('?')
-    http_request.path = unquote(p)
-    http_request.headers = dict([a.split('=') for a in h.split('&')])
-
-    LOG.debug('Overriden: %s' % http_request)
-
-    return self._mexe(http_request, sender, override_num_retries,
-                      retry_handler=retry_handler)
-
-
-def url_client_connect_to_region(region_name, **kw_params):
-  if 'host' in kw_params:
-    host = kw_params.pop('host')
-    if host not in ['', None]:
-      region = S3RegionInfo(
-          name='custom',
-          endpoint=host,
-          connection_cls=SelfSignedUrlS3Connection  # Override S3Connection class in connect_to_region of boto/s3/__init__.py
-      )
-      return region.connect(**kw_params)
-
-  return connect('s3', region_name, region_cls=S3RegionInfo,
-                   connection_cls=SelfSignedUrlS3Connection, **kw_params)
-
-
-
-# --------------------------------------------------------------------------------
-# Deprecated Client: to remove at v1
-#
-# This clients re-implement S3Connection methods via a PreSignedUrl either
-# provided by a RAZ server or another Boto lib. Request to S3 are then made via
-# requests and the raw XML is unmarshalling back to boto2 Python objects.
-#
-# Note: hooking-in the get/generate URL directly into S3Connection#make_request()
-# was found to be simpler and possible as boto itself sends signed Urls.
-# Handling various operations is relatively simple as defined by HTTP action and
-# paths. Most of the security is handled via header parameters.
-# --------------------------------------------------------------------------------
-
-class SignedUrlClient():
-  """
-  Share the unmarshalling from XML to boto Python objects from the requests calls.
-  """
-
-  def get_all_buckets(self, headers=None):
-    LOG.debug('get_all_buckets: %s' % headers)
-    kwargs = {'action': 'GET'}
-
-    signed_url = self.get_url_request(**kwargs)
-    LOG.debug(signed_url)
-
-    response = requests.get(signed_url)
-
-    LOG.debug(response)
-    LOG.debug(response.content)
-
-    rs = ResultSet([('Bucket', UrlBucket)])
-    h = boto.handler.XmlHandler(rs, None)
-    xml.sax.parseString(response.content, h)
-    LOG.debug(rs)
-
-    return rs
-
-
-
-class RazSignedUrlClient(SignedUrlClient):
-
-  def __init__(self):
-    self.raz = S3RazClient()
-
-  def get_url_request(self, action='GET', bucket_name=None, object_name=None, expiration=3600):
-    self.raz.get_url(bucket_name, object_name)
-
-
-
-class UrlKey(Key):
-
-  def open_read(self, headers=None, query_args='', override_num_retries=None, response_headers=None):
-    LOG.debug('open_read: %s' % self.name)
-
-    # Similar to Key.get_key('GET')
-    # data = self.resp.read(self.BufferSize)
-    # For seek: headers={"Range": "bytes=%d-" % pos}
-
-    if self.resp is None:
-      self.resp = self.bucket.get_key(key_name=self.name, validate=False, action='GET')
-
-
-  def read(self, size=0):
-    return self.resp.read(size) if self.resp else ''
-
-
-  def get_url_request(self, action='GET', **kwargs):
-    LOG.debug(kwargs)
-    signed_url = None
-
-    try:
-      # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
-      signed_url = self.generate_url(self.expiration, action, **kwargs)
-      LOG.debug('Generated url: %s' % signed_url)
-    except BotoClientError as e:
-      LOG.error(e)
-      if signed_url is None:
-        from aws.s3.s3fs import S3FileSystemException
-        raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
-
-    return signed_url
-
-
-class UrlBucket(Bucket):
-
-  def list(self, prefix='', delimiter='', marker='', headers=None, encoding_type=None):
-    params = {
-      'prefix': prefix,
-      'delimiter': delimiter
-    }
-    return self.get_all_keys(**params)
-
-
-  def get_key(self, key_name, headers=None, version_id=None, response_headers=None, validate=True, action='HEAD'):
-    LOG.debug('key name: %s %s' % (self.name, key_name))
-    kwargs = {'bucket': self.name, 'key': key_name}
-
-    # TODO: if GET --> max length to add
-
-    signed_url = self.connection.generate_url(3000, action, **kwargs)
-    LOG.debug('Generated url: %s' % signed_url)
-
-    if action == 'HEAD':
-      response = requests.head(signed_url)
-    else:
-      response = requests.get(signed_url)
-
-    LOG.debug(response)
-    LOG.debug(response.content)
-
-    response.getheader = response.headers.get
-    response.getheaders = lambda: response.headers
-
-    # Copied from boto2 bucket.py _get_key_internal()
-    if response.status_code / 100 == 2:
-      k = self.key_class(self)
-      provider = self.connection.provider
-      # k.metadata = boto.utils.get_aws_metadata(response.msg, provider)
-      for field in Key.base_fields:
-        k.__dict__[field.lower().replace('-', '_')] = \
-          response.getheader(field)
-      # the following machinations are a workaround to the fact that
-      # apache/fastcgi omits the content-length header on HEAD
-      # requests when the content-length is zero.
-      # See http://goo.gl/0Tdax for more details.
-      clen = response.getheader('content-length')
-      if clen:
-        k.size = int(response.getheader('content-length'))
-      else:
-        k.size = 0
-      k.name = key_name
-      k.handle_version_headers(response)
-      k.handle_encryption_headers(response)
-      k.handle_restore_headers(response)
-      k.handle_addl_headers(response.getheaders())
-
-      class MockResponse():
-        def __init__(self, resp):
-          self.resp = resp
-        def read(self, size):
-          return self.resp.content
-
-      k.resp = MockResponse(response)
-    else:
-      # Currently needed as 404 on directories via stats_key()
-      k = self.key_class(self, key_name)
-
-    return k
-
-
-  def get_all_keys(self, headers=None, **params):
-    kwargs = {'bucket': self.name, 'key': '', 'response_headers': params}
-
-    signed_url = self.connection.generate_url(3000, 'GET', **kwargs)
-    LOG.debug('Generated url: %s' % signed_url)
-
-    response = requests.get(signed_url)
-
-    LOG.debug('get_all_keys %s' % kwargs)
-    LOG.debug(params)
-    LOG.debug(response)
-    LOG.debug(response.content)
-
-    rs = ResultSet([('Contents', UrlKey), ('CommonPrefixes', Prefix)])
-    h = boto.handler.XmlHandler(rs, self)
-    xml.sax.parseString(response.content, h)
-    LOG.debug(rs)
-
-    return rs
-
-
-  def get_url_request(self, action='GET', **kwargs):
-    LOG.debug(kwargs)
-    signed_url = None
-
-    try:
-      # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.bucket.Bucket.generate_url
-      signed_url = self.generate_url(self.expiration, action, **kwargs)
-      LOG.debug('Generated url: %s' % signed_url)
-    except BotoClientError as e:
-      LOG.error(e)
-      if signed_url is None:
-        raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
-
-    return signed_url
-
-
-class SelfSignedUrlClient(SignedUrlClient):
-
-  def __init__(self, connection):
-    self.connection = connection
-    self.expiration = 3600
-
-    self.connection.make_request = None  # We make sure we never call via regular boto connection directly
-    self.connection.set_bucket_class(UrlBucket)  # Use our bucket class to keep overriding any direct call to S3 made from list buckets
-
-
-  def get_url_request(self, action='GET', **kwargs):
-    LOG.debug(kwargs)
-    signed_url = None
-
-    try:
-      # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.connection.S3Connection.generate_url
-      signed_url = self.connection.generate_url(self.expiration, action, **kwargs)
-      LOG.debug('Generated url: %s' % signed_url)
-    except BotoClientError as e:
-      LOG.error(e)
-      if signed_url is None:
-        raise IOError("Resource does not exist or permission missing : '%s'" % kwargs)
-
-    return signed_url
-
-
-  def get_bucket(self, bucket_name, validate=True, headers=None):
-    LOG.debug('get_bucket: %s' % bucket_name)
-    kwargs = {'action': 'GET', 'bucket': bucket_name}
-
-    signed_url = self.get_url_request(**kwargs)
-
-    response = requests.get(signed_url)
-
-    LOG.debug(response)
-    LOG.debug(response.content)
-
-    rs = self.connection.bucket_class(self.connection, bucket_name, key_class=UrlKey)  # Using content?
-    LOG.debug(rs)
-
-    return rs

+ 1 - 128
desktop/libs/aws/src/aws/s3/s3connection_test.py

@@ -25,7 +25,7 @@ from nose.tools import assert_equal, assert_true
 from desktop.conf import RAZ
 
 from aws.client import _make_client
-from aws.s3.s3connection import SelfSignedUrlClient, RazSignedUrlClient, SelfSignedUrlS3Connection, RazS3Connection
+from aws.s3.s3connection import RazS3Connection
 from aws.s3.s3test_utils import S3TestBase
 
 if sys.version_info[0] > 2:
@@ -86,130 +86,3 @@ class TestRazS3Connection():
         )
         assert_equal({}, http_request.params)
         assert_equal('', http_request.body)
-
-
-class TestSelfSignedUrlS3Connection():
-
-  def test_get_file(self):
-    with patch('aws.s3.s3connection.SelfSignedUrlS3Connection.generate_url') as generate_url:
-      with patch('aws.s3.s3connection.SelfSignedUrlS3Connection._mexe') as _mexe:
-        with patch('boto.connection.auth.get_auth_handler') as get_auth_handler:
-
-          generate_url.return_value = 'https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv?' + \
-              'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
-              '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-          _mexe.return_value = '[<Bucket: demo-gethue>, <Bucket: gethue-test>]'
-
-          client = SelfSignedUrlS3Connection(username='test')
-          http_request = Mock(
-            path='/gethue/data/customer.csv',
-            protocol='https',
-            host='s3.amazonaws.com'
-          )
-          client.build_base_http_request = Mock(return_value=http_request)
-
-          buckets = client.make_request(method='GET', bucket='gethue', key='data/customer.csv',)
-
-          assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', buckets)
-          _mexe.assert_called_with(http_request, None, None, retry_handler=None)
-
-          assert_equal('https://gethue-test.s3.amazonaws.com/gethue/data/customer.csv', http_request.path)
-          assert_equal(
-            {
-              'AWSAccessKeyId': 'AKIA23E77ZX2HVY76YGL',
-              'Signature': '3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D',
-              'Expires': '1617207304'
-            },
-            http_request.headers
-          )
-
-
-# -----------------------------------------------------------------------------------------------------------
-
-class TestSelfSignedUrlClient():
-
-  def setUp(self):
-    raise SkipTest()
-
-  def test_get_buckets(self):
-    with patch('aws.s3.s3connection.SelfSignedUrlClient.get_url_request') as get_url_request:
-      with patch('aws.s3.s3connection.requests.get') as requests_get:
-
-        get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
-            '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-        requests_get.return_value = Mock(
-          content=b'<?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult '
-            b'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>0429b0aed2900f450655928a09e06e7aaac9939bc9141fc5aeeccd8b93b9778f'
-            b'</ID><DisplayName>team</DisplayName></Owner><Buckets><Bucket><Name>demo-gethue</Name><CreationDate>2020-08-22T08:03:18.000Z'
-            b'</CreationDate></Bucket><Bucket><Name>gethue-test</Name><CreationDate>2021-03-31T14:47:14.000Z</CreationDate></Bucket>'
-            b'</Buckets></ListAllMyBucketsResult>'
-        )
-
-        connection = Mock()
-        buckets = SelfSignedUrlClient(connection=connection).get_all_buckets()
-
-        assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
-
-
-class TestRazSignedUrlClient():
-
-  def setUp(self):
-    raise SkipTest()
-
-  def test_get_buckets(self):
-    with patch('aws.s3.s3connection.RazSignedUrlClient.get_url_request') as get_url_request:
-      with patch('aws.s3.s3connection.requests.get') as requests_get:
-
-        # TODO: update with potentially slightly different URL/headers
-        get_url_request.return_value = 'https://gethue-test.s3.amazonaws.com/?AWSAccessKeyId=AKIA23E77ZX2HVY76YGL' + \
-            '&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-
-        requests_get.return_value = Mock(
-          content=b'<?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult '
-            b'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>0429b0aed2900f450655928a09e06e7aaac9939bc9141fc5aeeccd8b93b9778f'
-            b'</ID><DisplayName>team</DisplayName></Owner><Buckets><Bucket><Name>demo-gethue</Name><CreationDate>2020-08-22T08:03:18.000Z'
-            b'</CreationDate></Bucket><Bucket><Name>gethue-test</Name><CreationDate>2021-03-31T14:47:14.000Z</CreationDate></Bucket>'
-            b'</Buckets></ListAllMyBucketsResult>'
-        )
-
-        buckets = RazSignedUrlClient().get_all_buckets()
-
-        assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
-
-
-class TestSelfSignedUrlClientIntegration(S3TestBase):
-  #
-  # To trigger:
-  # TEST_S3_BUCKET=gethue-test ./build/env/bin/hue test specific aws.s3.s3connection_test
-
-  @classmethod
-  def setUpClass(cls):
-    S3TestBase.setUpClass()
-
-
-  def setUp(self):
-    super(TestSelfSignedUrlClientIntegration, self).setUp()
-
-    self.c = _make_client(identifier='default', user=None)
-    self.connection = self.c._s3_connection.connection
-
-
-  def test_list_buckets(self):
-    buckets = SelfSignedUrlClient(self.connection).get_all_buckets()
-
-    assert_equal('[<Bucket: demo-gethue>, <Bucket: gethue-test>]', str(buckets))
-
-
-  def test_list_file(self):
-    kwargs = {'action': 'GET', 'bucket': 'gethue-test', 'key': 'data/query-hive-weblogs.csv'}
-    url = SelfSignedUrlClient(self.connection).generate_url(**kwargs)
-
-    url = 'https://gethue-test.s3.amazonaws.com/data/query-hive-weblogs.csv?'
-    'AWSAccessKeyId=AKIA23E77ZX2HVY76YGL&Signature=3lhK%2BwtQ9Q2u5VDIqb4MEpoY3X4%3D&Expires=1617207304'
-
-    assert_true('data/query-hive-weblogs.csv' in url)
-    assert_true('AWSAccessKeyId=' in url)
-    assert_true('Signature=' in url)
-    assert_true('Expires=' in url)
-
-    response = requests.get(url)

+ 1 - 1
desktop/libs/aws/src/aws/s3/s3file.py

@@ -22,7 +22,7 @@ from boto.s3.keyfile import KeyFile
 
 from aws.conf import get_key_expiry
 from aws.s3 import translate_s3_error
-from aws.s3.s3connection import UrlKey
+
 
 DEFAULT_READ_SIZE = 1024 * 1024  # 1MB
 

+ 1 - 4
desktop/libs/aws/src/aws/s3/s3test_utils.py

@@ -54,10 +54,7 @@ class S3TestBase(unittest.TestCase):
 
     cls.path_prefix = 'test-hue/%s' % generate_id(size=16)
 
-    if aws_conf.IS_SELF_SIGNING_ENABLED.get():
-      cls.s3_connection = get_client(name='default', fs='s3a', user='hue')._s3_connection
-    else:
-      cls.s3_connection = aws.get_client('default').get_s3_connection()  # Probably broken nowadays
+    cls.s3_connection = aws.get_client('default').get_s3_connection()  # Probably broken nowadays
     cls.bucket = cls.s3_connection.get_bucket(cls.bucket_name, validate=True)
 
   @classmethod