Преглед на файлове

[S3] Stabilize RAZ authentication and enhance S3 client configuration (#4277)

This commit resolves critical `SignatureDoesNotMatch` errors in the boto3-based S3 filesystem when using RAZ authentication and introduces robust client configuration capabilities.

The primary RAZ authentication failures were traced to two root causes:

1.  **GET operations (`ListObjectsV2`) failed** because URL query parameters with empty values (e.g., `prefix=`) were being dropped before the request was sent to RAZ for signing, causing a canonical request mismatch. This is fixed by implementing clean URL parsing methods and preserving these empty values.

2.  **PUT operations (`PutObject`, `Create`) failed** because boto3 automatically uses chunked streaming uploads (`AwsChunkedWrapper`), which requires signing for a special header value (`STREAMING-UNSIGNED-PAYLOAD-TRAILER`). The RAZ client was previously signing for an incorrect hash. This is fixed by detecting the streaming wrapper and providing the correct hash to RAZ.

Additionally, this commit enhances the S3 client architecture:

-   The client creation lifecycle is refactored to allow for pre-configuration of settings before the boto3 session is instantiated.
-   Comprehensive SSL configuration is introduced, respecting global Hue settings (`ssl_validate`, `ssl_cacerts`) while allowing per-connector overrides for `ssl_cert_ca_verify`, `ca_bundle`, and `client_cert`.
Harsh Gupta преди 1 месец
родител
ревизия
2d7cf27db4

+ 236 - 49
desktop/core/src/desktop/lib/fs/s3/clients/auth/raz.py

@@ -17,10 +17,12 @@
 
 import logging
 from typing import Any, Dict, TYPE_CHECKING
-from urllib.parse import urlencode, urlparse, urlunparse
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import boto3
+from botocore import UNSIGNED
 from botocore.awsrequest import AWSRequest
+from botocore.httpchecksum import AwsChunkedWrapper
 
 from desktop.lib.fs.s3.clients.base import S3AuthProvider
 from desktop.lib.raz.clients import S3RazClient
@@ -33,111 +35,296 @@ LOG = logging.getLogger()
 
 class RazEventHandler:
   """
-  Handles RAZ integration with boto3's event system.
-  Intercepts requests before they're signed and sent to S3.
+  Handles RAZ (Ranger Authorization Service) integration with boto3's event system.
+
+  This handler intercepts boto3 requests before they are signed and applies
+  RAZ-signed headers for authenticated access to S3-compatible storage systems.
+
+  Key features:
+  - Handles both streaming and non-streaming uploads
+  - Properly extracts and preserves query parameters (including empty ones)
+  - Supports both AWS virtual-hosted style URLs and generic S3 path-style URLs
+  - Integrates with boto3's UNSIGNED signer to let RAZ handle all signing
   """
 
-  def __init__(self, user: str):
+  def __init__(self, user: str, connector_config: "ConnectorConfig"):
+    """
+    Initialize RAZ event handler.
+
+    Args:
+      user: Username for RAZ authentication
+      connector_config: S3 connector configuration containing provider and endpoint details
+    """
     self.user = user
-    # Use existing S3RazClient which has the correct get_url() interface
+    self.connector_config = connector_config
     self.raz_client = S3RazClient(username=user)
 
+  def _handle_choose_signer(self, **kwargs):
+    """
+    Handle choose-signer boto3 event.
+
+    Instructs boto3 to use the UNSIGNED signer, allowing RAZ to handle
+    all request signing via the before-sign event handler.
+
+    Returns:
+      UNSIGNED: boto3 signer constant indicating no signing should be performed
+    """
+    LOG.debug("Using UNSIGNED signer - RAZ will handle request signing")
+    return UNSIGNED
+
   def _handle_before_sign(self, request: AWSRequest, **kwargs) -> None:
     """
-    Handle before-sign event.
-    This is called before boto3 would normally sign the request.
-    We intercept here to get RAZ to sign instead.
+    Handle before-sign boto3 event.
+
+    This is the core RAZ integration method that:
+    1. Detects streaming vs non-streaming uploads and handles them appropriately
+    2. Extracts query parameters from URLs (preserving empty values)
+    3. Calls RAZ to get signed headers for the clean URL and parameters
+    4. Applies RAZ headers to the request and synchronizes URLs for AWS
+
+    Args:
+      request: The boto3 AWSRequest to be signed by RAZ
+      **kwargs: Additional boto3 event arguments (unused)
+
+    Raises:
+      Exception: If RAZ returns no headers or other signing failures occur
     """
     try:
       # Get request details
-      url = self._get_request_url(request)
       method = request.method
       headers = dict(request.headers)
       data = request.body
 
-      # Get RAZ signed headers
-      raz_headers = self.raz_client.get_url(action=method, url=url, headers=headers, data=data)
+      # Detect streaming uploads by checking for boto3's AwsChunkedWrapper
+      if isinstance(data, AwsChunkedWrapper):
+        # For streaming uploads, boto3 uses a special SHA256 value that RAZ must sign for
+        LOG.debug("Detected streaming upload - using STREAMING-UNSIGNED-PAYLOAD-TRAILER")
+
+        headers["x-amz-content-sha256"] = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+        data_for_raz = b""  # Empty data for signing purposes
+      else:
+        # For non-streaming requests, use the actual data
+        data_for_raz = data
+
+      # Get provider-appropriate URL format and extract query parameters
+      base_url = self._get_request_url(request)
+      parsed_url = urlparse(base_url)
+      clean_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", ""))
+
+      # Extract query parameters for separate RAZ handling
+      params = {}
+      if parsed_url.query:
+        # CRITICAL: keep_blank_values=True preserves empty parameters like prefix=
+        parsed_params = parse_qs(parsed_url.query, keep_blank_values=True)
+        params = {key: values[0] if values else "" for key, values in parsed_params.items()}
+
+      # Get signed headers from RAZ
+      raz_headers = self.raz_client.get_url(action=method, path=clean_url, params=params, headers=headers, data=data_for_raz)
 
       if not raz_headers:
         raise Exception("RAZ returned no signed headers")
 
-      # Update request headers with RAZ signed headers
-      request.headers.update(raz_headers)
+      # Apply all RAZ headers to the request
+      for header_name, header_value in raz_headers.items():
+        request.headers[header_name] = header_value
+
+      # Synchronize request URL with RAZ-signed URL for AWS providers
+      if self.connector_config.provider.lower() == "aws":
+        original_url = request.url
+        parsed_original = urlparse(original_url)
+        parsed_clean = urlparse(clean_url)
+
+        # Use RAZ's clean URL but preserve original query parameters
+        final_url = urlunparse(
+          (
+            parsed_clean.scheme,
+            parsed_clean.netloc,
+            parsed_clean.path,
+            parsed_original.params,
+            parsed_original.query,
+            parsed_original.fragment,
+          )
+        )
+
+        request.url = final_url
+        LOG.debug(f"Synchronized request URL with RAZ signature: {original_url} → {final_url}")
 
-      # Mark request as pre-signed to skip boto3 signing
-      request.context["pre_signed"] = True
+      LOG.info(f"RAZ authentication applied for {method} {clean_url}")
 
     except Exception as e:
-      LOG.error(f"Failed to sign request with RAZ: {e}")
+      LOG.error(f"RAZ authentication failed: {e}")
       raise
 
-  def _handle_before_send(self, request: AWSRequest, **kwargs) -> None:
+  def _get_request_url(self, request: AWSRequest) -> str:
     """
-    Handle before-send event.
-    This is called after signing but before sending.
-    We clean up any leftover AWS headers here.
+    Get request URL in RAZ-compatible format based on provider type.
+
+    AWS providers require virtual-hosted style URLs without explicit port 443,
+    while generic S3-compatible providers use their original endpoint format.
+
+    Args:
+      request: The boto3 AWSRequest to process
+
+    Returns:
+      str: URL formatted appropriately for the provider type
     """
-    # TODO: Is this needed?
-    # Remove any AWS specific headers that RAZ doesn't need
-    aws_headers = ["X-Amz-Security-Token", "X-Amz-Date", "X-Amz-Content-SHA256", "Authorization"]
+    provider = self.connector_config.provider.lower()
 
-    for header in aws_headers:
-      request.headers.pop(header, None)
+    if provider == "aws":
+      return self._get_aws_virtual_hosted_url(request)
+    else:
+      return self._get_generic_provider_url(request)
 
-  def _get_request_url(self, request: AWSRequest) -> str:
+  def _get_aws_virtual_hosted_url(self, request: AWSRequest) -> str:
     """
-    Get full request URL including query parameters.
-    Handles virtual hosted and path style URLs.
+    Convert AWS URLs to virtual-hosted style format.
+
+    AWS S3 virtual-hosted style uses bucket.s3.region.amazonaws.com format
+    and must not include explicit :443 port to ensure Host header consistency
+    with RAZ expectations.
+
+    Args:
+      request: The boto3 AWSRequest to convert
+
+    Returns:
+      str: Virtual-hosted style URL for AWS S3
     """
     url_parts = list(urlparse(request.url))
 
-    # Add query parameters
+    # Extract bucket and key from path
+    path = url_parts[2].lstrip("/")
+    path_parts = path.split("/", 1) if path else ["", ""]
+    bucket_name = path_parts[0] if path_parts[0] else None
+    key_path = path_parts[1] if len(path_parts) > 1 else ""
+
+    # Convert to virtual-hosted style URL for AWS
+    if bucket_name:
+      region = self.connector_config.region or "us-east-1"
+
+      # Build virtual-hosted hostname without explicit port
+      if "s3." in url_parts[1]:
+        # Regional endpoint: s3.us-west-2.amazonaws.com
+        virtual_host = f"{bucket_name}.{url_parts[1]}"
+      else:
+        # Standard endpoint: s3.amazonaws.com
+        virtual_host = f"{bucket_name}.s3.{region}.amazonaws.com"
+
+      # Set virtual-hosted URL components
+      url_parts[1] = virtual_host
+      url_parts[2] = f"/{key_path}" if key_path else "/"
+
+    # Add query parameters if present
     if request.params:
       query = urlencode(request.params)
       url_parts[4] = query
 
-    # Handle virtual hosted style URLs
-    if "s3." in url_parts[1] and request.context.get("bucket_name"):
-      bucket = request.context["bucket_name"]
-      url_parts[1] = f"{bucket}.{url_parts[1]}"
-      # Remove bucket from path
-      url_parts[2] = url_parts[2].replace(f"/{bucket}", "", 1)
+    final_url = urlunparse(url_parts)
+    return final_url
+
+  def _get_generic_provider_url(self, request: AWSRequest) -> str:
+    """
+    Get URL for generic S3-compatible providers.
+
+    Preserves the original endpoint format and adds explicit port
+    if configured in the connector endpoint. This ensures compatibility
+    with various S3-compatible storage systems (MinIO, NetApp, Dell, etc.).
+
+    Args:
+      request: The boto3 AWSRequest to process
+
+    Returns:
+      str: URL formatted for the generic S3 provider
+    """
+    url_parts = list(urlparse(request.url))
+
+    # Add port from connector endpoint if specified and not already present
+    if self.connector_config.endpoint:
+      endpoint_parts = urlparse(self.connector_config.endpoint)
+      if endpoint_parts.port and ":" not in url_parts[1]:
+        url_parts[1] = f"{url_parts[1]}:{endpoint_parts.port}"
+
+    # Add query parameters if present
+    if request.params:
+      query = urlencode(request.params)
+      url_parts[4] = query
 
     return urlunparse(url_parts)
 
 
 class RazAuthProvider(S3AuthProvider):
   """
-  Authentication provider using RAZ for request signing.
-  Uses boto3's event system to intercept and sign requests.
+  Authentication provider that integrates with Ranger Authorization Service (RAZ).
+
+  This provider uses boto3's event system to intercept requests before they are
+  signed and applies RAZ-generated authentication headers. It works by:
+  1. Registering event handlers with boto3's event system
+  2. Using UNSIGNED signer to prevent boto3 from signing requests
+  3. Intercepting requests in the before-sign event to apply RAZ headers
   """
 
   def __init__(self, connector_config: "ConnectorConfig", user: str):
-    super().__init__(connector_config, user)
+    """
+    Initialize RAZ authentication provider.
 
-    # Create RAZ event handler (uses global RAZ configuration internally)
-    self.raz_event_handler = RazEventHandler(user=user)
+    Args:
+      connector_config: S3 connector configuration
+      user: Username for RAZ authentication
+    """
+    super().__init__(connector_config, user)
 
-    # Store boto3 session with RAZ event handlers
-    self.session = boto3.Session(aws_access_key_id="dummy", aws_secret_access_key="dummy", region_name=connector_config.region)
+    # Create RAZ event handler with connector config for provider-aware URL formatting
+    self.raz_event_handler = RazEventHandler(user=user, connector_config=connector_config)
 
-    # Register RAZ handlers with the session's event system
+    # Create boto3 session and register RAZ event handlers
+    self.session = boto3.Session(region_name=connector_config.region)
+    self.session.events.register("choose-signer.*.*", self.raz_event_handler._handle_choose_signer)
     self.session.events.register("before-sign.*.*", self.raz_event_handler._handle_before_sign)
 
+    LOG.info(f"RAZ authentication provider initialized for user: {user}")
+
   def get_credentials(self) -> Dict[str, Any]:
     """
-    Return dummy credentials.
-    Real signing is done via event handlers.
+    Return empty credentials dict.
+
+    RAZ authentication doesn't use traditional AWS credentials.
+    Instead, authentication is handled through RAZ event handlers
+    using the UNSIGNED signer pattern.
+
+    Returns:
+      dict: Empty dict since credentials are handled by RAZ events
     """
-    return {"access_key_id": "dummy", "secret_access_key": "dummy"}
+    return {}
 
   def get_session_kwargs(self) -> Dict[str, Any]:
     """
     Get kwargs for creating boto3 session.
-    Returns pre-configured session with RAZ event handlers.
+
+    Returns minimal kwargs since RAZ handles authentication
+    through event handlers rather than session credentials.
+
+    Returns:
+      dict: Session kwargs with region name
     """
-    return {"aws_access_key_id": "dummy", "aws_secret_access_key": "dummy", "region_name": self.connector_config.region}
+    return {"region_name": self.connector_config.region}
+
+  def get_session(self):
+    """
+    Return the pre-configured session with RAZ event handlers.
+
+    This session has RAZ event handlers registered and should be used
+    directly rather than creating a new session, as the event handlers
+    are essential for RAZ authentication to work.
+
+    Returns:
+      Session: The boto3 session with RAZ event handlers registered
+    """
+    return self.session
 
   def refresh(self) -> None:
-    """No refresh needed as we sign per-request"""
+    """
+    Refresh credentials (no-op for RAZ).
+
+    RAZ authentication is performed per-request through event handlers,
+    so no credential refresh is needed.
+    """
     pass

+ 72 - 16
desktop/core/src/desktop/lib/fs/s3/clients/aws.py

@@ -23,7 +23,7 @@ from boto3.session import Session
 from botocore.credentials import Credentials
 from botocore.exceptions import ClientError
 
-from desktop.conf import RAZ
+from desktop.conf import default_ssl_cacerts, default_ssl_validate, RAZ
 from desktop.lib.fs.s3.clients.auth.iam import IAMAuthProvider
 from desktop.lib.fs.s3.clients.auth.idbroker import IDBrokerAuthProvider
 from desktop.lib.fs.s3.clients.auth.key import KeyAuthProvider
@@ -45,19 +45,11 @@ class AWSS3Client(S3ClientInterface):
   """
 
   def __init__(self, connector_config: "ConnectorConfig", user: str):
-    super().__init__(connector_config, user)
-
-    # AWS specific config
-    self.client_config.signature_version = "s3v4"
+    # Store options for pre-configuration
+    self._options = connector_config.options or {}
 
-    options = connector_config.options or {}
-    self.client_config.s3.update(
-      {
-        "payload_signing_enabled": True,
-        "use_accelerate_endpoint": options.get("use_accelerate", False),
-        "use_dualstack_endpoint": options.get("use_dualstack", False),
-      }
-    )
+    # Initialize with proper configuration applied BEFORE client creation
+    super().__init__(connector_config, user)
 
   def _create_auth_provider(self) -> S3AuthProvider:
     """Create appropriate auth provider based on connector config"""
@@ -73,18 +65,82 @@ class AWSS3Client(S3ClientInterface):
     else:
       return KeyAuthProvider(connector, self.user)
 
+  def _configure_client_settings(self) -> None:
+    """Configure AWS-specific client settings before client creation"""
+    # AWS specific config
+    self.client_config.signature_version = "s3v4"
+    self.client_config.s3.update(
+      {
+        "payload_signing_enabled": True,
+        "use_accelerate_endpoint": self._options.get("use_accelerate", False),
+        "use_dualstack_endpoint": self._options.get("use_dualstack", False),
+      }
+    )
+
+    # Apply SSL configuration (using global Hue SSL settings)
+    self._setup_ssl_config()
+
+  def _setup_ssl_config(self) -> None:
+    """Setup SSL verification configuration using global SSL settings with connector overrides"""
+    # Start with global Hue SSL settings
+    ssl_validate = default_ssl_validate()
+    ssl_cacerts = default_ssl_cacerts()
+
+    # Allow connector-specific overrides via options
+    if "ssl_cert_ca_verify" in self._options:
+      ssl_validate = self._options["ssl_cert_ca_verify"]
+      LOG.debug(f"Connector overrides SSL verification: {ssl_validate}")
+
+    if "ca_bundle" in self._options:
+      ssl_cacerts = self._options["ca_bundle"]
+      LOG.debug(f"Connector overrides CA bundle: {ssl_cacerts}")
+
+    # Apply SSL configuration
+    if ssl_validate:
+      if ssl_cacerts:
+        # Use custom CA bundle if specified
+        self.client_config.verify = ssl_cacerts
+        LOG.debug(f"Using CA bundle for SSL verification: {ssl_cacerts}")
+      else:
+        # Use default system CA certificates
+        self.client_config.verify = True
+        LOG.debug("Using system CA certificates for SSL verification")
+    else:
+      # Disable SSL verification
+      self.client_config.verify = False
+      LOG.warning("SSL certificate verification is DISABLED - use only for testing!")
+
+    # Client-side certificates (if needed)
+    if "client_cert" in self._options:
+      self.client_config.client_cert = self._options["client_cert"]
+
   def _create_session(self) -> Session:
     """Create boto3 session with credentials from auth provider"""
+    # For RAZ auth, use the pre-configured session with event handlers
+    if hasattr(self.auth_provider, "get_session"):
+      raz_session = self.auth_provider.get_session()
+      if raz_session is not None:
+        LOG.debug(f"Using pre-configured RAZ session with event handlers - ID: {id(raz_session)}")
+        return raz_session
+
+    # For other auth types, create new session
+    LOG.debug("Creating new session from auth provider kwargs")
     session_kwargs = self.auth_provider.get_session_kwargs()
-    return boto3.Session(**session_kwargs)
+    new_session = boto3.Session(**session_kwargs)
+    LOG.debug(f"Created new session - ID: {id(new_session)}")
+    return new_session
 
   def _create_client(self) -> Any:
     """Create boto3 S3 client"""
-    return self.session.client("s3", config=self.client_config, endpoint_url=self.connector_config.endpoint)
+    return self.session.client(
+      "s3", config=self.client_config, endpoint_url=self.connector_config.endpoint, verify=getattr(self.client_config, "verify", None)
+    )
 
   def _create_resource(self) -> Any:
     """Create boto3 S3 resource"""
-    return self.session.resource("s3", config=self.client_config, endpoint_url=self.connector_config.endpoint)
+    return self.session.resource(
+      "s3", config=self.client_config, endpoint_url=self.connector_config.endpoint, verify=getattr(self.client_config, "verify", None)
+    )
 
   def get_credentials(self) -> Optional[Credentials]:
     """Get current credentials"""

+ 8 - 1
desktop/core/src/desktop/lib/fs/s3/clients/base.py

@@ -87,7 +87,10 @@ class S3ClientInterface(ABC):
     # Initialize transfer config
     self.transfer_config = TRANSFER_CONFIG.copy()
 
-    # Initialize session and clients
+    # Allow subclasses to configure before client creation
+    self._configure_client_settings()
+
+    # Initialize session and clients (after configuration)
     self.session = self._create_session()
     self.s3_client = self._create_client()
     self.s3_resource = self._create_resource()
@@ -126,3 +129,7 @@ class S3ClientInterface(ABC):
   def get_region(self, bucket: str) -> str:
     """Get region for a bucket"""
     pass
+
+  def _configure_client_settings(self) -> None:
+    """Configure client settings before client creation. Override in subclasses."""
+    pass  # Default: no additional configuration

+ 71 - 24
desktop/core/src/desktop/lib/fs/s3/clients/generic.py

@@ -22,7 +22,7 @@ import boto3
 from boto3.session import Session
 from botocore.credentials import Credentials
 
-from desktop.conf import RAZ
+from desktop.conf import default_ssl_cacerts, default_ssl_validate, RAZ
 from desktop.lib.fs.s3.clients.auth.key import KeyAuthProvider
 from desktop.lib.fs.s3.clients.auth.raz import RazAuthProvider
 from desktop.lib.fs.s3.clients.base import S3AuthProvider, S3ClientInterface
@@ -41,9 +41,26 @@ class GenericS3Client(S3ClientInterface):
   """
 
   def __init__(self, connector_config: "ConnectorConfig", user: str):
+    # Store options for pre-configuration
+    self._options = connector_config.options or {}
+
+    # Initialize with proper configuration applied BEFORE client creation
     super().__init__(connector_config, user)
 
-    # Override client config for generic providers
+  def _create_auth_provider(self) -> S3AuthProvider:
+    """Create appropriate auth provider for generic providers"""
+    connector = self.connector_config
+
+    # Support RAZ authentication for generic providers
+    if RAZ.IS_ENABLED.get() and connector.auth_type == "raz":
+      return RazAuthProvider(connector, self.user)
+    else:
+      # Default to key auth for generic providers
+      return KeyAuthProvider(connector, self.user)
+
+  def _configure_client_settings(self) -> None:
+    """Configure Generic S3 client settings before client creation"""
+    # Generic provider config
     self.client_config.signature_version = self._get_signature_version()
     self.client_config.s3.update(
       {
@@ -55,41 +72,26 @@ class GenericS3Client(S3ClientInterface):
     # Apply provider-specific settings
     self._setup_provider_specific_config()
 
-  def _create_auth_provider(self) -> S3AuthProvider:
-    """Create appropriate auth provider for generic providers"""
-    connector = self.connector_config
-
-    # Support RAZ authentication for generic providers
-    if RAZ.IS_ENABLED.get() and connector.auth_type == "raz":
-      return RazAuthProvider(connector, self.user)
-    else:
-      # Default to key auth for generic providers
-      return KeyAuthProvider(connector, self.user)
+    # Apply SSL configuration (using global Hue SSL settings)
+    self._setup_ssl_config()
 
   def _get_signature_version(self) -> str:
     """Get signature version with option to override via configuration"""
-    options = self.connector_config.options or {}
-
     # Allow override via options for legacy systems
-    return options.get("signature_version", "s3v4")
+    return self._options.get("signature_version", "s3v4")
 
   def _setup_provider_specific_config(self) -> None:
     """Setup provider-specific configurations"""
     provider = self.connector_config.provider.lower()
-    options = self.connector_config.options or {}
 
     if provider == "netapp":
-      self._setup_netapp_config(options)
+      self._setup_netapp_config(self._options)
     elif provider == "dell":
-      self._setup_dell_config(options)
+      self._setup_dell_config(self._options)
     # Add more providers as needed
 
   def _setup_netapp_config(self, options: Dict[str, Any]) -> None:
-    """Setup Netapp StorageGRID specific config"""
-    # SSL verification
-    if "ssl_verify" in options:
-      self.client_config.verify = options["ssl_verify"]
-
+    """Setup Netapp StorageGRID specific config (SSL handled separately)"""
     # Custom headers
     if "custom_headers" in options:
       self.client_config.s3["custom_headers"] = options["custom_headers"]
@@ -106,10 +108,55 @@ class GenericS3Client(S3ClientInterface):
     if "multipart_chunksize" in options:
       self.transfer_config["multipart_chunksize"] = options["multipart_chunksize"]
 
+  def _setup_ssl_config(self) -> None:
+    """Setup SSL verification configuration using global SSL settings with connector overrides"""
+    # Start with global Hue SSL settings
+    ssl_validate = default_ssl_validate()
+    ssl_cacerts = default_ssl_cacerts()
+
+    # Allow connector-specific overrides via options
+    if "ssl_cert_ca_verify" in self._options:
+      ssl_validate = self._options["ssl_cert_ca_verify"]
+      LOG.debug(f"Connector overrides SSL verification: {ssl_validate}")
+
+    if "ca_bundle" in self._options:
+      ssl_cacerts = self._options["ca_bundle"]
+      LOG.debug(f"Connector overrides CA bundle: {ssl_cacerts}")
+
+    # Apply SSL configuration
+    if ssl_validate:
+      if ssl_cacerts:
+        # Use custom CA bundle if specified
+        self.client_config.verify = ssl_cacerts
+        LOG.debug(f"Using CA bundle for SSL verification: {ssl_cacerts}")
+      else:
+        # Use default system CA certificates
+        self.client_config.verify = True
+        LOG.debug("Using system CA certificates for SSL verification")
+    else:
+      # Disable SSL verification
+      self.client_config.verify = False
+      LOG.warning("SSL certificate verification is DISABLED - use only for testing!")
+
+    # Client-side certificates (if needed)
+    if "client_cert" in self._options:
+      self.client_config.client_cert = self._options["client_cert"]
+
   def _create_session(self) -> Session:
     """Create boto3 session"""
+    # For RAZ auth, use the pre-configured session with event handlers
+    if hasattr(self.auth_provider, "get_session"):
+      raz_session = self.auth_provider.get_session()
+      if raz_session is not None:
+        LOG.debug("Using pre-configured RAZ session with event handlers")
+        return raz_session
+
+    # For other auth types, create new session
+    LOG.debug("Creating new session from auth provider kwargs")
     session_kwargs = self.auth_provider.get_session_kwargs()
-    return boto3.Session(**session_kwargs)
+    new_session = boto3.Session(**session_kwargs)
+    LOG.debug(f"Created new session - ID: {id(new_session)}")
+    return new_session
 
   def _create_client(self) -> Any:
     """Create boto3 S3 client"""

+ 4 - 4
desktop/core/src/desktop/lib/fs/s3/conf_utils.py

@@ -251,7 +251,7 @@ class BucketConfig:
   region: Optional[str] = None
   options: Optional[Dict[str, any]] = None
 
-  def get_effective_home_path(self, user: str = None) -> str:
+  def get_effective_home_path(self, user: str = None, auth_type: str = None) -> str:
     """
     Get effective home path for this bucket, handling relative paths and user context.
 
@@ -274,7 +274,7 @@ class BucketConfig:
         path += "/"
 
     # Handle RAZ user directory logic
-    if user and RAZ.IS_ENABLED.get():
+    if user and RAZ.IS_ENABLED.get() and auth_type == "raz":
       from desktop.models import _handle_user_dir_raz
 
       path = _handle_user_dir_raz(user, path)
@@ -490,7 +490,7 @@ def get_s3_home_directory(user: Optional[str] = None, connector_id: str = None,
 
     if bucket_name:
       bucket_config = connector.get_bucket_config(bucket_name)
-      return bucket_config.get_effective_home_path(user)
+      return bucket_config.get_effective_home_path(user, connector.auth_type)
 
     # No bucket available, return generic path
     return "s3a://"
@@ -609,7 +609,7 @@ def get_default_bucket_home_path(connector_id: str = None, user: str = None) ->
       # Single bucket - use its home path as default
       bucket_name = bucket_names[0]
       bucket_config = connector.get_bucket_config(bucket_name)
-      home_path = bucket_config.get_effective_home_path(user)
+      home_path = bucket_config.get_effective_home_path(user, connector.auth_type)
       LOG.debug(f"Using single bucket '{bucket_name}' home path: {home_path}")
       return home_path
 

+ 6 - 7
desktop/core/src/desktop/lib/raz/clients.py

@@ -19,7 +19,6 @@ import logging
 from desktop.conf import RAZ
 from desktop.lib.raz.raz_client import get_raz_client
 
-
 LOG = logging.getLogger()
 
 
@@ -28,7 +27,7 @@ class S3RazClient():
   def __init__(self, username):
     self.username = username
 
-  def get_url(self, action='GET', path=None, headers=None, data=None):
+  def get_url(self, action='GET', path=None, params=None, headers=None, data=None):
     '''
     Example of headers:
     {
@@ -36,8 +35,8 @@ class S3RazClient():
       u'Host': u'hue-testing.s3-us-west-2.amazonaws.com',
       u'X-Amz-Security-Token': u'IQoJb3JpZ2luX2Vj...C',
       u'X-Amz-Date': u'20210604T102022Z',
-      u'Authorization': u'AWS4-HMAC-SHA256 Credential=ASIAYO3P24NAOAYMMDNN/20210604/us-west-2/s3/aws4_request, 
-                          SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token, 
+      u'Authorization': u'AWS4-HMAC-SHA256 Credential=ASIAYO3P24NAOAYMMDNN/20210604/us-west-2/s3/aws4_request,
+                          SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token,
                           Signature=d341a194c2998c64b6fc726b69d0c3c2b97d520265f80df7e1bc1ac59a21ef94',
       u'User-Agent': u'user:csso_gethue_user'
     }
@@ -49,7 +48,7 @@ class S3RazClient():
       service='s3',
     )
 
-    return c.check_access(method=action, url=path, headers=headers, data=data)
+    return c.check_access(method=action, url=path, params=params, headers=headers, data=data)
 
 
 class GSRazClient():
@@ -65,8 +64,8 @@ class GSRazClient():
       u'Host': u'hue-testing.s3-us-west-2.amazonaws.com',
       u'X-Amz-Security-Token': u'IQoJb3JpZ2luX2Vj...C',
       u'X-Amz-Date': u'20210604T102022Z',
-      u'Authorization': u'AWS4-HMAC-SHA256 Credential=ASIAYO3P24NAOAYMMDNN/20210604/us-west-2/s3/aws4_request, 
-                          SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token, 
+      u'Authorization': u'AWS4-HMAC-SHA256 Credential=ASIAYO3P24NAOAYMMDNN/20210604/us-west-2/s3/aws4_request,
+                          SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-security-token,
                           Signature=d341a194c2998c64b6fc726b69d0c3c2b97d520265f80df7e1bc1ac59a21ef94',
       u'User-Agent': u'user:csso_gethue_user'
     }