Эх сурвалжийг харах

HUE-6327 [aws] Support new regions and provide reasonable defaults for proxy configs

Jenny Kim 8 жил өмнө
parent
commit
af50391

+ 7 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -199,11 +199,17 @@ def view(request, path):
             return display(request, path)
     except (IOError, WebHdfsException), e:
         msg = _("Cannot access: %(path)s. ") % {'path': escape(path)}
+
         if "Connection refused" in e.message:
             msg += _(" The HDFS REST service is not available. ")
-        if request.user.is_superuser and not _is_hdfs_superuser(request):
+
+        if request.fs._get_scheme(path) == 'hdfs':
+          if request.user.is_superuser and not _is_hdfs_superuser(request):
             msg += _(' Note: you are a Hue admin but not a HDFS superuser, "%(superuser)s" or part of HDFS supergroup, "%(supergroup)s".') \
                 % {'superuser': request.fs.superuser, 'supergroup': request.fs.supergroup}
+        elif request.fs._get_scheme(path) == 's3a':
+            msg += _(' S3 filesystem exception.')
+
         if request.is_ajax():
           exception = {
             'error': msg

+ 7 - 2
desktop/conf.dist/hue.ini

@@ -1371,8 +1371,13 @@
       ## region=us-east-1
 
       # Endpoint overrides
+      ## host=s3.amazonaws.com
+
+      # Proxy address and port
       ## proxy_address=
-      ## proxy_port=
+      ## proxy_port=8080
+      ## proxy_user=
+      ## proxy_pass=
 
       # Secure connections are the default, but this can be explicitly overridden:
       ## is_secure=true
@@ -1380,7 +1385,7 @@
       # The default calling format uses https://<bucket-name>.s3.amazonaws.com but
       # this may not make sense if DNS is not configured in this way for custom endpoints.
       # e.g. Use boto.s3.connection.OrdinaryCallingFormat for https://s3.amazonaws.com/<bucket-name>
-      ## calling_format=boto.s3.connection.S3Connection.DefaultCallingFormat
+      ## calling_format=boto.s3.connection.OrdinaryCallingFormat
 
 
 ###########################################################################

+ 7 - 2
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1372,9 +1372,14 @@
       # AWS region to use
       ## region=us-east-1
 
+      # Endpoint overrides
+      ## host=s3.amazonaws.com
+
       # Endpoint overrides
       ## proxy_address=
-      ## proxy_port=
+      ## proxy_port=8080
+      ## proxy_user=
+      ## proxy_pass=
 
       # Secure connections are the default, but this can be explicitly overridden:
       ## is_secure=true
@@ -1382,7 +1387,7 @@
       # The default calling format uses https://<bucket-name>.s3.amazonaws.com but
       # this may not make sense if DNS is not configured in this way for custom endpoints.
       # e.g. Use boto.s3.connection.OrdinaryCallingFormat for https://s3.amazonaws.com/<bucket-name>
-      ## calling_format=boto.s3.connection.S3Connection.DefaultCallingFormat
+      ## calling_format=boto.s3.connection.OrdinaryCallingFormat
 
 
 ###########################################################################

+ 49 - 21
desktop/libs/aws/src/aws/client.py

@@ -15,12 +15,19 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+import logging
+import os
+
 import boto
 import boto.s3
 import boto.s3.connection
 import boto.utils
 
 from aws.conf import get_default_region, has_iam_metadata, DEFAULT_CALLING_FORMAT
+from aws.s3.s3fs import S3FileSystemException
+
+
+LOG = logging.getLogger(__name__)
 
 
 HTTP_SOCKET_TIMEOUT_S = 60
@@ -28,15 +35,18 @@ HTTP_SOCKET_TIMEOUT_S = 60
 
 class Client(object):
   def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
-               timeout=HTTP_SOCKET_TIMEOUT_S, proxy_address=None,
-               proxy_port=None, calling_format=None, is_secure=True):
+               timeout=HTTP_SOCKET_TIMEOUT_S, host=None, proxy_address=None, proxy_port=None, proxy_user=None,
+               proxy_pass=None, calling_format=None, is_secure=True):
     self._access_key_id = aws_access_key_id
     self._secret_access_key = aws_secret_access_key
     self._security_token = aws_security_token
     self._region = region.lower() if region else get_default_region()
     self._timeout = timeout
+    self._host = host
     self._proxy_address = proxy_address
     self._proxy_port = proxy_port
+    self._proxy_user = proxy_user
+    self._proxy_pass = proxy_pass
     self._calling_format = DEFAULT_CALLING_FORMAT if calling_format is None else calling_format
     self._is_secure = is_secure
 
@@ -61,37 +71,55 @@ class Client(object):
       aws_secret_access_key=secret_access_key,
       aws_security_token=security_token,
       region=conf.REGION.get(),
+      host=conf.HOST.get(),
       proxy_address=conf.PROXY_ADDRESS.get(),
       proxy_port=conf.PROXY_PORT.get(),
+      proxy_user=conf.PROXY_USER.get(),
+      proxy_pass=conf.PROXY_PASS.get(),
       calling_format=conf.CALLING_FORMAT.get(),
       is_secure=conf.IS_SECURE.get()
     )
 
   def get_s3_connection(self):
-    # First attempt to connect via specified credentials
-    if self._proxy_address is not None and self._proxy_port is not None:
-      connection = boto.s3.connection.S3Connection(aws_access_key_id=self._access_key_id,
-        aws_secret_access_key=self._secret_access_key,
-        security_token=self._security_token,
-        is_secure=self._is_secure,
-        calling_format=self._calling_format,
-        proxy=self._proxy_address,
-        proxy_port=self._proxy_port)
-    elif self._region:
-      connection = boto.s3.connect_to_region(self._region,
-        aws_access_key_id=self._access_key_id,
-        aws_secret_access_key=self._secret_access_key,
-        security_token=self._security_token)
-    else:
-      connection = boto.s3.connection.S3Connection(aws_access_key_id=self._access_key_id,
-        aws_secret_access_key=self._secret_access_key,
-        security_token=self._security_token)
+    # Use V4 signature support by default
+    os.environ['S3_USE_SIGV4'] = 'True'
+
+    kwargs = {
+      'aws_access_key_id': self._access_key_id,
+      'aws_secret_access_key': self._secret_access_key,
+      'security_token': self._security_token,
+      'is_secure': self._is_secure,
+      'calling_format': self._calling_format
+    }
+
+    # Add proxy if configured
+    if self._proxy_address is not None:
+      kwargs.update({'proxy': self._proxy_address})
+      if self._proxy_port is not None:
+        kwargs.update({'proxy_port': self._proxy_port})
+      if self._proxy_user is not None:
+        kwargs.update({'proxy_user': self._proxy_user})
+      if self._proxy_pass is not None:
+        kwargs.update({'proxy_pass': self._proxy_pass})
+
+    # Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM
+    try:
+      if self._host is not None:
+        kwargs.update({'host': self._host})
+        connection = boto.s3.connection.S3Connection(**kwargs)
+      elif self._region:
+        connection = boto.s3.connect_to_region(self._region, **kwargs)
+      else:
+        connection = boto.s3.connection.S3Connection(**kwargs)
+    except Exception, e:
+      LOG.exception(e)
+      raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
 
     if connection is None:
       # If no connection, attemt to fallback to IAM instance metadata
       connection = boto.connect_s3()
 
       if connection is None:
-        raise ValueError('Can not construct S3 Connection for region %s' % self._region)
+        raise S3FileSystemException('Can not construct S3 Connection for region %s' % self._region)
 
     return connection

+ 22 - 4
desktop/libs/aws/src/aws/conf.py

@@ -25,7 +25,7 @@ from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, co
 from hadoop.core_site import get_s3a_access_key, get_s3a_secret_key
 
 
-DEFAULT_CALLING_FORMAT='boto.s3.connection.S3Connection.DefaultCallingFormat'
+DEFAULT_CALLING_FORMAT='boto.s3.connection.OrdinaryCallingFormat'
 
 
 def get_default_access_key_id():
@@ -95,18 +95,36 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
         default='us-east-1',
         type=str
       ),
+      HOST=Config(
+        help=_('Alternate address for the S3 endpoint.'),
+        key='host',
+        default=None,
+        type=str
+      ),
       PROXY_ADDRESS=Config(
-        help=_('Alternate address for endpoint.'),
+        help=_('Proxy address to use for the S3 connection.'),
         key='proxy_address',
         default=None,
         type=str
       ),
       PROXY_PORT=Config(
-        help=_('Alternate port for endpoint.'),
+        help=_('Proxy port to use for the S3 connection.'),
         key='proxy_port',
-        default=None,
+        default=8080,
         type=int
       ),
+      PROXY_USER=Config(
+        help=_('Proxy user to use for the S3 connection.'),
+        key='proxy_user',
+        default=None,
+        type=str
+      ),
+      PROXY_PASS=Config(
+        help=_('Proxy password to use for the S3 connection.'),
+        key='proxy_pass',
+        default=None,
+        type=str
+      ),
       CALLING_FORMAT=Config(
         key='calling_format',
         default=DEFAULT_CALLING_FORMAT,