Bladeren bron

HUE-6327 [aws] Fix filebrowser test that inspects scheme, catching any permissions errors

Jenny Kim 8 jaren geleden
bovenliggende
commit
578c3f720c

+ 13 - 16
apps/filebrowser/src/filebrowser/views.py

@@ -32,7 +32,6 @@ from datetime import datetime
 from cStringIO import StringIO
 from cStringIO import StringIO
 from gzip import GzipFile
 from gzip import GzipFile
 
 
-from django.contrib import messages
 from django.contrib.auth.models import User, Group
 from django.contrib.auth.models import User, Group
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
 from django.template.defaultfilters import stringformat, filesizeformat
 from django.template.defaultfilters import stringformat, filesizeformat
@@ -197,18 +196,25 @@ def view(request, path):
             return listdir_paged(request, path)
             return listdir_paged(request, path)
         else:
         else:
             return display(request, path)
             return display(request, path)
+    except S3FileSystemException, e:
+        msg = _("S3 filesystem exception.")
+        if request.is_ajax():
+            exception = {
+                'error': smart_str(e)
+            }
+            return JsonResponse(exception)
+        else:
+            raise PopupException(msg, detail=e)
     except (IOError, WebHdfsException), e:
     except (IOError, WebHdfsException), e:
         msg = _("Cannot access: %(path)s. ") % {'path': escape(path)}
         msg = _("Cannot access: %(path)s. ") % {'path': escape(path)}
 
 
         if "Connection refused" in e.message:
         if "Connection refused" in e.message:
             msg += _(" The HDFS REST service is not available. ")
             msg += _(" The HDFS REST service is not available. ")
 
 
-        if request.fs._get_scheme(path) == 'hdfs':
-          if request.user.is_superuser and not _is_hdfs_superuser(request):
-            msg += _(' Note: you are a Hue admin but not a HDFS superuser, "%(superuser)s" or part of HDFS supergroup, "%(supergroup)s".') \
-                % {'superuser': request.fs.superuser, 'supergroup': request.fs.supergroup}
-        elif request.fs._get_scheme(path) == 's3a':
-            msg += _(' S3 filesystem exception.')
+        if request.fs._get_scheme(path).lower() == 'hdfs':
+            if request.user.is_superuser and not _is_hdfs_superuser(request):
+                msg += _(' Note: you are a Hue admin but not a HDFS superuser, "%(superuser)s" or part of HDFS supergroup, "%(supergroup)s".') \
+                        % {'superuser': request.fs.superuser, 'supergroup': request.fs.supergroup}
 
 
         if request.is_ajax():
         if request.is_ajax():
           exception = {
           exception = {
@@ -217,15 +223,6 @@ def view(request, path):
           return JsonResponse(exception)
           return JsonResponse(exception)
         else:
         else:
           raise PopupException(msg , detail=e)
           raise PopupException(msg , detail=e)
-    except S3FileSystemException, e:
-        msg = _("S3 filesystem exception.")
-        if request.is_ajax():
-            exception = {
-                'error': smart_str(e)
-            }
-            return JsonResponse(exception)
-        else:
-            raise PopupException(msg, detail=e)
 
 
 
 
 def home_relative_view(request, path):
 def home_relative_view(request, path):

+ 5 - 4
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -25,6 +25,7 @@ from django.contrib.auth.models import User
 
 
 from aws.conf import has_s3_access
 from aws.conf import has_s3_access
 from aws.s3 import S3A_ROOT
 from aws.s3 import S3A_ROOT
+from aws.s3.s3fs import S3FileSystemException
 
 
 
 
 class ProxyFS(object):
 class ProxyFS(object):
@@ -57,9 +58,9 @@ class ProxyFS(object):
       try:
       try:
         user = User.objects.get(username=self.user)
         user = User.objects.get(username=self.user)
         if not has_s3_access(rewrite_user(user)):
         if not has_s3_access(rewrite_user(user)):
-          raise IOError(errno.EPERM, "Missing permissions for %s on %s" % (self.user, path,))
+          raise S3FileSystemException("Missing permissions for %s on %s" % (self.user, path,))
       except User.DoesNotExist:
       except User.DoesNotExist:
-        raise IOError(errno.EPERM, "Can't check permissions for %s on %s" % (self.user, path))
+        raise S3FileSystemException("Can't check permissions for %s on %s" % (self.user, path))
 
 
     split = urlparse(path)
     split = urlparse(path)
     if split.scheme:
     if split.scheme:
@@ -70,11 +71,11 @@ class ProxyFS(object):
   def _get_fs(self, path):
   def _get_fs(self, path):
     scheme = self._get_scheme(path)
     scheme = self._get_scheme(path)
     if not scheme:
     if not scheme:
-      raise IOError(errno.EINVAL, 'Can not figure out scheme for path "%s"' % path)
+      raise S3FileSystemException('Can not figure out scheme for path "%s"' % path)
     try:
     try:
       return self._fs_dict[scheme]
       return self._fs_dict[scheme]
     except KeyError:
     except KeyError:
-      raise IOError(errno.EINVAL, 'Unknown scheme %s, available schemes: %s' % (scheme, self._fs_dict.keys()))
+      raise S3FileSystemException('Unknown scheme %s, available schemes: %s' % (scheme, self._fs_dict.keys()))
 
 
   def _get_fs_pair(self, src, dst):
   def _get_fs_pair(self, src, dst):
     """
     """

+ 5 - 3
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -43,8 +43,10 @@ BUCKET_NAME_PATTERN = re.compile("^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_\-]*
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
-class S3FileSystemException(Exception):
-  pass
+class S3FileSystemException(IOError):
+
+  def __init__(self, *args, **kwargs):
+    super(S3FileSystemException, self).__init__(*args, **kwargs)
 
 
 
 
 def auth_error_handler(view_fn):
 def auth_error_handler(view_fn):
@@ -52,7 +54,7 @@ def auth_error_handler(view_fn):
     try:
     try:
       return view_fn(*args, **kwargs)
       return view_fn(*args, **kwargs)
     except (S3ResponseError, IOError), e:
     except (S3ResponseError, IOError), e:
-      if 'Forbidden' in str(e) or e.status == 403:
+      if 'Forbidden' in str(e) or (hasattr(e, 'status') and e.status == 403):
         path = kwargs.get('path')
         path = kwargs.get('path')
         if not path and len(args) > 1:
         if not path and len(args) > 1:
           path = args[1]  # We assume that the path is the first argument
           path = args[1]  # We assume that the path is the first argument

+ 4 - 3
desktop/libs/aws/src/aws/s3/upload.py

@@ -30,6 +30,7 @@ from django.utils.translation import ugettext as _
 
 
 from aws import get_s3fs
 from aws import get_s3fs
 from aws.s3 import parse_uri
 from aws.s3 import parse_uri
+from aws.s3.s3fs import S3FileSystemException
 
 
 
 
 DEFAULT_WRITE_SIZE = 1024 * 1024 * 50  # TODO: set in configuration (currently 50 MiB)
 DEFAULT_WRITE_SIZE = 1024 * 1024 * 50  # TODO: set in configuration (currently 50 MiB)
@@ -80,7 +81,7 @@ class S3FileUploadHandler(FileUploadHandler):
         self._mp = self._bucket.initiate_multipart_upload(self.target_path)
         self._mp = self._bucket.initiate_multipart_upload(self.target_path)
         self.file = SimpleUploadedFile(name=file_name, content='')
         self.file = SimpleUploadedFile(name=file_name, content='')
         raise StopFutureHandlers()
         raise StopFutureHandlers()
-      except S3FileUploadError, e:
+      except (S3FileUploadError, S3FileSystemException), e:
         LOG.error("Encountered error in S3UploadHandler check_access: %s" % e)
         LOG.error("Encountered error in S3UploadHandler check_access: %s" % e)
         self.request.META['upload_failed'] = e
         self.request.META['upload_failed'] = e
         raise StopUpload()
         raise StopUpload()
@@ -131,7 +132,7 @@ class S3FileUploadHandler(FileUploadHandler):
 
 
   def _check_access(self):
   def _check_access(self):
     if not self._fs.check_access(self.destination, permission='WRITE'):
     if not self._fs.check_access(self.destination, permission='WRITE'):
-      raise S3FileUploadError(_('Insufficient permissions to write to S3 path "%s".') % self.destination)
+      raise S3FileSystemException('Insufficient permissions to write to S3 path "%s".' % self.destination)
 
 
 
 
   def _get_scheme(self):
   def _get_scheme(self):
@@ -140,7 +141,7 @@ class S3FileUploadHandler(FileUploadHandler):
       if dst_parts > 0:
       if dst_parts > 0:
         return dst_parts[0].upper()
         return dst_parts[0].upper()
       else:
       else:
-        raise IOError('Destination does not start with a valid scheme.')
+        raise S3FileSystemException('Destination does not start with a valid scheme.')
     else:
     else:
       return None
       return None