소스 검색

HUE-4668 [fb] S3 rename directory raises IOError

Fixes copy and rename functions and always raises renderable error now
Jenny Kim 9 년 전
부모
커밋
0f52f8a10b
2개의 변경된 파일16개의 추가작업 그리고 13개의 파일을 삭제
  1. 12 9
      desktop/libs/aws/src/aws/s3/s3fs.py
  2. 4 4
      desktop/libs/aws/src/aws/s3/s3fs_test.py

+ 12 - 9
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -16,7 +16,6 @@
 
 from __future__ import absolute_import
 
-import errno
 import itertools
 import logging
 import os
@@ -173,7 +172,7 @@ class S3FileSystem(object):
   def open(self, path, mode='r'):
     key = self._get_key(path, validate=True)
     if key is None:
-      raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
+      raise S3FileSystemException("No such file or directory: '%s'" % path)
     return s3file.open(key, mode=mode)
 
   @translate_s3_error
@@ -206,7 +205,7 @@ class S3FileSystem(object):
     stats = self._stats(path)
     if stats:
       return stats
-    raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
+    raise S3FileSystemException("No such file or directory: '%s'" % path)
 
   @translate_s3_error
   def listdir_stats(self, path, glob=None):
@@ -286,7 +285,7 @@ class S3FileSystem(object):
       if stats.isDir:
         return None
       else:
-        raise IOError(errno.ENOTDIR, "'%s' already exists and is not a directory" % path)
+        raise S3FileSystemException("'%s' already exists and is not a directory" % path)
     path = self._append_separator(path)  # folder-key should ends by /
     self.create(path)  # create empty object
 
@@ -297,7 +296,7 @@ class S3FileSystem(object):
   @translate_s3_error
   def copyfile(self, src, dst, *args, **kwargs):
     if self.isdir(dst):
-      raise IOError(errno.EINVAL, "Copy dst '%s' is a directory" % dst)
+      raise S3FileSystemException("Copy dst '%s' is a directory" % dst)
     self._copy(src, dst, recursive=False, use_src_basename=False)
 
   @translate_s3_error
@@ -312,7 +311,7 @@ class S3FileSystem(object):
     dst = s3.abspath(src, dst)
     dst_st = self._stats(dst)
     if src_st.isDir and dst_st and not dst_st.isDir:
-      raise IOError(errno.EEXIST, "Cannot overwrite non-directory '%s' with directory '%s'" % (dst, src))
+      raise S3FileSystemException("Cannot overwrite non-directory '%s' with directory '%s'" % (dst, src))
 
     src_bucket, src_key = s3.parse_uri(src)[:2]
     dst_bucket, dst_key = s3.parse_uri(dst)[:2]
@@ -332,8 +331,12 @@ class S3FileSystem(object):
 
     for key in src_bucket.list(prefix=src_key):
       if not key.name.startswith(src_key):
-        raise RuntimeError(_("Invalid key to transform: %s") % key.name)
+        raise S3FileSystemException(_("Invalid key to transform: %s") % key.name)
       dst_name = posixpath.normpath(s3.join(dst_key, key.name[cut:]))
+
+      if self.isdir(normpath(self.join(S3A_ROOT, key.bucket.name, key.name))):
+        dst_name = self._append_separator(dst_name)
+
       key.copy(dst_bucket, dst_name)
 
   @translate_s3_error
@@ -347,9 +350,9 @@ class S3FileSystem(object):
   @translate_s3_error
   def rename_star(self, old_dir, new_dir):
     if not self.isdir(old_dir):
-      raise IOError(errno.ENOTDIR, "'%s' is not a directory" % old_dir)
+      raise S3FileSystemException("'%s' is not a directory" % old_dir)
     if self.isfile(new_dir):
-      raise IOError(errno.ENOTDIR, "'%s' is not a directory" % new_dir)
+      raise S3FileSystemException("'%s' is not a directory" % new_dir)
     ls = self.listdir(old_dir)
     for entry in ls:
       self.rename(s3.join(old_dir, entry), s3.join(new_dir, entry))

+ 4 - 4
desktop/libs/aws/src/aws/s3/s3fs_test.py

@@ -26,7 +26,7 @@ from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group
 
 from aws.s3 import join, parse_uri
-from aws.s3.s3fs import S3FileSystem
+from aws.s3.s3fs import S3FileSystem, S3FileSystemException
 from aws.s3.s3test_utils import S3TestBase, generate_id
 from aws.s3.upload import DEFAULT_WRITE_SIZE
 
@@ -48,7 +48,7 @@ class S3FSTest(S3TestBase):
     path = self.get_test_path('test_open.txt')
 
     with self.cleaning(path):
-      assert_raises(IOError, self.fs.open, path)
+      assert_raises(S3FileSystemException, self.fs.open, path)
 
       key = self.get_key(path)
       key.set_contents_from_string('Hello')
@@ -104,7 +104,7 @@ class S3FSTest(S3TestBase):
   def test_stats(self):
     assert_raises(ValueError, self.fs.stats, 'ftp://archive')
     not_exists = self.get_test_path('does_not_exist')
-    assert_raises(IOError, self.fs.stats, not_exists)
+    assert_raises(S3FileSystemException, self.fs.stats, not_exists)
 
     root_stat = self.fs.stats('s3a://')
     eq_(True, root_stat.isDir)
@@ -154,7 +154,7 @@ class S3FSTest(S3TestBase):
       assert_true(self.fs.exists(join(dst_folder_path, 'file.txt')))
 
       # Copy directory to file should fail.
-      assert_raises(IOError, self.fs.copy, src_path, dst_file_path, True)
+      assert_raises(S3FileSystemException, self.fs.copy, src_path, dst_file_path, True)
 
 
   def test_copy_remote_dir(self):