Ver código fonte

HUE-4136 [fb] Fix normpath and parent path logic for aws fs

Jenny Kim 9 anos atrás
pai
commit
3b33dd9

+ 2 - 2
apps/filebrowser/src/filebrowser/views.py

@@ -331,7 +331,7 @@ def listdir(request, path, chooser):
 
     # Include parent dir, unless at filesystem root.
     if not request.fs.isroot(path):
-        parent_path = request.fs.join(path, "..")
+        parent_path = request.fs.parent_path(path)
         parent_stat = request.fs.stats(parent_path)
         # The 'path' field would be absolute, but we want its basename to be
         # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
@@ -415,7 +415,7 @@ def listdir_paged(request, path):
 
     # Include parent dir always as second option, unless at filesystem root.
     if not request.fs.isroot(path):
-        parent_path = request.fs.join(path, "..")
+        parent_path = request.fs.parent_path(path)
         parent_stat = request.fs.stats(parent_path)
         # The 'path' field would be absolute, but we want its basename to be
         # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.

+ 3 - 0
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -119,6 +119,9 @@ class ProxyFS(object):
   def isroot(self, path):
     return self._get_fs(path).isroot(path)
 
+  def parent_path(self, path):
+    return self._get_fs(path).parent_path(path)
+
   def join(self, first, *comp_list):
     return self._get_fs(first).join(first, *comp_list)
 

+ 1 - 1
desktop/core/src/desktop/lib/fsmanager.py

@@ -31,7 +31,7 @@ DEFAULT_SCHEMA = 'hdfs'
 
 FS_GETTERS = {
   "hdfs": cluster.get_hdfs,
-#   "s3": aws.get_s3fs
+  "s3": aws.get_s3fs
 }
 
 

+ 19 - 3
desktop/libs/aws/src/aws/s3/__init__.py

@@ -26,7 +26,7 @@ import time
 from functools import wraps
 
 from boto.exception import S3ResponseError
-from hadoop.fs import normpath
+from hadoop.fs import normpath as fs_normpath
 
 
 ERRNO_MAP = {
@@ -84,8 +84,10 @@ def abspath(cd, uri):
   abspath('s3://bucket/key', key2') == 's3://bucket/key/key2'
   abspath('s3://bucket/key', 's3://bucket2/key2') == 's3://bucket2/key2'
   """
-  if not uri.lower().startswith(S3_ROOT):
-    uri = normpath(join(cd, '..', uri))
+  if cd.lower().startswith(S3_ROOT):
+    uri = join(cd, uri)
+  else:
+    uri = normpath(join(cd, uri))
   return uri
 
 
@@ -101,6 +103,20 @@ def join(*comp_list):
   return joined
 
 
+def normpath(path):
+  """
+  Return normalized path but ignore leading S3_ROOT prefix if it exists
+  """
+  if path.lower().startswith(S3_ROOT):
+    if is_root(path):
+      normalized = path
+    else:
+      normalized = '%s%s' % (S3_ROOT, fs_normpath(path[len(S3_ROOT):]))
+  else:
+    normalized = fs_normpath(path)
+  return normalized
+
+
 def s3datetime_to_timestamp(datetime):
   """
   Returns timestamp (seconds) by datetime string from S3 API responses.

+ 2 - 4
desktop/libs/aws/src/aws/s3/s3_test.py

@@ -45,11 +45,9 @@ def test_join():
 
 
 def test_abspath():
-  raise SkipTest()
-
   a = s3.abspath
-  eq_('s3://a/b/d', a('s3://a/b/c', 'd'))
-  eq_('s3://d', a('s3://a/b/c', 's3://d'))
+  eq_('s3://a/b/c/d', a('s3://a/b/c', 'd'))
+  eq_('s3://a/b/c/d', a('/a/b/c', 'd'))
 
 
 def test_is_root():

+ 14 - 3
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -29,11 +29,9 @@ from boto.s3.key import Key
 from boto.s3.prefix import Prefix
 
 from aws import s3
-from aws.s3 import translate_s3_error, s3file
+from aws.s3 import normpath, s3file, translate_s3_error, S3_ROOT
 from aws.s3.s3stat import S3Stat
 
-from hadoop.fs import normpath
-
 
 DEFAULT_READ_SIZE = 1024 * 1024  # 1MB
 LOG = logging.getLogger(__name__)
@@ -117,6 +115,19 @@ class S3FileSystem(object):
   def normpath(path):
     return normpath(path)
 
+  @staticmethod
+  def parent_path(path):
+    parent_dir = S3FileSystem._append_separator(path)
+    if not s3.is_root(parent_dir):
+      bucket_name, key_name, basename = s3.parse_uri(path)
+      if not basename:  # bucket is top-level so return root
+        parent_dir = S3_ROOT
+      else:
+        bucket_path = '%s%s' % (S3_ROOT, bucket_name)
+        key_path = '/'.join(key_name.split('/')[:-1])
+        parent_dir = s3.abspath(bucket_path, key_path)
+    return parent_dir
+
   @translate_s3_error
   def open(self, path, mode='r'):
     key = self._get_key(path, validate=True)

+ 0 - 2
desktop/libs/aws/src/aws/s3/s3test_utils.py

@@ -41,8 +41,6 @@ def generate_id(size=6, chars=string.ascii_uppercase + string.digits):
 class S3TestBase(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    raise SkipTest()
-
     cls.bucket_name = get_test_bucket()
 
     cls._should_skip = False

+ 4 - 0
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -198,6 +198,10 @@ class Hdfs(object):
       return res[1:]
     return res
 
+  @staticmethod
+  def parent_path(path):
+    return Hdfs.join(path, "..")
+
   @staticmethod
   def urlsplit(url):
     """