浏览代码

HUE-9152 [filebrowser] Fix operations on directories with Cyrillic names

Ying Chen 5 年之前
父节点
当前提交
db807aa2e7

+ 19 - 8
apps/filebrowser/src/filebrowser/views.py

@@ -191,7 +191,7 @@ def download(request, path):
 
 
 def view(request, path):
 def view(request, path):
     """Dispatches viewing of a path to either index() or fileview(), depending on type."""
     """Dispatches viewing of a path to either index() or fileview(), depending on type."""
-    decoded_path = urllib_unquote(path)
+    decoded_path = unquote_url(path)
     if path != decoded_path:
     if path != decoded_path:
       path = decoded_path
       path = decoded_path
     # default_to_home is set in bootstrap.js
     # default_to_home is set in bootstrap.js
@@ -256,7 +256,7 @@ def home_relative_view(request, path):
 
 
 def edit(request, path, form=None):
 def edit(request, path, form=None):
     """Shows an edit form for the given path. Path does not necessarily have to exist."""
     """Shows an edit form for the given path. Path does not necessarily have to exist."""
-    decoded_path = urllib_unquote(path)
+    decoded_path = unquote_url(path)
     if path != decoded_path:
     if path != decoded_path:
       path = decoded_path
       path = decoded_path
     try:
     try:
@@ -314,7 +314,7 @@ def save_file(request):
     form = EditorForm(request.POST)
     form = EditorForm(request.POST)
     is_valid = form.is_valid()
     is_valid = form.is_valid()
     path = form.cleaned_data.get('path')
     path = form.cleaned_data.get('path')
-    decoded_path = urllib_unquote(path)
+    decoded_path = unquote_url(path)
     if path != decoded_path:
     if path != decoded_path:
       path = decoded_path
       path = decoded_path
 
 
@@ -1175,7 +1175,12 @@ def touch(request):
         # No absolute path specification allowed.
         # No absolute path specification allowed.
         if posixpath.sep in name:
         if posixpath.sep in name:
             raise PopupException(_("Could not name file \"%s\": Slashes are not allowed in filenames." % name))
             raise PopupException(_("Could not name file \"%s\": Slashes are not allowed in filenames." % name))
-        request.fs.create(request.fs.join(urllib_unquote(path), urllib_unquote(name)))
+        request.fs.create(
+            request.fs.join(
+                urllib_unquote(path.encode('utf-8') if not isinstance(path, str) else path),
+                urllib_unquote(name.encode('utf-8') if not isinstance(name, str) else name)
+            )
+        )
 
 
     return generic_op(TouchForm, request, smart_touch, ["path", "name"], "path")
     return generic_op(TouchForm, request, smart_touch, ["path", "name"], "path")
 
 
@@ -1200,7 +1205,10 @@ def move(request):
         for arg in args:
         for arg in args:
             if arg['src_path'] == arg['dest_path']:
             if arg['src_path'] == arg['dest_path']:
                 raise PopupException(_('Source path and destination path cannot be same'))
                 raise PopupException(_('Source path and destination path cannot be same'))
-            request.fs.rename(urllib_unquote(arg['src_path']), urllib_unquote(arg['dest_path']))
+            request.fs.rename(
+                urllib_unquote(arg['src_path'].encode('utf-8') if not isinstance(arg['src_path'], str) else arg['src_path']),
+                urllib_unquote(arg['dest_path'].encode('utf-8') if not isinstance(arg['dest_path'], str) else arg['dest_path'])
+            )
     return generic_op(RenameFormSet, request, bulk_move, ["src_path", "dest_path"], None,
     return generic_op(RenameFormSet, request, bulk_move, ["src_path", "dest_path"], None,
                       data_extractor=formset_data_extractor(recurring, params),
                       data_extractor=formset_data_extractor(recurring, params),
                       arg_extractor=formset_arg_extractor,
                       arg_extractor=formset_arg_extractor,
@@ -1215,7 +1223,7 @@ def copy(request):
         for arg in args:
         for arg in args:
             if arg['src_path'] == arg['dest_path']:
             if arg['src_path'] == arg['dest_path']:
                 raise PopupException(_('Source path and destination path cannot be same'))
                 raise PopupException(_('Source path and destination path cannot be same'))
-            request.fs.copy(urllib_unquote(arg['src_path']), urllib_unquote(arg['dest_path']), recursive=True, owner=request.user)
+            request.fs.copy(unquote_url(arg['src_path']), unquote_url(arg['dest_path']), recursive=True, owner=request.user)
     return generic_op(CopyFormSet, request, bulk_copy, ["src_path", "dest_path"], None,
     return generic_op(CopyFormSet, request, bulk_copy, ["src_path", "dest_path"], None,
                       data_extractor=formset_data_extractor(recurring, params),
                       data_extractor=formset_data_extractor(recurring, params),
                       arg_extractor=formset_arg_extractor,
                       arg_extractor=formset_arg_extractor,
@@ -1319,8 +1327,8 @@ def _upload_file(request):
 
 
     if form.is_valid():
     if form.is_valid():
         uploaded_file = request.FILES['hdfs_file']
         uploaded_file = request.FILES['hdfs_file']
-        dest = scheme_absolute_path(urllib_unquote(request.GET['dest']), urllib_unquote(form.cleaned_data['dest']))
-        filepath = request.fs.join(dest, uploaded_file.name)
+        dest = scheme_absolute_path(unquote_url(request.GET['dest']), unquote_url(request.GET['dest']))
+        filepath = request.fs.join(dest, unquote_url(uploaded_file.name))
 
 
         if request.fs.isdir(dest) and posixpath.sep in uploaded_file.name:
         if request.fs.isdir(dest) and posixpath.sep in uploaded_file.name:
             raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
             raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
@@ -1421,6 +1429,9 @@ def truncate(toTruncate, charsToKeep=50):
     else:
     else:
         return toTruncate
         return toTruncate
 
 
+def unquote_url(url):
+  url = urllib_unquote(url.encode('utf-8') if not isinstance(url, str) else url)
+  return url.decode('utf-8') if isinstance(url, bytes) else url
 
 
 def _is_hdfs_superuser(request):
 def _is_hdfs_superuser(request):
   return request.user.username == request.fs.superuser or request.user.groups.filter(name__exact=request.fs.supergroup).exists()
   return request.user.username == request.fs.superuser or request.user.groups.filter(name__exact=request.fs.supergroup).exists()

+ 109 - 1
apps/filebrowser/src/filebrowser/views_test.py

@@ -53,6 +53,7 @@ from desktop.lib.view_util import location_to_url
 from desktop.conf import is_oozie_enabled
 from desktop.conf import is_oozie_enabled
 from hadoop import pseudo_hdfs4
 from hadoop import pseudo_hdfs4
 from hadoop.conf import UPLOAD_CHUNK_SIZE
 from hadoop.conf import UPLOAD_CHUNK_SIZE
+from hadoop.fs.webhdfs import WebHdfs
 from useradmin.models import User, Group
 from useradmin.models import User, Group
 
 
 from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, MAX_SNAPPY_DECOMPRESSION_SIZE
 from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, MAX_SNAPPY_DECOMPRESSION_SIZE
@@ -60,10 +61,11 @@ from filebrowser.lib.rwx import expand_mode
 from filebrowser.views import snappy_installed
 from filebrowser.views import snappy_installed
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
-  from urllib.parse import unquote as urllib_unquote
+  from urllib.parse import unquote as urllib_unquote, urlparse
   open_file = open
   open_file = open
 else:
 else:
   from urllib import unquote as urllib_unquote
   from urllib import unquote as urllib_unquote
+  from urlparse import urlparse
   open_file = file
   open_file = file
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
@@ -128,6 +130,112 @@ class TestFileBrowser():
         assert_equal(1, len(dir_listing))
         assert_equal(1, len(dir_listing))
 
 
 
 
+  def test_listdir_paged_with_non_ascii(self):
+    parent_dir = Mock(
+      isDir=True,
+      size=0,
+      path=u'/user/systest/test5/Tжейкоб/..',
+      mtime=1581717441.0,
+      mode=16877,
+      user=u'systest',
+      type=u'DIRECTORY',
+      to_json_dict=Mock(
+        return_value={'size': 0, 'group': u'supergroup', 'blockSize': 0, 'replication': 0, 'user': u'systest',
+                      'mtime': 1581717441.0, 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/..',
+                      'atime': 0.0, 'mode': 16877}
+      )
+    )
+    parent_dir.name = u'..'
+    self_dir = Mock(
+      isDir=True,
+      size=0,
+      path=u'/user/systest/test5/Tжейкоб',
+      mtime=1581717441.0,
+      mode=16877,
+      user=u'systest',
+      type=u'DIRECTORY',
+      to_json_dict=Mock(
+        return_value={'size': 0, 'group': u'supergroup', 'blockSize': 0, 'replication': 0, 'user': u'systest',
+                      'mtime': 1581717441.0, 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431',
+                      'atime': 0.0, 'mode': 16877}
+      )
+    )
+    self_dir.name = u'Tжейкоб'
+    file_1 = Mock(
+      isDir=False,
+      size=9,
+      path=u'/user/systest/test5/Tжейкоб/file_1.txt',
+      mtime=1581670301.0, mode=33279,
+      user=u'systest',
+      type=u'FILE',
+      to_json_dict=Mock(
+        return_value={'size': 9, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
+                      'mtime': 1581670301.0,
+                      'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/file_1.txt',
+                      'atime': 1581708019.0, 'mode': 33279}
+      )
+    )
+    file_1.name = u'file_1.txt'
+    file_2 = Mock(
+      isDir=False,
+      size=0,
+      path=u'/user/systest/test5/Tжейкоб/文件_2.txt',
+      mtime=1581707672.0,
+      mode=33188,
+      user=u'systest',
+      type=u'FILE',
+      to_json_dict=Mock(
+        return_value={'size': 18, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
+                      'mtime': 1581707672.0,
+                      'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/\u6587\u4ef6_2.txt',
+                      'atime': 1581707672.0, 'mode': 33188}
+      )
+    )
+    file_2.name = u'文件_2.txt'
+    file_3 = Mock(
+      isDir=False,
+      size=0,
+      path=u'/user/systest/test5/Tжейкоб/employés_file.txt',
+      mtime=1581039792.0,
+      mode=33188,
+      user=u'systest',
+      type=u'FILE',
+      to_json_dict=Mock(
+        return_value={'size': 0, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
+                      'mtime': 1581039792.0,
+                      'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/employ\xe9s_file.txt',
+                      'atime': 1581708003.0, 'mode': 33188}
+      )
+    )
+    file_3.name = u'employés_file.txt'
+
+    with patch('desktop.middleware.fsmanager.get_filesystem') as get_filesystem:
+      with patch('filebrowser.views.snappy_installed') as snappy_installed:
+        snappy_installed.return_value = False
+        get_filesystem.return_value = Mock(
+          stats=Mock(
+            return_value=self_dir
+          ),
+          normpath=WebHdfs.norm_path,
+          is_sentry_managed=Mock(return_value=False),
+          listdir_stats=Mock(
+            return_value=[parent_dir, file_1, file_2, file_3]
+          ),
+          superuser='hdfs',
+          supergroup='hdfs'
+        )
+
+        response = self.client.get('/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1?pagesize=45&pagenum=1&filter=&sortby=name&descending=false&format=json&_=1581670214204')
+
+        assert_equal(200, response.status_code)
+        dir_listing = json.loads(response.content)['files']
+        assert_equal(5, len(dir_listing))
+        assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5",' in response.content, response.content)
+        assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1",' in response.content, response.content)
+        assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2Ffile_1.txt",' in response.content, response.content)
+        assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2F%E6%96%87%E4%BB%B6_2.txt",' in response.content, response.content)
+        assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2Femploy%C3%A9s_file.txt",' in response.content, response.content)
+
 class TestFileBrowserWithHadoop(object):
 class TestFileBrowserWithHadoop(object):
   requires_hadoop = True
   requires_hadoop = True
   integration = True
   integration = True

+ 1 - 1
desktop/libs/aws/src/aws/s3/upload.py

@@ -142,7 +142,7 @@ class S3FileUploadHandler(FileUploadHandler):
   def _get_scheme(self):
   def _get_scheme(self):
     if self.destination:
     if self.destination:
       dst_parts = self.destination.split('://')
       dst_parts = self.destination.split('://')
-      if dst_parts > 0:
+      if dst_parts:
         return dst_parts[0].upper()
         return dst_parts[0].upper()
       else:
       else:
         raise S3FileSystemException('Destination does not start with a valid scheme.')
         raise S3FileSystemException('Destination does not start with a valid scheme.')

+ 1 - 1
desktop/libs/azure/src/azure/abfs/upload.py

@@ -122,7 +122,7 @@ class ABFSFileUploadHandler(FileUploadHandler):
   def _get_scheme(self):
   def _get_scheme(self):
     if self.destination:
     if self.destination:
       dst_parts = self.destination.split('://')
       dst_parts = self.destination.split('://')
-      if dst_parts > 0:
+      if dst_parts:
         return dst_parts[0].upper()
         return dst_parts[0].upper()
       else:
       else:
         raise ABFSFileSystemException('Destination does not start with a valid scheme.')
         raise ABFSFileSystemException('Destination does not start with a valid scheme.')

+ 4 - 0
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -238,6 +238,10 @@ class WebHdfs(Hdfs):
     """
     """
     Return normalized path but ignore leading scheme prefix if it exists
     Return normalized path but ignore leading scheme prefix if it exists
     """
     """
+    return WebHdfs.norm_path(path)
+
+  @staticmethod
+  def norm_path(path):
     path = fs_normpath(path)
     path = fs_normpath(path)
     #fs_normpath clears scheme:/ to scheme: which doesn't make sense
     #fs_normpath clears scheme:/ to scheme: which doesn't make sense
     split = urlparse(path)
     split = urlparse(path)