|
|
@@ -53,6 +53,7 @@ from desktop.lib.view_util import location_to_url
|
|
|
from desktop.conf import is_oozie_enabled
|
|
|
from hadoop import pseudo_hdfs4
|
|
|
from hadoop.conf import UPLOAD_CHUNK_SIZE
|
|
|
+from hadoop.fs.webhdfs import WebHdfs
|
|
|
from useradmin.models import User, Group
|
|
|
|
|
|
from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, MAX_SNAPPY_DECOMPRESSION_SIZE
|
|
|
@@ -60,10 +61,11 @@ from filebrowser.lib.rwx import expand_mode
|
|
|
from filebrowser.views import snappy_installed
|
|
|
|
|
|
if sys.version_info[0] > 2:
|
|
|
- from urllib.parse import unquote as urllib_unquote
|
|
|
+ from urllib.parse import unquote as urllib_unquote, urlparse
|
|
|
open_file = open
|
|
|
else:
|
|
|
from urllib import unquote as urllib_unquote
|
|
|
+ from urlparse import urlparse
|
|
|
open_file = file
|
|
|
|
|
|
if sys.version_info[0] > 2:
|
|
|
@@ -128,6 +130,112 @@ class TestFileBrowser():
|
|
|
assert_equal(1, len(dir_listing))
|
|
|
|
|
|
|
|
|
+ def test_listdir_paged_with_non_ascii(self):
|
|
|
+ parent_dir = Mock(
|
|
|
+ isDir=True,
|
|
|
+ size=0,
|
|
|
+ path=u'/user/systest/test5/Tжейкоб/..',
|
|
|
+ mtime=1581717441.0,
|
|
|
+ mode=16877,
|
|
|
+ user=u'systest',
|
|
|
+ type=u'DIRECTORY',
|
|
|
+ to_json_dict=Mock(
|
|
|
+ return_value={'size': 0, 'group': u'supergroup', 'blockSize': 0, 'replication': 0, 'user': u'systest',
|
|
|
+ 'mtime': 1581717441.0, 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/..',
|
|
|
+ 'atime': 0.0, 'mode': 16877}
|
|
|
+ )
|
|
|
+ )
|
|
|
+ parent_dir.name = u'..'
|
|
|
+ self_dir = Mock(
|
|
|
+ isDir=True,
|
|
|
+ size=0,
|
|
|
+ path=u'/user/systest/test5/Tжейкоб',
|
|
|
+ mtime=1581717441.0,
|
|
|
+ mode=16877,
|
|
|
+ user=u'systest',
|
|
|
+ type=u'DIRECTORY',
|
|
|
+ to_json_dict=Mock(
|
|
|
+ return_value={'size': 0, 'group': u'supergroup', 'blockSize': 0, 'replication': 0, 'user': u'systest',
|
|
|
+ 'mtime': 1581717441.0, 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431',
|
|
|
+ 'atime': 0.0, 'mode': 16877}
|
|
|
+ )
|
|
|
+ )
|
|
|
+ self_dir.name = u'Tжейкоб'
|
|
|
+ file_1 = Mock(
|
|
|
+ isDir=False,
|
|
|
+ size=9,
|
|
|
+ path=u'/user/systest/test5/Tжейкоб/file_1.txt',
|
|
|
+ mtime=1581670301.0, mode=33279,
|
|
|
+ user=u'systest',
|
|
|
+ type=u'FILE',
|
|
|
+ to_json_dict=Mock(
|
|
|
+ return_value={'size': 9, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
|
|
|
+ 'mtime': 1581670301.0,
|
|
|
+ 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/file_1.txt',
|
|
|
+ 'atime': 1581708019.0, 'mode': 33279}
|
|
|
+ )
|
|
|
+ )
|
|
|
+ file_1.name = u'file_1.txt'
|
|
|
+ file_2 = Mock(
|
|
|
+ isDir=False,
|
|
|
+ size=0,
|
|
|
+ path=u'/user/systest/test5/Tжейкоб/文件_2.txt',
|
|
|
+ mtime=1581707672.0,
|
|
|
+ mode=33188,
|
|
|
+ user=u'systest',
|
|
|
+ type=u'FILE',
|
|
|
+ to_json_dict=Mock(
|
|
|
+ return_value={'size': 18, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
|
|
|
+ 'mtime': 1581707672.0,
|
|
|
+ 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/\u6587\u4ef6_2.txt',
|
|
|
+ 'atime': 1581707672.0, 'mode': 33188}
|
|
|
+ )
|
|
|
+ )
|
|
|
+ file_2.name = u'文件_2.txt'
|
|
|
+ file_3 = Mock(
|
|
|
+ isDir=False,
|
|
|
+ size=0,
|
|
|
+ path=u'/user/systest/test5/Tжейкоб/employés_file.txt',
|
|
|
+ mtime=1581039792.0,
|
|
|
+ mode=33188,
|
|
|
+ user=u'systest',
|
|
|
+ type=u'FILE',
|
|
|
+ to_json_dict=Mock(
|
|
|
+ return_value={'size': 0, 'group': u'supergroup', 'blockSize': 134217728, 'replication': 1, 'user': u'systest',
|
|
|
+ 'mtime': 1581039792.0,
|
|
|
+ 'path': u'/user/systest/test5/T\u0436\u0435\u0439\u043a\u043e\u0431/employ\xe9s_file.txt',
|
|
|
+ 'atime': 1581708003.0, 'mode': 33188}
|
|
|
+ )
|
|
|
+ )
|
|
|
+ file_3.name = u'employés_file.txt'
|
|
|
+
|
|
|
+ with patch('desktop.middleware.fsmanager.get_filesystem') as get_filesystem:
|
|
|
+ with patch('filebrowser.views.snappy_installed') as snappy_installed:
|
|
|
+ snappy_installed.return_value = False
|
|
|
+ get_filesystem.return_value = Mock(
|
|
|
+ stats=Mock(
|
|
|
+ return_value=self_dir
|
|
|
+ ),
|
|
|
+ normpath=WebHdfs.norm_path,
|
|
|
+ is_sentry_managed=Mock(return_value=False),
|
|
|
+ listdir_stats=Mock(
|
|
|
+ return_value=[parent_dir, file_1, file_2, file_3]
|
|
|
+ ),
|
|
|
+ superuser='hdfs',
|
|
|
+ supergroup='hdfs'
|
|
|
+ )
|
|
|
+
|
|
|
+ response = self.client.get('/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1?pagesize=45&pagenum=1&filter=&sortby=name&descending=false&format=json&_=1581670214204')
|
|
|
+
|
|
|
+ assert_equal(200, response.status_code)
|
|
|
+ dir_listing = json.loads(response.content)['files']
|
|
|
+ assert_equal(5, len(dir_listing))
|
|
|
+ assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5",' in response.content, response.content)
|
|
|
+ assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1",' in response.content, response.content)
|
|
|
+ assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2Ffile_1.txt",' in response.content, response.content)
|
|
|
+ assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2F%E6%96%87%E4%BB%B6_2.txt",' in response.content, response.content)
|
|
|
+ assert_true(b'"url": "/filebrowser/view=%2Fuser%2Fsystest%2Ftest5%2FT%D0%B6%D0%B5%D0%B9%D0%BA%D0%BE%D0%B1%2Femploy%C3%A9s_file.txt",' in response.content, response.content)
|
|
|
+
|
|
|
class TestFileBrowserWithHadoop(object):
|
|
|
requires_hadoop = True
|
|
|
integration = True
|