|
@@ -16,7 +16,7 @@
|
|
|
|
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import absolute_import
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+from mock import MagicMock, patch
|
|
|
from nose.plugins.attrib import attr
|
|
from nose.plugins.attrib import attr
|
|
|
from nose.tools import assert_raises, assert_false, eq_
|
|
from nose.tools import assert_raises, assert_false, eq_
|
|
|
from nose import SkipTest
|
|
from nose import SkipTest
|
|
@@ -29,136 +29,79 @@ from desktop.lib.django_test_util import make_logged_in_client
|
|
|
from desktop.lib.test_utils import add_permission, remove_from_group
|
|
from desktop.lib.test_utils import add_permission, remove_from_group
|
|
|
|
|
|
|
|
|
|
|
|
|
-@attr('integration')
|
|
|
|
|
def test_fs_selection():
|
|
def test_fs_selection():
|
|
|
- try:
|
|
|
|
|
- from mock import MagicMock
|
|
|
|
|
- except ImportError:
|
|
|
|
|
- raise SkipTest("Skips until HUE-2947 is resolved")
|
|
|
|
|
-
|
|
|
|
|
make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
|
user = User.objects.get(username='test')
|
|
user = User.objects.get(username='test')
|
|
|
- add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
|
|
|
- add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
|
|
|
-
|
|
|
|
|
- s3fs, adls, hdfs = MagicMock("s3_access"), MagicMock("adls_access"), MagicMock()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
- proxy_fs.setuser(user)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.isdir('s3a://bucket/key')
|
|
|
|
|
- s3fs.isdir.assert_called_once_with('s3a://bucket/key')
|
|
|
|
|
- assert_false(hdfs.isdir.called)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.isfile('hdfs://localhost:42/user/alice/file')
|
|
|
|
|
- hdfs.isfile.assert_called_once_with('hdfs://localhost:42/user/alice/file')
|
|
|
|
|
- assert_false(s3fs.isfile.called)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.isdir('adl://net/key')
|
|
|
|
|
- s3fs.isdir.assert_called_once_with('adl://net/key')
|
|
|
|
|
- assert_false(hdfs.isdir.called)
|
|
|
|
|
|
|
+ with patch('desktop.lib.fs.ProxyFS._has_access') as _has_access:
|
|
|
|
|
+ _has_access.return_value = True
|
|
|
|
|
|
|
|
- proxy_fs.isdir('adl:/key')
|
|
|
|
|
- s3fs.isdir.assert_called_once_with('adl:/key')
|
|
|
|
|
- assert_false(hdfs.isdir.called)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.open('/user/alice/file')
|
|
|
|
|
- hdfs.open.assert_called_once_with('/user/alice/file')
|
|
|
|
|
- assert_false(s3fs.open.called)
|
|
|
|
|
-
|
|
|
|
|
- assert_raises(IOError, proxy_fs.stats, 'ftp://host')
|
|
|
|
|
|
|
+ s3fs, adls, hdfs, abfs = MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
|
|
|
+ proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs)}, 'hdfs')
|
|
|
|
|
+ proxy_fs.setuser(user)
|
|
|
|
|
|
|
|
|
|
+ proxy_fs.isdir('s3a://bucket/key')
|
|
|
|
|
+ s3fs.isdir.assert_called_once_with('s3a://bucket/key')
|
|
|
|
|
+ assert_false(hdfs.isdir.called)
|
|
|
|
|
|
|
|
-# TODO: remove after HUE-2947 is resolved
|
|
|
|
|
-def test__get_fs():
|
|
|
|
|
- make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
|
|
|
- user = User.objects.get(username='test')
|
|
|
|
|
- add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
|
|
|
- add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
|
|
|
|
|
+ proxy_fs.isfile('hdfs://localhost:42/user/alice/file')
|
|
|
|
|
+ hdfs.isfile.assert_called_once_with('hdfs://localhost:42/user/alice/file')
|
|
|
|
|
+ assert_false(s3fs.isfile.called)
|
|
|
|
|
|
|
|
- s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
- proxy_fs.setuser(user)
|
|
|
|
|
|
|
+ proxy_fs.isdir('adl://net/key')
|
|
|
|
|
+ adls.isdir.assert_called_once_with('adl://net/key')
|
|
|
|
|
+ assert_false(hdfs.isdir.called)
|
|
|
|
|
|
|
|
- f = proxy_fs._get_fs
|
|
|
|
|
|
|
+ proxy_fs.isdir('abfs://net/key')
|
|
|
|
|
+ abfs.isdir.assert_called_once_with('abfs://net/key')
|
|
|
|
|
+ assert_false(hdfs.isdir.called)
|
|
|
|
|
|
|
|
- eq_(f('s3a://bucket'), s3fs)
|
|
|
|
|
- eq_(f('S3A://bucket/key'), s3fs)
|
|
|
|
|
- eq_(f('adl:/path'), adls)
|
|
|
|
|
- eq_(f('adl://net/path'), adls)
|
|
|
|
|
- eq_(f('hdfs:/path'), hdfs)
|
|
|
|
|
- eq_(f('hdfs://net/path'), hdfs)
|
|
|
|
|
- eq_(f('/tmp'), hdfs)
|
|
|
|
|
|
|
+ assert_raises(IOError, proxy_fs.stats, 'ftp://host')
|
|
|
|
|
|
|
|
- assert_raises(IOError, f, 'ftp://host')
|
|
|
|
|
|
|
+def wrapper(mock):
|
|
|
|
|
+ def tmp(*args, **kwargs):
|
|
|
|
|
+ return mock
|
|
|
|
|
+ return tmp
|
|
|
|
|
|
|
|
|
|
|
|
|
-@attr('integration')
|
|
|
|
|
def test_multi_fs_selection():
|
|
def test_multi_fs_selection():
|
|
|
- try:
|
|
|
|
|
- from mock import MagicMock
|
|
|
|
|
- except ImportError:
|
|
|
|
|
- raise SkipTest("Skips until HUE-2947 is resolved")
|
|
|
|
|
-
|
|
|
|
|
make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
|
user = User.objects.get(username='test')
|
|
user = User.objects.get(username='test')
|
|
|
- add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
|
|
|
- add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
|
|
|
-
|
|
|
|
|
- s3fs, adls, hdfs = MagicMock("s3_access"), MagicMock("adls_access"), MagicMock()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
- proxy_fs.setuser(user)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.copy('s3a://bucket1/key', 's3a://bucket2/key')
|
|
|
|
|
- s3fs.copy.assert_called_once_with('s3a://bucket1/key', 's3a://bucket2/key')
|
|
|
|
|
- assert_false(hdfs.copy.called)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.copyfile('s3a://bucket/key', 'key2')
|
|
|
|
|
- s3fs.copyfile.assert_called_once_with('s3a://bucket/key', 'key2')
|
|
|
|
|
- assert_false(hdfs.copyfile.called)
|
|
|
|
|
-
|
|
|
|
|
- proxy_fs.copyfile('adl://net/key', 'key2')
|
|
|
|
|
- s3fs.copyfile.assert_called_once_with('adl://net/key', 'key2')
|
|
|
|
|
- assert_false(hdfs.copyfile.called)
|
|
|
|
|
|
|
|
|
|
- proxy_fs.copyfile('adl:/key', 'key2')
|
|
|
|
|
- s3fs.copyfile.assert_called_once_with('adl:/key', 'key2')
|
|
|
|
|
- assert_false(hdfs.copyfile.called)
|
|
|
|
|
|
|
+ with patch('desktop.lib.fs.ProxyFS._has_access') as _has_access:
|
|
|
|
|
+ _has_access.return_value = True
|
|
|
|
|
|
|
|
- proxy_fs.rename('/tmp/file', 'shmile')
|
|
|
|
|
- hdfs.rename.assert_called_once_with('/tmp/file', 'shmile')
|
|
|
|
|
- assert_false(s3fs.rename.called)
|
|
|
|
|
|
|
+ s3fs, adls, hdfs, abfs = MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
|
|
|
+ proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs)}, 'hdfs')
|
|
|
|
|
+ proxy_fs.setuser(user)
|
|
|
|
|
|
|
|
- # Will be addressed in HUE-2934
|
|
|
|
|
- assert_raises(NotImplementedError, proxy_fs.copy_remote_dir, 's3a://bucket/key', '/tmp/dir')
|
|
|
|
|
|
|
+ proxy_fs.copy('s3a://bucket1/key', 's3a://bucket2/key')
|
|
|
|
|
+ s3fs.copy.assert_called_once_with('s3a://bucket1/key', 's3a://bucket2/key')
|
|
|
|
|
+ assert_false(hdfs.copy.called)
|
|
|
|
|
|
|
|
|
|
+ proxy_fs.copyfile('s3a://bucket/key', 'key2')
|
|
|
|
|
+ s3fs.copyfile.assert_called_once_with('s3a://bucket/key', 'key2')
|
|
|
|
|
+ assert_false(hdfs.copyfile.called)
|
|
|
|
|
|
|
|
-# TODO: remove after HUE-2947 is resolved
|
|
|
|
|
-def test__get_fs_pair():
|
|
|
|
|
- make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
|
|
|
- user = User.objects.get(username='test')
|
|
|
|
|
- add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
|
|
|
- add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
|
|
|
|
|
+ proxy_fs.copyfile('adl://net/key', 'key2')
|
|
|
|
|
+ adls.copyfile.assert_called_once_with('adl://net/key', 'key2')
|
|
|
|
|
+ assert_false(hdfs.copyfile.called)
|
|
|
|
|
|
|
|
- s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
- proxy_fs.setuser(user)
|
|
|
|
|
|
|
+ proxy_fs.copyfile('abfs:/key', 'key2')
|
|
|
|
|
+ abfs.copyfile.assert_called_once_with('abfs:/key', 'key2')
|
|
|
|
|
+ assert_false(hdfs.copyfile.called)
|
|
|
|
|
|
|
|
- f = proxy_fs._get_fs_pair
|
|
|
|
|
|
|
+ proxy_fs.rename('/tmp/file', 'shmile')
|
|
|
|
|
+ hdfs.rename.assert_called_once_with('/tmp/file', 'shmile')
|
|
|
|
|
+ assert_false(s3fs.rename.called)
|
|
|
|
|
|
|
|
- eq_(f('s3a://bucket1/key', 's3a://bucket2/key'), (s3fs, s3fs))
|
|
|
|
|
- eq_(f('s3a://bucket/key', 'key2'), (s3fs, s3fs))
|
|
|
|
|
- eq_(f('adl://net/key', 'key2'), (adls, adls))
|
|
|
|
|
- eq_(f('adl:/key', 'key2'), (adls, adls))
|
|
|
|
|
- eq_(f('/tmp/file', 'shmile'), (hdfs, hdfs))
|
|
|
|
|
-
|
|
|
|
|
- assert_raises(IOError, f, 'ftp://host', 'key2')
|
|
|
|
|
|
|
+ # Will be addressed in HUE-2934
|
|
|
|
|
+ assert_raises(NotImplementedError, proxy_fs.copy_remote_dir, 's3a://bucket/key', 'adl://tmp/dir') # Exception can only be thrown if scheme is specified, else default to 1st scheme
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_constructor_given_invalid_arguments():
|
|
def test_constructor_given_invalid_arguments():
|
|
|
assert_raises(ValueError, ProxyFS, {'s3a': {}}, 'hdfs')
|
|
assert_raises(ValueError, ProxyFS, {'s3a': {}}, 'hdfs')
|
|
|
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
|
class MockFs():
|
|
class MockFs():
|
|
|
def __init__(self, filebrowser_action=None):
|
|
def __init__(self, filebrowser_action=None):
|
|
|
self.user = None
|
|
self.user = None
|
|
@@ -178,20 +121,22 @@ class TestFsPermissions(object):
|
|
|
user_client = make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
user_client = make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
|
|
|
user = User.objects.get(username='test')
|
|
user = User.objects.get(username='test')
|
|
|
|
|
|
|
|
- s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
|
|
+ s3fs, adls, hdfs, abfs = MockFs("s3_access"), MockFs("adls_access"), MockFs(), MockFs("abfs_access")
|
|
|
|
|
+ proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs)}, 'hdfs')
|
|
|
proxy_fs.setuser(user)
|
|
proxy_fs.setuser(user)
|
|
|
|
|
|
|
|
f = proxy_fs._get_fs
|
|
f = proxy_fs._get_fs
|
|
|
|
|
|
|
|
remove_from_group(user.username, 'has_s3')
|
|
remove_from_group(user.username, 'has_s3')
|
|
|
remove_from_group(user.username, 'has_adls')
|
|
remove_from_group(user.username, 'has_adls')
|
|
|
|
|
+ remove_from_group(user.username, 'has_abfs')
|
|
|
|
|
|
|
|
# No perms by default
|
|
# No perms by default
|
|
|
assert_raises(Exception, f, 's3a://bucket')
|
|
assert_raises(Exception, f, 's3a://bucket')
|
|
|
assert_raises(Exception, f, 'S3A://bucket/key')
|
|
assert_raises(Exception, f, 'S3A://bucket/key')
|
|
|
assert_raises(Exception, f, 'adl://net/key')
|
|
assert_raises(Exception, f, 'adl://net/key')
|
|
|
assert_raises(Exception, f, 'adl:/key')
|
|
assert_raises(Exception, f, 'adl:/key')
|
|
|
|
|
+ assert_raises(Exception, f, 'abfs:/key')
|
|
|
f('hdfs://path')
|
|
f('hdfs://path')
|
|
|
f('/tmp')
|
|
f('/tmp')
|
|
|
|
|
|
|
@@ -199,23 +144,26 @@ class TestFsPermissions(object):
|
|
|
# Add perm
|
|
# Add perm
|
|
|
add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
add_permission('test', 'has_s3', permname='s3_access', appname='filebrowser')
|
|
|
add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
|
|
|
|
|
+ add_permission('test', 'has_abfs', permname='abfs_access', appname='filebrowser')
|
|
|
|
|
|
|
|
f('s3a://bucket')
|
|
f('s3a://bucket')
|
|
|
f('S3A://bucket/key')
|
|
f('S3A://bucket/key')
|
|
|
f('adl://net/key')
|
|
f('adl://net/key')
|
|
|
f('adl:/key')
|
|
f('adl:/key')
|
|
|
|
|
+ f('abfs:/key')
|
|
|
f('hdfs://path')
|
|
f('hdfs://path')
|
|
|
f('/tmp')
|
|
f('/tmp')
|
|
|
finally:
|
|
finally:
|
|
|
remove_from_group(user.username, 'has_s3')
|
|
remove_from_group(user.username, 'has_s3')
|
|
|
remove_from_group(user.username, 'has_adls')
|
|
remove_from_group(user.username, 'has_adls')
|
|
|
|
|
+ remove_from_group(user.username, 'has_abfs')
|
|
|
|
|
|
|
|
def test_fs_permissions_admin_user(self):
|
|
def test_fs_permissions_admin_user(self):
|
|
|
user_client = make_logged_in_client(username='admin', groupname='default', recreate=True, is_superuser=True)
|
|
user_client = make_logged_in_client(username='admin', groupname='default', recreate=True, is_superuser=True)
|
|
|
user = User.objects.get(username='admin')
|
|
user = User.objects.get(username='admin')
|
|
|
|
|
|
|
|
- s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
|
|
|
|
|
- proxy_fs = ProxyFS({'s3a': s3fs, 'hdfs': hdfs, 'adl': adls}, 'hdfs')
|
|
|
|
|
|
|
+ s3fs, adls, hdfs, abfs = MockFs("s3_access"), MockFs("adls_access"), MockFs(), MockFs("abfs_access")
|
|
|
|
|
+ proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs)}, 'hdfs')
|
|
|
proxy_fs.setuser(user)
|
|
proxy_fs.setuser(user)
|
|
|
|
|
|
|
|
f = proxy_fs._get_fs
|
|
f = proxy_fs._get_fs
|
|
@@ -224,5 +172,6 @@ class TestFsPermissions(object):
|
|
|
f('S3A://bucket/key')
|
|
f('S3A://bucket/key')
|
|
|
f('adl://net/key')
|
|
f('adl://net/key')
|
|
|
f('adl:/key')
|
|
f('adl:/key')
|
|
|
|
|
+ f('abfs:/key')
|
|
|
f('hdfs://path')
|
|
f('hdfs://path')
|
|
|
f('/tmp')
|
|
f('/tmp')
|