|
|
@@ -14,37 +14,137 @@
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
-from __future__ import absolute_import
|
|
|
|
|
|
-import logging
|
|
|
-import json
|
|
|
import os
|
|
|
-import pytest
|
|
|
-import unittest
|
|
|
-import tempfile
|
|
|
+import json
|
|
|
import time
|
|
|
+import logging
|
|
|
+import tempfile
|
|
|
|
|
|
+import pytest
|
|
|
from django.contrib.auth.models import User
|
|
|
from django.test import TestCase
|
|
|
|
|
|
-from desktop.lib.django_test_util import make_logged_in_client
|
|
|
-from desktop.lib.test_utils import grant_access, add_to_group, add_permission, remove_from_group
|
|
|
-
|
|
|
-from azure.abfs.__init__ import abfspath
|
|
|
+from azure.abfs.__init__ import abfspath, get_abfs_home_directory
|
|
|
from azure.abfs.abfs import ABFS
|
|
|
-from azure.active_directory import ActiveDirectory
|
|
|
-from azure.conf import ABFS_CLUSTERS,AZURE_ACCOUNTS, is_abfs_enabled
|
|
|
-
|
|
|
from azure.abfs.upload import DEFAULT_WRITE_SIZE
|
|
|
+from azure.active_directory import ActiveDirectory
|
|
|
+from azure.conf import ABFS_CLUSTERS, AZURE_ACCOUNTS, is_abfs_enabled
|
|
|
+from desktop.conf import RAZ
|
|
|
+from desktop.lib.django_test_util import make_logged_in_client
|
|
|
+from desktop.lib.test_utils import add_permission, add_to_group, grant_access, remove_from_group
|
|
|
+from filebrowser.conf import REMOTE_STORAGE_HOME
|
|
|
|
|
|
LOG = logging.getLogger()
|
|
|
|
|
|
-"""
|
|
|
-Interfaces for ADLS via HttpFs/WebHDFS
|
|
|
-"""
|
|
|
+
|
|
|
+@pytest.mark.django_db
|
|
|
+def test_get_abfs_home_directory():
|
|
|
+ client = make_logged_in_client(username="test", groupname="test", recreate=True, is_superuser=False)
|
|
|
+ user = User.objects.get(username="test")
|
|
|
+
|
|
|
+ client_not_me = make_logged_in_client(username="test_not_me", groupname="test_not_me", recreate=True, is_superuser=False)
|
|
|
+ user_not_me = User.objects.get(username="test_not_me")
|
|
|
+
|
|
|
+ # When REMOTE_STORAGE_HOME ends with /user in RAZ ABFS environment.
|
|
|
+ resets = [RAZ.IS_ENABLED.set_for_testing(True), REMOTE_STORAGE_HOME.set_for_testing('abfs://gethue-container/user')]
|
|
|
+
|
|
|
+ try:
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-container/user/test'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-container/user/test_not_me'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+ # When ABFS filesystem's DEFAULT_HOME_PATH ends with /user in RAZ ABFS environment.
|
|
|
+ resets = [
|
|
|
+ RAZ.IS_ENABLED.set_for_testing(True),
|
|
|
+ ABFS_CLUSTERS.set_for_testing({'default': {'default_home_path': 'abfs://gethue-other-container/user'}}),
|
|
|
+ ]
|
|
|
+
|
|
|
+ try:
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/user/test'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/user/test_not_me'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+ # When ABFS filesystem's DEFAULT_HOME_PATH is set in non-RAZ ABFS environment.
|
|
|
+ resets = [
|
|
|
+ RAZ.IS_ENABLED.set_for_testing(False),
|
|
|
+ ABFS_CLUSTERS.set_for_testing({'default': {'default_home_path': 'abfs://gethue-other-container/test-dir'}}),
|
|
|
+ ]
|
|
|
+
|
|
|
+ try:
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/test-dir'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/test-dir'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+ # When both REMOTE_STORAGE_HOME and ABFS filesystem's DEFAULT_HOME_PATH are set in RAZ ABFS environment.
|
|
|
+ resets = [
|
|
|
+ RAZ.IS_ENABLED.set_for_testing(True),
|
|
|
+ REMOTE_STORAGE_HOME.set_for_testing('abfs://gethue-container/user'),
|
|
|
+ ABFS_CLUSTERS.set_for_testing({'default': {'default_home_path': 'abfs://gethue-other-container/user'}}),
|
|
|
+ ]
|
|
|
+
|
|
|
+ try:
|
|
|
+ # Gives preference to REMOTE_STORAGE_HOME for of backward compatibility.
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-container/user/test'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-container/user/test_not_me'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+ # When ABFS filesystem's DEFAULT_HOME_PATH is set but path does not end with ../user or ../user/ in RAZ ABFS environment.
|
|
|
+ resets = [
|
|
|
+ RAZ.IS_ENABLED.set_for_testing(True),
|
|
|
+ ABFS_CLUSTERS.set_for_testing({'default': {'default_home_path': 'abfs://gethue-other-container/dir'}}),
|
|
|
+ ]
|
|
|
+
|
|
|
+ try:
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/dir'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://gethue-other-container/dir'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+ # When some different path is set in both RAZ and non-RAZ ABFS environment.
|
|
|
+ resets = [
|
|
|
+ RAZ.IS_ENABLED.set_for_testing(True),
|
|
|
+ REMOTE_STORAGE_HOME.set_for_testing('s3a://gethue-bucket/user'),
|
|
|
+ ABFS_CLUSTERS.set_for_testing({'default': {'default_home_path': 's3a://gethue-other-bucket/dir'}}),
|
|
|
+ ]
|
|
|
+
|
|
|
+ try:
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user)
|
|
|
+ assert default_abfs_home_path == 'abfs://'
|
|
|
+
|
|
|
+ default_abfs_home_path = get_abfs_home_directory(user_not_me)
|
|
|
+ assert default_abfs_home_path == 'abfs://'
|
|
|
+ finally:
|
|
|
+ for reset in resets:
|
|
|
+ reset()
|
|
|
+
|
|
|
+
|
|
|
@pytest.mark.integration
|
|
|
class ABFSTestBase(TestCase):
|
|
|
-
|
|
|
def setup_method(self, method):
|
|
|
if not is_abfs_enabled():
|
|
|
pytest.skip("Skipping Test")
|
|
|
@@ -53,61 +153,60 @@ class ABFSTestBase(TestCase):
|
|
|
grant_access('test', 'test', 'filebrowser')
|
|
|
add_to_group('test')
|
|
|
self.user = User.objects.get(username="test")
|
|
|
-
|
|
|
- self.test_fs = 'abfs://test' + (str(int(time.time()) ))
|
|
|
+
|
|
|
+ self.test_fs = 'abfs://test' + (str(int(time.time())))
|
|
|
LOG.debug("%s" % self.test_fs)
|
|
|
self.client.mkdir(self.test_fs)
|
|
|
|
|
|
def teardown_method(self, method):
|
|
|
self.client.rmtree(self.test_fs)
|
|
|
-
|
|
|
+
|
|
|
def test_list(self):
|
|
|
testfile = 'abfs://'
|
|
|
filesystems = self.client.listdir(testfile)
|
|
|
LOG.debug("%s" % filesystems)
|
|
|
assert filesystems is not None, filesystems
|
|
|
-
|
|
|
- pathing = self.client.listdir(testfile + filesystems[0], {"recursive" : "true"} )
|
|
|
+
|
|
|
+ pathing = self.client.listdir(testfile + filesystems[0], {"recursive": "true"})
|
|
|
LOG.debug("%s" % pathing)
|
|
|
assert pathing is not None, pathing
|
|
|
-
|
|
|
+
|
|
|
directory = self.client.listdir(testfile + filesystems[0] + '/' + pathing[0])
|
|
|
LOG.debug("%s" % directory)
|
|
|
assert directory is not None, directory
|
|
|
-
|
|
|
+
|
|
|
directory = self.client.listdir(self.test_fs)
|
|
|
LOG.debug("%s" % directory)
|
|
|
assert directory is not None, directory
|
|
|
-
|
|
|
+
|
|
|
directory = self.client.listdir(abfspath(self.test_fs))
|
|
|
LOG.debug("%s" % directory)
|
|
|
assert directory is not None, directory
|
|
|
-
|
|
|
+
|
|
|
pathing = self.client._statsf(filesystems[276])
|
|
|
LOG.debug("%s" % pathing)
|
|
|
assert pathing is not None, pathing
|
|
|
-
|
|
|
+
|
|
|
pathing = self.client._statsf(filesystems[277])
|
|
|
LOG.debug("%s" % pathing)
|
|
|
assert pathing is not None, pathing
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
def test_existence(self):
|
|
|
test_fs = self.test_fs
|
|
|
test_dir = test_fs + '/test_existence'
|
|
|
test_file = test_dir + '/test.txt'
|
|
|
self.client.mkdir(test_dir)
|
|
|
self.client.create(test_file)
|
|
|
-
|
|
|
- #Testing root and filesystems
|
|
|
+
|
|
|
+ # Testing root and filesystems
|
|
|
assert self.client.exists('abfs://')
|
|
|
assert self.client.exists(test_fs)
|
|
|
-
|
|
|
- #testing created directories and files
|
|
|
+
|
|
|
+ # testing created directories and files
|
|
|
assert self.client.exists(test_dir)
|
|
|
assert self.client.exists(test_file)
|
|
|
assert not self.client.exists(test_dir + 'a')
|
|
|
-
|
|
|
+
|
|
|
def test_stat_output(self):
|
|
|
"""
|
|
|
Only tests if the stat outputs something
|
|
|
@@ -119,59 +218,58 @@ class ABFSTestBase(TestCase):
|
|
|
self.client.mkdir(test_dir)
|
|
|
self.client.mkdir(test_dir2)
|
|
|
self.client.mkdir(test_dir3)
|
|
|
-
|
|
|
- #testing filesystems
|
|
|
+
|
|
|
+ # testing filesystems
|
|
|
result = self.client.stats(test_fs)
|
|
|
LOG.debug("%s" % result)
|
|
|
assert result is not None, result
|
|
|
result = self.client.listdir_stats(test_fs)
|
|
|
LOG.debug("%s" % result)
|
|
|
-
|
|
|
- #testing directories
|
|
|
+
|
|
|
+ # testing directories
|
|
|
result = self.client.stats(test_dir)
|
|
|
LOG.debug("%s" % result)
|
|
|
result = self.client.listdir_stats(test_dir)
|
|
|
LOG.debug("%s" % result)
|
|
|
-
|
|
|
+
|
|
|
result = self.client.stats(test_dir2)
|
|
|
LOG.debug("%s" % result)
|
|
|
result = self.client.listdir_stats(test_dir2)
|
|
|
LOG.debug("%s" % result)
|
|
|
-
|
|
|
+
|
|
|
result = self.client.stats(test_dir3)
|
|
|
LOG.debug("%s" % result)
|
|
|
result = self.client.listdir_stats(test_dir3)
|
|
|
LOG.debug("%s" % result)
|
|
|
-
|
|
|
+
|
|
|
def test_mkdir(self):
|
|
|
test_dir = self.test_fs + '/test_mkdir'
|
|
|
assert not self.client.exists(test_dir)
|
|
|
-
|
|
|
+
|
|
|
self.client.mkdir(test_dir)
|
|
|
assert self.client.exists(test_dir)
|
|
|
self.client.isdir(test_dir)
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
def test_append_and_flush(self):
|
|
|
test_fs = self.test_fs
|
|
|
test_file = test_fs + '/test.txt'
|
|
|
self.client.create(test_file)
|
|
|
-
|
|
|
+
|
|
|
test_string = "This is a test."
|
|
|
test_len = len(test_string)
|
|
|
- resp = self.client._append(test_file, test_string) #only works with strings
|
|
|
+ resp = self.client._append(test_file, test_string) # only works with strings
|
|
|
LOG.debug("%s" % self.client.stats(test_file))
|
|
|
try:
|
|
|
LOG.debug("%s" % resp)
|
|
|
- resp = self.client.read(test_file, length = test_len)
|
|
|
- except:
|
|
|
+ resp = self.client.read(test_file, length=test_len)
|
|
|
+ except Exception:
|
|
|
LOG.debug("Not written yet")
|
|
|
-
|
|
|
- self.client.flush(test_file, {"position" : test_len} )
|
|
|
+
|
|
|
+ self.client.flush(test_file, {"position": test_len})
|
|
|
resp = self.client.read(test_file)
|
|
|
assert resp == test_string
|
|
|
self.client.remove(test_file)
|
|
|
-
|
|
|
+
|
|
|
def test_rename(self):
|
|
|
test_fs = self.test_fs
|
|
|
test_dir = test_fs + '/test'
|
|
|
@@ -180,67 +278,66 @@ class ABFSTestBase(TestCase):
|
|
|
test_file = test_fs + '/test.txt'
|
|
|
test_file2 = test_fs + '/test2.txt'
|
|
|
test_file3 = test_fs + '/test 3.txt'
|
|
|
-
|
|
|
+
|
|
|
self.client.mkdir(test_dir)
|
|
|
assert self.client.exists(test_dir)
|
|
|
assert not self.client.exists(test_dir2)
|
|
|
-
|
|
|
+
|
|
|
self.client.rename(test_dir, test_dir2)
|
|
|
assert not self.client.exists(test_dir)
|
|
|
assert self.client.exists(test_dir2)
|
|
|
-
|
|
|
+
|
|
|
self.client.create(test_file)
|
|
|
assert self.client.exists(test_file)
|
|
|
assert not self.client.exists(test_file2)
|
|
|
-
|
|
|
+
|
|
|
self.client.rename(test_file, test_file2)
|
|
|
assert not self.client.exists(test_file)
|
|
|
assert self.client.exists(test_file2)
|
|
|
-
|
|
|
+
|
|
|
self.client.rename(test_dir2, test_dir3)
|
|
|
assert not self.client.exists(test_dir2)
|
|
|
assert self.client.exists(test_dir3)
|
|
|
-
|
|
|
+
|
|
|
self.client.rename(test_dir3, test_dir2)
|
|
|
assert not self.client.exists(test_dir3)
|
|
|
assert self.client.exists(test_dir2)
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
def test_chmod(self):
|
|
|
test_dir = self.test_fs + '/test_chmod'
|
|
|
self.client.mkdir(test_dir)
|
|
|
- test_dir_permission = test_dir +'/test'
|
|
|
- test_file_permission = test_dir +'/test.txt'
|
|
|
-
|
|
|
+ test_dir_permission = test_dir + '/test'
|
|
|
+ test_file_permission = test_dir + '/test.txt'
|
|
|
+
|
|
|
self.client.create(test_file_permission)
|
|
|
self.client.chmod(test_file_permission, '0777')
|
|
|
self.client.stats(test_file_permission)
|
|
|
-
|
|
|
+
|
|
|
self.client.mkdir(test_dir_permission)
|
|
|
self.client.chmod(test_dir_permission, '0000')
|
|
|
self.client.chmod(test_dir_permission, '0777')
|
|
|
self.client.stats(test_dir_permission)
|
|
|
-
|
|
|
+
|
|
|
def test_chown(self):
|
|
|
test_dir = self.test_fs + '/test_chown'
|
|
|
self.client.mkdir(test_dir)
|
|
|
- test_dir_permission = test_dir +'/test'
|
|
|
- test_file_permission = test_dir +'/test.txt'
|
|
|
-
|
|
|
+ test_dir_permission = test_dir + '/test'
|
|
|
+ test_file_permission = test_dir + '/test.txt'
|
|
|
+
|
|
|
self.client.create(test_file_permission)
|
|
|
- self.client.chown(test_file_permission, group = '$superuser' )
|
|
|
+ self.client.chown(test_file_permission, group='$superuser')
|
|
|
self.client.stats(test_file_permission)
|
|
|
-
|
|
|
+
|
|
|
self.client.mkdir(test_dir_permission)
|
|
|
- self.client.chown(test_dir_permission, group = '$superuser')
|
|
|
+ self.client.chown(test_dir_permission, group='$superuser')
|
|
|
self.client.stats(test_dir_permission)
|
|
|
-
|
|
|
+
|
|
|
def test_create_with_file_permissions(self):
|
|
|
test_dir = self.test_fs + '/test_chown'
|
|
|
test_file = test_dir + '/test.txt'
|
|
|
self.client.mkdir(test_dir)
|
|
|
- self.client.create(test_file, headers = {'x-ms-permissions' : '0777'})
|
|
|
-
|
|
|
+ self.client.create(test_file, headers={'x-ms-permissions': '0777'})
|
|
|
+
|
|
|
def test_upload(self):
|
|
|
with tempfile.NamedTemporaryFile() as local_file:
|
|
|
# Make sure we can upload larger than the UPLOAD chunk size
|
|
|
@@ -251,7 +348,7 @@ class ABFSTestBase(TestCase):
|
|
|
dest_dir = self.test_fs + '/test_upload'
|
|
|
local_file = local_file.name
|
|
|
dest_path = '%s/%s' % (dest_dir, os.path.basename(local_file))
|
|
|
-
|
|
|
+
|
|
|
add_permission(self.user.username, 'has_abfs', permname='abfs_access', appname='filebrowser')
|
|
|
# Just upload the current python file
|
|
|
try:
|
|
|
@@ -259,15 +356,14 @@ class ABFSTestBase(TestCase):
|
|
|
response = json.loads(resp.content)
|
|
|
finally:
|
|
|
remove_from_group(self.user.username, 'has_abfs')
|
|
|
-
|
|
|
+
|
|
|
assert 0 == response['status'], response
|
|
|
stats = self.client.stats(dest_path)
|
|
|
|
|
|
actual = self.client.read(dest_path)
|
|
|
expected = file(local_file).read()
|
|
|
assert actual == expected, 'files do not match: %s != %s' % (len(actual), len(expected))
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
def test_copy_file(self):
|
|
|
test_fs = self.test_fs
|
|
|
testdir1 = test_fs + '/testcpy1'
|
|
|
@@ -276,19 +372,18 @@ class ABFSTestBase(TestCase):
|
|
|
self.client.mkdir(testdir1)
|
|
|
self.client.mkdir(testdir2)
|
|
|
self.client.create(test_file)
|
|
|
-
|
|
|
+
|
|
|
test_string = "This is a test."
|
|
|
test_len = len(test_string)
|
|
|
resp = self.client._append(test_file, test_string)
|
|
|
- self.client.flush(test_file, {"position" : test_len} )
|
|
|
-
|
|
|
+ self.client.flush(test_file, {"position": test_len})
|
|
|
+
|
|
|
self.client.copy(test_file, testdir2)
|
|
|
self.client.stats(testdir2 + '/test.txt')
|
|
|
resp = self.client.read(testdir2 + '/test.txt')
|
|
|
resp2 = self.client.read(test_file)
|
|
|
assert resp == resp2, "Files %s and %s are not equal" % (test_file, testdir2 + '/test.txt')
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
def test_copy_dir(self):
|
|
|
test_fs = self.test_fs
|
|
|
testdir1 = test_fs + '/testcpy1'
|
|
|
@@ -299,12 +394,11 @@ class ABFSTestBase(TestCase):
|
|
|
self.client.mkdir(testdir2)
|
|
|
self.client.mkdir(test_dir3)
|
|
|
self.client.mkdir(test_dir4)
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
self.client.copy(test_dir3, testdir2)
|
|
|
self.client.stats(testdir2 + '/test')
|
|
|
self.client.stats(testdir2 + '/test/test2')
|
|
|
-
|
|
|
+
|
|
|
@staticmethod
|
|
|
def test_static_methods():
|
|
|
test_dir = 'abfss://testfs/test_static/'
|
|
|
@@ -315,5 +409,3 @@ class ABFSTestBase(TestCase):
|
|
|
LOG.debug("%s" % parent)
|
|
|
join_path = ABFS.join(test_dir, 'test1')
|
|
|
LOG.debug("%s" % join_path)
|
|
|
-
|
|
|
-
|