Browse Source

[ozone] Add unit tests and update existing tests for ofs

Harshg999 2 years ago
parent
commit
08d1474354

+ 90 - 8
apps/filebrowser/src/filebrowser/views_test.py

@@ -53,7 +53,7 @@ from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group, add_permission, remove_from_group
 from desktop.lib.view_util import location_to_url
-from desktop.conf import is_oozie_enabled, RAZ
+from desktop.conf import is_oozie_enabled, RAZ, is_ofs_enabled, OZONE
 
 from hadoop import pseudo_hdfs4
 from hadoop.conf import UPLOAD_CHUNK_SIZE
@@ -1530,6 +1530,45 @@ class TestADLSAccessPermissions(object):
     finally:
       remove_from_group(self.user.username, 'has_adls')
 
+class TestOFSAccessPermissions(object):
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    grant_access('test', 'test', 'filebrowser')
+    add_to_group('test')
+
+    self.user = User.objects.get(username="test")
+
+  def test_no_default_permissions(self):
+    response = self.client.get('/filebrowser/view=ofs://')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume/bucket')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume/bucket/hue')
+    assert_equal(500, response.status_code)
+
+    response = self.client.post('/filebrowser/rmtree', dict(path=['ofs://volume/bucket/hue']))
+    assert_equal(500, response.status_code)
+
+    # 500 for real currently
+    assert_raises(IOError, self.client.get, '/filebrowser/edit=ofs://volume/bucket/hue')
+
+  def test_has_default_permissions(self):
+    if not is_ofs_enabled():
+      raise SkipTest
+
+    add_permission(self.user.username, 'has_ofs', permname='ofs_access', appname='filebrowser')
+
+    try:
+      response = self.client.get('/filebrowser/view=ofs://')
+      assert_equal(200, response.status_code)
+    finally:
+      remove_from_group(self.user.username, 'has_ofs')
+
 
 class TestFileChooserRedirect(object):
 
@@ -1540,7 +1579,7 @@ class TestFileChooserRedirect(object):
 
     self.user = User.objects.get(username="test")
 
-  def test_hdfs_redirect(self):
+  def test_fs_redirect(self):
     with patch('desktop.lib.fs.proxyfs.ProxyFS.isdir') as is_dir:
       is_dir.return_value = True
 
@@ -1550,11 +1589,34 @@ class TestFileChooserRedirect(object):
       assert_equal(302, response.status_code)
       assert_equal('/filebrowser/view=%2Fuser%2Ftest', response.url)
 
+      # OFS - default_ofs_home
+      reset = OZONE['default'].WEBHDFS_URL.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_ofs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=ofs%3A%2F%2F', response.url)
+      finally:
+        reset()
+
+      reset = OZONE['default'].WEBHDFS_URL.set_for_testing('http://localhost:9778/webhdfs/v1')
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_ofs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=ofs%3A%2F%2F', response.url)
+      finally:
+        reset()
+
       # ABFS - default_abfs_home
-      response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+      reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
 
-      assert_equal(302, response.status_code)
-      assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
+      finally:
+        reset()
 
       reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing('abfs://data-container@mystorage.dfs.core.windows.net')
       try:
@@ -1592,10 +1654,14 @@ class TestFileChooserRedirect(object):
           reset()
 
       # S3A - default_s3_home
-      response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+      reset = REMOTE_STORAGE_HOME.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_s3_home')
 
-      assert_equal(302, response.status_code)
-      assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
+      finally:
+        reset()
 
       reset = REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket')
       try:
@@ -1683,3 +1749,19 @@ class TestNormalizePath(object):
 
     normalized = _normalize_path(path)
     assert_equal(path, normalized)
+
+  def test_ofs_correction(self):
+    path = 'ofs:%2Fsome%2Fpath'
+    expected_corrected_path = 'ofs://some/path'
+
+    normalized_once = _normalize_path(path)
+    assert_equal(expected_corrected_path, normalized_once)
+
+    normalized_twice = _normalize_path(normalized_once)
+    assert_equal(expected_corrected_path, normalized_twice)
+
+  def test_ofs_correction_already_correct(self):
+    path = 'ofs://some/path'
+
+    normalized = _normalize_path(path)
+    assert_equal(path, normalized)

+ 116 - 0
desktop/core/src/desktop/lib/fs/ozone/ofs_test.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import assert_equal
+
+from desktop import conf
+from desktop.lib.fs.ozone.client import _make_ofs_client
+
+
+class TestOFSClient(object):
+
+  @classmethod
+  def setUpClass(cls):
+    cls._resets = [
+      conf.OZONE['default'].FS_DEFAULTFS.set_for_testing('ofs://ozone1'),
+      conf.OZONE['default'].LOGICAL_NAME.set_for_testing('test-logical-name'),
+      conf.OZONE['default'].WEBHDFS_URL.set_for_testing('http://gethue-ozone:9778/webhdfs/v1'),
+      conf.OZONE['default'].SECURITY_ENABLED.set_for_testing(True),
+      conf.OZONE['default'].SSL_CERT_CA_VERIFY.set_for_testing(True),
+      conf.OZONE['default'].TEMP_DIR.set_for_testing('/tmp')
+    ]
+
+    cls.ofs_client = _make_ofs_client('default')
+
+
+  def test_client_attributes(self):
+    assert_equal(self.ofs_client._url, 'http://gethue-ozone:9778/webhdfs/v1')
+    assert_equal(self.ofs_client._superuser, None)
+    assert_equal(self.ofs_client._security_enabled, True)
+    assert_equal(self.ofs_client._ssl_cert_ca_verify, True)
+    assert_equal(self.ofs_client._temp_dir, '/tmp')
+    assert_equal(self.ofs_client._umask, 530)
+    assert_equal(self.ofs_client._fs_defaultfs, 'ofs://ozone1')
+    assert_equal(self.ofs_client._logical_name, 'test-logical-name')
+    assert_equal(self.ofs_client._supergroup, None)
+    assert_equal(self.ofs_client._scheme, 'ofs')
+    assert_equal(self.ofs_client._netloc, 'ozone1')
+    assert_equal(self.ofs_client._is_remote, True)
+    assert_equal(self.ofs_client._has_trash_support, False)
+    assert_equal(self.ofs_client.expiration, None)
+    assert_equal(self.ofs_client._filebrowser_action, 'ofs_access')
+
+
+  def test_strip_normpath(self):
+    test_path = self.ofs_client.strip_normpath('ofs://vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+    test_path = self.ofs_client.strip_normpath('ofs:/vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+    test_path = self.ofs_client.strip_normpath('/vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+
+  def test_normpath(self):
+    test_path = self.ofs_client.normpath('ofs://')
+    assert_equal(test_path, 'ofs://')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key/')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key//')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1//key//')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+
+  def test_isroot(self):
+    is_root = self.ofs_client.isroot('ofs://vol1/buk1/key')
+    assert_equal(is_root, False)
+
+    is_root = self.ofs_client.isroot('ofs://')
+    assert_equal(is_root, True)
+
+
+  def test_parent_path(self):
+    parent_path = self.ofs_client.parent_path('ofs://')
+    assert_equal(parent_path, 'ofs://')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1/dir1/file1.csv')
+    assert_equal(parent_path, 'ofs://vol1/buk1/dir1')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1/key')
+    assert_equal(parent_path, 'ofs://vol1/buk1')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1')
+    assert_equal(parent_path, 'ofs://vol1/')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1')
+    assert_equal(parent_path, 'ofs://')
+
+
+  @classmethod
+  def tearDownClass(cls):
+    for reset in cls._resets:
+      reset()
+
+

+ 56 - 0
desktop/core/src/desktop/lib/fs/ozone/ofsstat_test.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import assert_equal
+from desktop.lib.fs.ozone.ofsstat import OzoneFSStat
+
+
+class TestOzoneFSStat(object):
+  def setUp(self):
+    test_file_status = {
+      'pathSuffix': 'testfile.csv', 'type': 'FILE', 'length': 32, 'owner': 'hueadmin', 'group': 'huegroup',
+      'permission': '666', 'accessTime': 1677914460588, 'modificationTime': 1677914460588, 'blockSize': 268435456, 'replication': 3}
+
+    test_parent_path = '/gethue/'
+
+    self.stat = OzoneFSStat(test_file_status, test_parent_path)
+
+
+  def test_stat_attributes(self):
+    assert_equal(self.stat.name, 'testfile.csv')
+    assert_equal(self.stat.path, 'ofs://gethue/testfile.csv')
+    assert_equal(self.stat.isDir, False)
+    assert_equal(self.stat.type, 'FILE')
+    assert_equal(self.stat.atime, 1677914460)
+    assert_equal(self.stat.mtime, 1677914460)
+    assert_equal(self.stat.user, 'hueadmin')
+    assert_equal(self.stat.group, 'huegroup')
+    assert_equal(self.stat.size, 32)
+    assert_equal(self.stat.blockSize, 268435456)
+    assert_equal(self.stat.replication, 3)
+    assert_equal(self.stat.aclBit, None)
+    assert_equal(self.stat.fileId, None)
+    assert_equal(self.stat.mode, 33206)
+
+
+  def test_to_json_dict(self):
+    expected_json_dict = {
+      'path': 'ofs://gethue/testfile.csv', 'size': 32, 'atime': 1677914460, 'mtime': 1677914460, 'mode': 33206, 'user': 'hueadmin',
+      'group': 'huegroup', 'blockSize': 268435456, 'replication': 3}
+
+    assert_equal(self.stat.to_json_dict(), expected_json_dict)

+ 37 - 10
desktop/core/src/desktop/lib/fs/proxyfs_test.py

@@ -42,8 +42,10 @@ def test_fs_selection():
   with patch('desktop.lib.fs.ProxyFS._has_access') as _has_access:
     _has_access.return_value = True
 
-    s3fs, adls, hdfs, abfs, gs = MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
-    proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs)}, 'hdfs')
+    s3fs, adls, hdfs, abfs, gs, ofs = MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
+    proxy_fs = ProxyFS({
+      's3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs), 'ofs': wrapper(ofs)},
+      'hdfs')
     proxy_fs.setuser(user)
 
     proxy_fs.isdir('s3a://bucket/key')
@@ -66,6 +68,10 @@ def test_fs_selection():
     gs.isdir.assert_called_once_with('gs://net/key')
     assert_false(hdfs.isdir.called)
 
+    proxy_fs.isdir('ofs://volume/bucket/key')
+    ofs.isdir.assert_called_once_with('ofs://volume/bucket/key')
+    assert_false(hdfs.isdir.called)
+
     assert_raises(IOError, proxy_fs.stats, 'ftp://host')
 
 def wrapper(mock):
@@ -81,8 +87,10 @@ def test_multi_fs_selection():
   with patch('desktop.lib.fs.ProxyFS._has_access') as _has_access:
     _has_access.return_value = True
 
-    s3fs, adls, hdfs, abfs, gs = MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
-    proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs)}, 'hdfs')
+    s3fs, adls, hdfs, abfs, gs, ofs = MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
+    proxy_fs = ProxyFS({
+      's3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs), 'ofs': wrapper(ofs)},
+      'hdfs')
     proxy_fs.setuser(user)
 
     proxy_fs.copy('s3a://bucket1/key', 's3a://bucket2/key')
@@ -109,8 +117,12 @@ def test_multi_fs_selection():
     gs.copyfile.assert_called_once_with('gs://bucket/key', 'key2')
     assert_false(hdfs.copyfile.called)
 
-    # Will be addressed in HUE-2934
-    assert_raises(NotImplementedError, proxy_fs.copy_remote_dir, 's3a://bucket/key', 'adl://tmp/dir') # Exception can only be thrown if scheme is specified, else default to 1st scheme
+    proxy_fs.copyfile('ofs://volume/bucket/key', 'key2')
+    ofs.copyfile.assert_called_once_with('ofs://volume/bucket/key', 'key2')
+    assert_false(hdfs.copyfile.called)
+
+    # Exception can only be thrown if scheme is specified, else default to 1st scheme
+    assert_raises(NotImplementedError, proxy_fs.copy_remote_dir, 's3a://bucket/key', 'adl://tmp/dir')
 
 
 def test_constructor_given_invalid_arguments():
@@ -136,8 +148,12 @@ class TestFsPermissions(object):
     user_client = make_logged_in_client(username='test', groupname='default', recreate=True, is_superuser=False)
     user = User.objects.get(username='test')
 
-    s3fs, adls, hdfs, abfs, gs = MockFs("s3_access"), MockFs("adls_access"), MockFs(), MockFs("abfs_access"), MockFs("gs_access")
-    proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs)}, 'hdfs')
+    s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
+    abfs, gs, ofs = MockFs("abfs_access"), MockFs("gs_access"), MockFs("ofs_access")
+
+    proxy_fs = ProxyFS({
+      's3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs), 'ofs': wrapper(ofs)},
+      'hdfs')
     proxy_fs.setuser(user)
 
     f = proxy_fs._get_fs
@@ -146,6 +162,7 @@ class TestFsPermissions(object):
     remove_from_group(user.username, 'has_adls')
     remove_from_group(user.username, 'has_abfs')
     remove_from_group(user.username, 'has_gs')
+    remove_from_group(user.username, 'has_ofs')
 
     # No perms by default
     assert_raises(Exception, f, 's3a://bucket')
@@ -154,6 +171,7 @@ class TestFsPermissions(object):
     assert_raises(Exception, f, 'adl:/key')
     assert_raises(Exception, f, 'abfs:/key')
     assert_raises(Exception, f, 'gs://bucket/key')
+    assert_raises(Exception, f, 'ofs://volume/bucket/key')
     f('hdfs://path')
     f('/tmp')
 
@@ -163,6 +181,7 @@ class TestFsPermissions(object):
       add_permission('test', 'has_adls', permname='adls_access', appname='filebrowser')
       add_permission('test', 'has_abfs', permname='abfs_access', appname='filebrowser')
       add_permission('test', 'has_gs', permname='gs_access', appname='filebrowser')
+      add_permission('test', 'has_ofs', permname='ofs_access', appname='filebrowser')
 
       f('s3a://bucket')
       f('S3A://bucket/key')
@@ -172,18 +191,24 @@ class TestFsPermissions(object):
       f('hdfs://path')
       f('/tmp')
       f('gs://bucket')
+      f('ofs://volume/bucket/key')
     finally:
       remove_from_group(user.username, 'has_s3')
       remove_from_group(user.username, 'has_adls')
       remove_from_group(user.username, 'has_abfs')
       remove_from_group(user.username, 'has_gs')
+      remove_from_group(user.username, 'has_ofs')
 
   def test_fs_permissions_admin_user(self):
     user_client = make_logged_in_client(username='admin', groupname='default', recreate=True, is_superuser=True)
     user = User.objects.get(username='admin')
 
-    s3fs, adls, hdfs, abfs, gs = MockFs("s3_access"), MockFs("adls_access"), MockFs(), MockFs("abfs_access"), MockFs("gs_access")
-    proxy_fs = ProxyFS({'s3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs)}, 'hdfs')
+    s3fs, adls, hdfs = MockFs("s3_access"), MockFs("adls_access"), MockFs()
+    abfs, gs, ofs = MockFs("abfs_access"), MockFs("gs_access"), MockFs("ofs_access")
+
+    proxy_fs = ProxyFS({
+      's3a': wrapper(s3fs), 'hdfs': wrapper(hdfs), 'adl': wrapper(adls), 'abfs': wrapper(abfs), 'gs': wrapper(gs), 'ofs': wrapper(ofs)},
+      'hdfs')
     proxy_fs.setuser(user)
 
     f = proxy_fs._get_fs
@@ -196,3 +221,5 @@ class TestFsPermissions(object):
     f('hdfs://path')
     f('/tmp')
     f('gs://bucket/key')
+    f('ofs://volume/bucket/key')
+