ソースを参照

[raz] Default to accessible path in Importer filechooser

- When trying to import file via Importer and choosing `Remote file` as option, the filechooser used to give path error in RAZ enabled env.
Harshg999 4 年 前
コミット
cefdcd028c

+ 2 - 2
apps/filebrowser/src/filebrowser/views.py

@@ -219,7 +219,7 @@ def view(request, path):
   # default_abfs_home is set in jquery.filechooser.js
   if 'default_abfs_home' in request.GET:
     from azure.abfs.__init__ import get_home_dir_for_abfs
-    home_dir_path = get_home_dir_for_abfs()
+    home_dir_path = get_home_dir_for_abfs(request.user)
     if request.fs.isdir(home_dir_path):
       return format_preserving_redirect(
           request,
@@ -227,7 +227,7 @@ def view(request, path):
       )
 
   if 'default_s3_home' in request.GET:
-    home_dir_path = get_s3_home_directory()
+    home_dir_path = get_s3_home_directory(request.user)
     if request.fs.isdir(home_dir_path):
       return format_preserving_redirect(
           request,

+ 60 - 9
apps/filebrowser/src/filebrowser/views_test.py

@@ -50,7 +50,7 @@ from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group, add_permission, remove_from_group
 from desktop.lib.view_util import location_to_url
-from desktop.conf import is_oozie_enabled
+from desktop.conf import is_oozie_enabled, RAZ
 from hadoop import pseudo_hdfs4
 from hadoop.conf import UPLOAD_CHUNK_SIZE
 from hadoop.fs.webhdfs import WebHdfs
@@ -1469,6 +1469,7 @@ class TestADLSAccessPermissions(object):
     finally:
       remove_from_group(self.user.username, 'has_adls')
 
+
 class TestFileChooserRedirect(object):
 
   def setUp(self):
@@ -1484,38 +1485,88 @@ class TestFileChooserRedirect(object):
 
       # HDFS - default_to_home
       response = self.client.get('/filebrowser/view=%2F?default_to_home')
-      LOG.info("Response: %s" % response.status_code)
+
       assert_equal(302, response.status_code)
       assert_equal('/filebrowser/view=%2Fuser%2Ftest', response.url)
 
       # ABFS - default_abfs_home
       response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
-      LOG.info("Response: %s" % response.status_code)
+
       assert_equal(302, response.status_code)
       assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
 
-      reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(
-                  'abfs://data-container@mystorage.dfs.core.windows.net'
-              )
+      reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing('abfs://data-container@mystorage.dfs.core.windows.net')
       try:
         response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
-        LOG.info("Response: %s" % response.status_code)
+
         assert_equal(302, response.status_code)
         assert_equal('/filebrowser/view=abfs%3A%2F%2Fdata-container', response.url)
       finally:
         reset()
 
+      resets = [
+        RAZ.IS_ENABLED.set_for_testing(True),
+        REMOTE_STORAGE_HOME.set_for_testing('abfs://data-container')
+      ]
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=abfs%3A%2F%2Fdata-container', response.url)
+      finally:
+        for reset in resets:
+          reset()
+
+      resets = [
+        RAZ.IS_ENABLED.set_for_testing(True),
+        REMOTE_STORAGE_HOME.set_for_testing('abfs://data-container/user')
+      ]
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=abfs%3A%2F%2Fdata-container%2Fuser%2Ftest', response.url)
+      finally:
+        for reset in resets:
+          reset()
+
       # S3A - default_s3_home
       response = self.client.get('/filebrowser/view=%2F?default_s3_home')
-      LOG.info("Response: %s" % response.status_code)
+
       assert_equal(302, response.status_code)
       assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
 
       reset = REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket')
       try:
         response = self.client.get('/filebrowser/view=%2F?default_s3_home')
-        LOG.info("Response: %s" % response.status_code)
+
         assert_equal(302, response.status_code)
         assert_equal('/filebrowser/view=s3a%3A%2F%2Fmy_bucket', response.url)
       finally:
         reset()
+
+      resets = [
+        RAZ.IS_ENABLED.set_for_testing(True),
+        REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket')
+      ]
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=s3a%3A%2F%2Fmy_bucket', response.url)
+      finally:
+        for reset in resets:
+          reset()
+
+      resets = [
+        RAZ.IS_ENABLED.set_for_testing(True),
+        REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket/user')
+      ]
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=s3a%3A%2F%2Fmy_bucket%2Fuser%2Ftest', response.url)
+      finally:
+        for reset in resets:
+          reset()

+ 7 - 1
desktop/core/src/desktop/models.py

@@ -1733,6 +1733,12 @@ def get_remote_home_storage(user=None):
     if get_raz_api_url() and get_raz_s3_default_bucket():
       remote_home_storage = 's3a://%(bucket)s' % get_raz_s3_default_bucket()
 
+  remote_home_storage = _handle_user_dir_raz(user, remote_home_storage)
+
+  return remote_home_storage
+
+
+def _handle_user_dir_raz(user, remote_home_storage):
   # In RAZ env, apppend username so that it defaults to user's dir and doesn't give 403 error
   if user and remote_home_storage and RAZ.IS_ENABLED.get() and remote_home_storage.endswith('/user'):
     remote_home_storage += '/' + user.username
@@ -2008,7 +2014,7 @@ class ClusterConfig(object):
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
       from azure.abfs.__init__ import get_home_dir_for_abfs
-      home_path = remote_home_storage if remote_home_storage else get_home_dir_for_abfs().encode('utf-8')
+      home_path = remote_home_storage if remote_home_storage else get_home_dir_for_abfs(self.user).encode('utf-8')
       interpreters.append({
         'type': 'abfs',
         'displayName': _('ABFS'),

+ 0 - 1
desktop/core/src/desktop/models_tests.py

@@ -89,7 +89,6 @@ class TestClusterConfig(object):
 
 
   def test_get_remote_storage_home(self):
-
     # When default home ends with /user in RAZ ADLS env.
     resets = [
       RAZ.IS_ENABLED.set_for_testing(True),

+ 8 - 4
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -35,6 +35,7 @@ from aws import s3
 from aws.conf import get_default_region, get_locations, PERMISSION_ACTION_S3
 from aws.s3 import normpath, s3file, translate_s3_error, S3A_ROOT
 from aws.s3.s3stat import S3Stat
+
 from filebrowser.conf import REMOTE_STORAGE_HOME
 
 if sys.version_info[0] > 2:
@@ -83,10 +84,13 @@ def auth_error_handler(view_fn):
   return decorator
 
 
-def get_s3_home_directory():
-  return REMOTE_STORAGE_HOME.get() \
-         if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() \
-         else 's3a://'
+def get_s3_home_directory(user=None):
+  from desktop.models import _handle_user_dir_raz
+
+  remote_home_s3 = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else 's3a://'
+  remote_home_s3 = _handle_user_dir_raz(user, remote_home_s3)
+
+  return remote_home_s3
 
 
 class S3FileSystem(object):

+ 18 - 4
desktop/libs/azure/src/azure/abfs/__init__.py

@@ -27,6 +27,9 @@ from nose.tools import assert_not_equal
 from hadoop.fs import normpath as fs_normpath
 from azure.conf import get_default_abfs_fs
 
+from desktop.conf import RAZ
+from filebrowser.conf import REMOTE_STORAGE_HOME
+
 LOG = logging.getLogger(__name__)
 
 ABFS_PATH_RE = re.compile('^/*[aA][bB][fF][sS]{1,2}://([$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9])(@[^.]*?\.dfs\.core\.windows\.net)?(/(.*?)/?)?$') #check this first for problems
@@ -164,15 +167,26 @@ def abfspath(path, fs_defaultfs = None):
   LOG.debug("%s" % path)
   return path
 
-def get_home_dir_for_abfs():
+
+def get_home_dir_for_abfs(user=None):
   """
-  Attempts to go to the directory set by the user in the configuration file. If not defaults to abfs://
+  Attempts to go to the directory set in the config file or core-site.xml else defaults to abfs://
   """
+  from desktop.models import _handle_user_dir_raz
+
   try:
     filesystem = parse_uri(get_default_abfs_fs())[0]
-    return "abfs://" + filesystem
+    remote_home_abfs = "abfs://" + filesystem
   except:
-    return 'abfs://'
+    remote_home_abfs = 'abfs://'
+
+  # Check from remote_storage_home config only for RAZ env
+  if RAZ.IS_ENABLED.get() and hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get():
+    remote_home_abfs = REMOTE_STORAGE_HOME.get()
+    remote_home_abfs = _handle_user_dir_raz(user, remote_home_abfs)
+
+  return remote_home_abfs
+
 
 def abfsdatetime_to_timestamp(datetime):
   """