Преглед изворни кода

[ozone] Connect ofs like other filesystems

- Add ofs as a separate fs in proxyfs
- Add ozone to user and group permissions
- Update APIs with ofs
- Default users to ofs root directory
Harshg999 пре 2 година
родитељ
комит
f050809f66

+ 3 - 0
apps/filebrowser/src/filebrowser/api.py

@@ -20,6 +20,7 @@ import logging
 from desktop.lib.django_util import JsonResponse
 from desktop.lib import fsmanager
 from desktop.lib.i18n import smart_unicode
+from desktop.lib.fs.ozone.ofs import get_ofs_home_directory
 
 from azure.abfs.__init__ import get_home_dir_for_abfs
 from aws.s3.s3fs import get_s3_home_directory
@@ -67,6 +68,8 @@ def get_filesystems_with_home_dirs(request): # Using as a public API only for no
       user_home_dir = get_s3_home_directory(request.user)
     elif fs == 'abfs':
       user_home_dir = get_home_dir_for_abfs(request.user)
+    elif fs == 'ofs':
+      user_home_dir = get_ofs_home_directory()
 
     filesystems.append({
       'file_system': fs,

+ 3 - 0
apps/filebrowser/src/filebrowser/forms.py

@@ -29,6 +29,7 @@ from django.forms.formsets import formset_factory, BaseFormSet
 
 from aws.s3 import S3A_ROOT, normpath as s3_normpath
 from azure.abfs.__init__ import ABFS_ROOT, normpath as abfs_normpath
+from desktop.lib.fs.ozone import OFS_ROOT, normpath as ofs_normpath
 from desktop.lib import i18n
 from hadoop.fs import normpath
 from useradmin.models import User, Group
@@ -80,6 +81,8 @@ class PathField(CharField):
       cleaned_path = s3_normpath(cleaned_path)
     elif value.lower().startswith(ABFS_ROOT):
       cleaned_path = abfs_normpath(cleaned_path)
+    elif value.lower().startswith(OFS_ROOT):
+      cleaned_path = ofs_normpath(cleaned_path)
     else:
       cleaned_path = normpath(cleaned_path)
     return cleaned_path

+ 3 - 2
apps/filebrowser/src/filebrowser/settings.py

@@ -24,12 +24,13 @@ IS_URL_NAMESPACED = True
 
 from aws.conf import PERMISSION_ACTION_S3
 from azure.conf import PERMISSION_ACTION_ADLS, PERMISSION_ACTION_ABFS
-from desktop.conf import PERMISSION_ACTION_GS
+from desktop.conf import PERMISSION_ACTION_GS, PERMISSION_ACTION_OFS
 
 
 PERMISSION_ACTIONS = (
   (PERMISSION_ACTION_S3, "Access to S3 from filebrowser and filepicker."),
   (PERMISSION_ACTION_ADLS, "Access to ADLS from filebrowser and filepicker."),
   (PERMISSION_ACTION_ABFS, "Access to ABFS from filebrowser and filepicker."),
-  (PERMISSION_ACTION_GS, "Access to GS from filebrowser and filepicker.")
+  (PERMISSION_ACTION_GS, "Access to GS from filebrowser and filepicker."),
+  (PERMISSION_ACTION_OFS, "Access to OFS from filebrowser and filepicker.")
 )

+ 14 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -56,6 +56,7 @@ from desktop.lib.django_util import JsonResponse
 from desktop.lib.export_csvxls import file_reader
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.fs import splitpath
+from desktop.lib.fs.ozone.ofs import get_ofs_home_directory
 from desktop.lib.i18n import smart_str
 from desktop.lib.paths import SAFE_CHARACTERS_URI, SAFE_CHARACTERS_URI_COMPONENTS
 from desktop.lib.tasks.compress_files.compress_utils import compress_files_in_hdfs
@@ -146,7 +147,8 @@ def _decode_slashes(path):
   # This is a fix for some installations where the path is still having the slash (/) encoded
   # as %2F while the rest of the path is actually decoded. 
   encoded_slash = '%2F'
-  if path.startswith(encoded_slash) or path.startswith('abfs:' + encoded_slash) or path.startswith('s3a:' + encoded_slash):
+  if path.startswith(encoded_slash) or path.startswith('abfs:' + encoded_slash) or \
+    path.startswith('s3a:' + encoded_slash) or path.startswith('ofs:' + encoded_slash):
     path = path.replace(encoded_slash, '/')
 
   return path
@@ -159,6 +161,8 @@ def _normalize_path(path):
     path = path.replace('abfs:/', 'abfs://')
   if path.startswith('s3a:/') and not path.startswith('s3a://'):
     path = path.replace('s3a:/', 's3a://')
+  if path.startswith('ofs:/') and not path.startswith('ofs://'):
+    path = path.replace('ofs:/', 'ofs://')
 
   return path
 
@@ -246,6 +250,15 @@ def view(request, path):
           '/filebrowser/view=' + urllib_quote(home_dir_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
       )
 
+  # default_ofs_home is set in jquery.filechooser.js
+  if 'default_ofs_home' in request.GET:
+    home_dir_path = get_ofs_home_directory()
+    if request.fs.isdir(home_dir_path):
+      return format_preserving_redirect(
+          request,
+          '/filebrowser/view=' + urllib_quote(home_dir_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+      )
+
   # default_to_home is set in jquery.filechooser.js
   if 'default_to_home' in request.GET:
     home_dir_path = request.user.get_home_directory()

+ 1 - 0
apps/useradmin/src/useradmin/models.py

@@ -297,6 +297,7 @@ def update_app_permissions(**kwargs):
             not (new_dp.app == 'filebrowser' and new_dp.action == 'gs_access' and not is_idbroker_enabled('gs')) and \
             not (new_dp.app == 'filebrowser' and new_dp.action == 'adls_access') and \
             not (new_dp.app == 'filebrowser' and new_dp.action == 'abfs_access') and \
+            not (new_dp.app == 'filebrowser' and new_dp.action == 'ofs_access') and \
             not (new_dp.app == 'oozie' and new_dp.action == 'disable_editor_access'):
           GroupPermission.objects.create(group=default_group, hue_permission=new_dp)
 

+ 11 - 6
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -26,7 +26,8 @@ from crequest.middleware import CrequestMiddleware
 from useradmin.models import User
 
 from desktop.auth.backend import is_admin
-from desktop.conf import DEFAULT_USER, ENABLE_ORGANIZATIONS
+from desktop.conf import DEFAULT_USER, ENABLE_ORGANIZATIONS, is_ofs_enabled
+from desktop.lib.fs.ozone import OFS_ROOT
 
 from aws.conf import is_raz_s3
 from aws.s3.s3fs import get_s3_home_directory
@@ -205,21 +206,25 @@ class ProxyFS(object):
   def create(self, path, *args, **kwargs):
     self._get_fs(path).create(path, *args, **kwargs)
 
+  def get_content_summary(self, path):
+    return self._get_fs(path).get_content_summary(path)
+
   def create_home_dir(self, home_path=None):
     """
-    Initially home_path will have path value for HDFS and if it is configured in Hue, try creating the user home dir for it first.
-    Then we check if S3/ABFS is configured in Hue via RAZ. If yes, try creating user home dir for them next.
+    Initially home_path will have path value for HDFS, try creating the user home dir for it first.
+    Then, we check if S3/ABFS is configured via RAZ. If yes, try creating user home dir for them next.
     """
     from desktop.conf import RAZ # Imported dynamically in order to have proper value.
 
-    if home_path is None:
-      home_path = self.get_home_dir()
-
     try:
       self._get_fs(home_path).create_home_dir(home_path)
     except Exception as e:
       LOG.debug('Error creating HDFS home directory for path %s : %s' % (home_path, str(e)))
 
+    # All users will have access to Ozone root.
+    if is_ofs_enabled():
+      LOG.debug('Creation of user home path is not supported in Ozone. Redirect to %s' % OFS_ROOT)
+
     # Get the new home_path for S3/ABFS when RAZ is enabled.
     if is_raz_s3():
       home_path = get_s3_home_directory(User.objects.get(username=self.getuser()))

+ 10 - 3
desktop/core/src/desktop/lib/fsmanager.py

@@ -23,12 +23,13 @@ import logging
 import aws.client
 import azure.client
 import desktop.lib.fs.gc.client
+import desktop.lib.fs.ozone.client
 
 from aws.conf import is_enabled as is_s3_enabled, has_s3_access
 from azure.conf import is_adls_enabled, is_abfs_enabled, has_adls_access, has_abfs_access
 
 
-from desktop.conf import is_gs_enabled, has_gs_access, DEFAULT_USER
+from desktop.conf import is_gs_enabled, has_gs_access, DEFAULT_USER, is_ofs_enabled, has_ofs_access
 
 from desktop.lib.fs.proxyfs import ProxyFS
 from desktop.lib.python_util import current_ms_from_utc
@@ -38,7 +39,7 @@ from hadoop.cluster import get_hdfs, _make_filesystem
 from hadoop.conf import has_hdfs_enabled
 
 
-SUPPORTED_FS = ['hdfs', 's3a', 'adl', 'abfs', 'gs']
+SUPPORTED_FS = ['hdfs', 's3a', 'adl', 'abfs', 'gs', 'ofs']
 CLIENT_CACHE = None
 _DEFAULT_USER = DEFAULT_USER.get()
 
@@ -65,6 +66,8 @@ def has_access(fs=None, user=None):
     return has_abfs_access(user)
   elif fs == 'gs':
     return has_gs_access(user)
+  elif fs == 'ofs':
+    return has_ofs_access(user)
 
 
 def is_enabled(fs):
@@ -78,6 +81,8 @@ def is_enabled(fs):
     return is_abfs_enabled()
   elif fs == 'gs':
     return is_gs_enabled()
+  elif fs == 'ofs':
+    return is_ofs_enabled()
 
 
 def is_enabled_and_has_access(fs=None, user=None):
@@ -95,13 +100,15 @@ def _make_client(fs, name, user):
     return azure.client._make_abfs_client(name, user)
   elif fs == 'gs':
     return desktop.lib.fs.gc.client._make_client(name, user)
+  elif fs == 'ofs':
+    return desktop.lib.fs.ozone.client._make_ofs_client(name, user)
   return None
 
 
 def _get_client(fs=None):
   if fs == 'hdfs':
     return get_hdfs
-  elif fs in ['s3a', 'adl', 'abfs', 'gs']:
+  elif fs in ['s3a', 'adl', 'abfs', 'gs', 'ofs']:
     return partial(_get_client_cached, fs)
   return None
 

+ 11 - 0
desktop/core/src/desktop/models.py

@@ -2033,6 +2033,17 @@ class ClusterConfig(object):
         'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
+    if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('ofs', self.user):
+      from desktop.lib.fs.ozone.ofs import get_ofs_home_directory
+      home_path = get_ofs_home_directory().encode('utf-8')
+      interpreters.append({
+        'type': 'ofs',
+        'displayName': _('Ozone'),
+        'buttonName': _('Browse'),
+        'tooltip': _('Ozone'),
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
+      })
+
     if 'metastore' in self.apps:
       interpreters.append({
         'type': 'tables',

+ 2 - 1
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -766,7 +766,8 @@ class WebHdfs(Hdfs):
     if dir_mode is None:
       dir_mode = self.getDefaultDirPerms()
 
-    self.do_as_user(owner, self.mkdir, destination, mode=dir_mode)
+    if not self.exists(destination):
+      self.do_as_user(owner, self.mkdir, destination, mode=dir_mode)
 
     for stat in self.listdir_stats(source):
       source_file = stat.path