浏览代码

HUE-9455 [filebrowser] part-1 File system user home directory is wrong in S3 only configuration

ayush.goyal 5 年之前
父节点
当前提交
5e83f68915

+ 9 - 3
apps/filebrowser/src/filebrowser/conf.py

@@ -61,7 +61,13 @@ ENABLE_EXTRACT_UPLOADED_ARCHIVE = Config(
 
 REDIRECT_DOWNLOAD = Config(
   key="redirect_download",
-  help=_(
-    'Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.'),
+  help=_("Redirect client to WebHdfs or S3 for file download. Note: Turning this on will "\
+    "override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3."),
   type=coerce_bool,
-  default=False)
+  default=False)
+
+REMOTE_STORAGE_HOME = Config(
+  key="remote_storage_home",
+  type=str,
+  default=None,
+  help="Optionally set this if you want a different home directory path. e.g. s3a://gethue.")

+ 6 - 4
apps/useradmin/src/useradmin/models.py

@@ -33,9 +33,9 @@ check a permission. Thirdly, you may wish to do so manually, by using something
 Permissions may be granted to groups, but not, currently, to users. A user's abilities is the union of all permissions the group
 has access to.
 
-Note that Django itself has a notion of users, groups, and permissions. We re-use Django's notion of users and groups, but ignore its notion of
-permissions. The permissions notion in Django is strongly tied to what models you may or may not edit, and there are elaborations to
-manipulate this row by row. This does not map nicely onto actions which may not relate to database models.
+Note that Django itself has a notion of users, groups, and permissions. We re-use Django's notion of users and groups, but ignore its
+notion of permissions. The permissions notion in Django is strongly tied to what models you may or may not edit, and there are
+elaborations to manipulate this row by row. This does not map nicely onto actions which may not relate to database models.
 """
 import collections
 import json
@@ -58,6 +58,8 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.idbroker.conf import is_idbroker_enabled
 from desktop.monkey_patches import monkey_patch_username_validator
 
+from filebrowser.conf import REMOTE_STORAGE_HOME
+
 from useradmin.conf import DEFAULT_USER_GROUP
 from useradmin.permissions import HuePermission, GroupPermission, LdapGroup
 
@@ -173,7 +175,7 @@ def create_profile_for_user(user):
   p = UserProfile()
   p.user = user
   p.last_activity = dtz.now()
-  p.home_directory = "/user/%s" % p.user.username
+  p.home_directory = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else  "/user/%s" % p.user.username
   try:
     p.save()
     return p

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1481,6 +1481,9 @@
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   ## redirect_download=false
 
+  # Optionally set this if you want a different home directory path. e.g. s3a://gethue.
+  ## remote_storage_home=s3a://gethue
+
 ###########################################################################
 # Settings to configure Pig
 ###########################################################################

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1466,6 +1466,9 @@
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   ## redirect_download=false
 
+  # Optionally set this if you want a different home directory path. e.g. s3a://gethue.
+  ## remote_storage_home=s3a://gethue 
+
 
 ###########################################################################
 # Settings to configure Pig

+ 25 - 14
desktop/core/src/desktop/models.py

@@ -60,6 +60,8 @@ from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
 from desktop.redaction import global_redaction_engine
 from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
 
+from filebrowser.conf import REMOTE_STORAGE_HOME
+
 if sys.version_info[0] > 2:
   from urllib.parse import quote as urllib_quote
 else:
@@ -499,8 +501,10 @@ class DocumentManager(models.Manager):
                   owner = install_sample_user()
                 else:
                   owner = dashboard.owner
-                dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
-                Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
+                dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard',
+                                                         owner=owner, description=dashboard.label, data=dashboard.properties)
+                Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label,
+                                      extra='search-dashboard')
                 dashboard.save()
     except Exception as e:
       LOG.exception('error syncing search')
@@ -630,7 +634,8 @@ class DocumentManager(models.Manager):
 
 class Document(models.Model):
 
-  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
+  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'),
+                            help_text=_t('User who can own the job.'), related_name='doc_owner')
   name = models.CharField(default='', max_length=255)
   description = models.TextField(default='')
 
@@ -967,7 +972,8 @@ class Document2QueryMixin(object):
       documents = documents.filter(type__in=types)
 
     if search_text:
-      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) | Q(search__icontains=search_text))
+      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) |
+                                   Q(search__icontains=search_text))
 
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
       documents = documents.order_by(order_by)
@@ -976,7 +982,7 @@ class Document2QueryMixin(object):
 
 
 class Document2QuerySet(QuerySet, Document2QueryMixin):
-    pass
+  pass
 
 
 class Document2Manager(models.Manager, Document2QueryMixin):
@@ -1605,8 +1611,8 @@ class Directory(Document2):
     # Get documents that are direct children, or shared with but not owned by the current user
     documents = Document2.objects.filter(
         Q(parent_directory=self) |
-        ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
-          ~Q(owner=user) )
+        ((Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
+          ~Q(owner=user))
       )
 
     documents = documents.exclude(is_history=True).exclude(is_managed=True)
@@ -1759,7 +1765,8 @@ class ClusterConfig(object):
         default_interpreter = []
         default_app = apps[user_default_app['app']]
         if default_app.get('interpreters'):
-          interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
+          interpreters = [interpreter for interpreter in default_app['interpreters']
+                          if interpreter['type'] == user_default_app['interpreter']]
           if interpreters:
             default_interpreter = interpreters
     except UserPreferences.DoesNotExist:
@@ -1909,6 +1916,7 @@ class ClusterConfig(object):
       hdfs_connectors.append(_('Files'))
 
     for hdfs_connector in hdfs_connectors:
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else self.user.get_home_directory().encode('utf-8')
       interpreters.append({
         'type': 'hdfs',
         'displayName': hdfs_connector,
@@ -1916,37 +1924,39 @@ class ClusterConfig(object):
         'tooltip': hdfs_connector,
         'page': '/filebrowser/' + (
           not self.user.is_anonymous() and
-          'view=' + urllib_quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
+          'view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
         )
       })
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else 'S3A://'.encode('utf-8')
       interpreters.append({
         'type': 's3',
         'displayName': _('S3'),
         'buttonName': _('Browse'),
         'tooltip': _('S3'),
-        'page': '/filebrowser/view=' + urllib_quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else 'adl:/'.encode('utf-8')
       interpreters.append({
         'type': 'adls',
         'displayName': _('ADLS'),
         'buttonName': _('Browse'),
         'tooltip': _('ADLS'),
-        'page': '/filebrowser/view=' + urllib_quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
       from azure.abfs.__init__ import get_home_dir_for_ABFS
-
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else get_home_dir_for_ABFS().encode('utf-8')
       interpreters.append({
         'type': 'abfs',
         'displayName': _('ABFS'),
         'buttonName': _('Browse'),
         'tooltip': _('ABFS'),
-        'page': '/filebrowser/view=' + urllib_quote(get_home_dir_for_ABFS().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
     if 'metastore' in self.apps:
@@ -2112,7 +2122,8 @@ class ClusterConfig(object):
       return None
 
   def get_hive_metastore_interpreters(self):
-    return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
+    return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user)
+            if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
 
 
 class Cluster(object):