浏览代码

HUE-9455 [filebrowser] part-1 File system user home directory is wrong in S3 only configuration

ayush.goyal 5 年之前
父节点
当前提交
5e83f68915

+ 9 - 3
apps/filebrowser/src/filebrowser/conf.py

@@ -61,7 +61,13 @@ ENABLE_EXTRACT_UPLOADED_ARCHIVE = Config(
 
 
 REDIRECT_DOWNLOAD = Config(
 REDIRECT_DOWNLOAD = Config(
   key="redirect_download",
   key="redirect_download",
-  help=_(
-    'Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.'),
+  help=_("Redirect client to WebHdfs or S3 for file download. Note: Turning this on will "\
+    "override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3."),
   type=coerce_bool,
   type=coerce_bool,
-  default=False)
+  default=False)
+
+REMOTE_STORAGE_HOME = Config(
+  key="remote_storage_home",
+  type=str,
+  default=None,
+  help="Optionally set this if you want a different home directory path. e.g. s3a://gethue.")

+ 6 - 4
apps/useradmin/src/useradmin/models.py

@@ -33,9 +33,9 @@ check a permission. Thirdly, you may wish to do so manually, by using something
 Permissions may be granted to groups, but not, currently, to users. A user's abilities is the union of all permissions the group
 Permissions may be granted to groups, but not, currently, to users. A user's abilities is the union of all permissions the group
 has access to.
 has access to.
 
 
-Note that Django itself has a notion of users, groups, and permissions. We re-use Django's notion of users and groups, but ignore its notion of
-permissions. The permissions notion in Django is strongly tied to what models you may or may not edit, and there are elaborations to
-manipulate this row by row. This does not map nicely onto actions which may not relate to database models.
+Note that Django itself has a notion of users, groups, and permissions. We re-use Django's notion of users and groups, but ignore its
+notion of permissions. The permissions notion in Django is strongly tied to what models you may or may not edit, and there are
+elaborations to manipulate this row by row. This does not map nicely onto actions which may not relate to database models.
 """
 """
 import collections
 import collections
 import json
 import json
@@ -58,6 +58,8 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.idbroker.conf import is_idbroker_enabled
 from desktop.lib.idbroker.conf import is_idbroker_enabled
 from desktop.monkey_patches import monkey_patch_username_validator
 from desktop.monkey_patches import monkey_patch_username_validator
 
 
+from filebrowser.conf import REMOTE_STORAGE_HOME
+
 from useradmin.conf import DEFAULT_USER_GROUP
 from useradmin.conf import DEFAULT_USER_GROUP
 from useradmin.permissions import HuePermission, GroupPermission, LdapGroup
 from useradmin.permissions import HuePermission, GroupPermission, LdapGroup
 
 
@@ -173,7 +175,7 @@ def create_profile_for_user(user):
   p = UserProfile()
   p = UserProfile()
   p.user = user
   p.user = user
   p.last_activity = dtz.now()
   p.last_activity = dtz.now()
-  p.home_directory = "/user/%s" % p.user.username
+  p.home_directory = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else  "/user/%s" % p.user.username
   try:
   try:
     p.save()
     p.save()
     return p
     return p

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1481,6 +1481,9 @@
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   ## redirect_download=false
   ## redirect_download=false
 
 
+  # Optionally set this if you want a different home directory path. e.g. s3a://gethue.
+  ## remote_storage_home=s3a://gethue
+
 ###########################################################################
 ###########################################################################
 # Settings to configure Pig
 # Settings to configure Pig
 ###########################################################################
 ###########################################################################

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1466,6 +1466,9 @@
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   # Redirect client to WebHdfs or S3 for file download. Note: Turning this on will override notebook/redirect_whitelist for user selected file downloads on WebHdfs & S3.
   ## redirect_download=false
   ## redirect_download=false
 
 
+  # Optionally set this if you want a different home directory path. e.g. s3a://gethue.
+  ## remote_storage_home=s3a://gethue 
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Pig
 # Settings to configure Pig

+ 25 - 14
desktop/core/src/desktop/models.py

@@ -60,6 +60,8 @@ from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
 from desktop.redaction import global_redaction_engine
 from desktop.redaction import global_redaction_engine
 from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
 from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
 
 
+from filebrowser.conf import REMOTE_STORAGE_HOME
+
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   from urllib.parse import quote as urllib_quote
   from urllib.parse import quote as urllib_quote
 else:
 else:
@@ -499,8 +501,10 @@ class DocumentManager(models.Manager):
                   owner = install_sample_user()
                   owner = install_sample_user()
                 else:
                 else:
                   owner = dashboard.owner
                   owner = dashboard.owner
-                dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
-                Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
+                dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard',
+                                                         owner=owner, description=dashboard.label, data=dashboard.properties)
+                Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label,
+                                      extra='search-dashboard')
                 dashboard.save()
                 dashboard.save()
     except Exception as e:
     except Exception as e:
       LOG.exception('error syncing search')
       LOG.exception('error syncing search')
@@ -630,7 +634,8 @@ class DocumentManager(models.Manager):
 
 
 class Document(models.Model):
 class Document(models.Model):
 
 
-  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
+  owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'),
+                            help_text=_t('User who can own the job.'), related_name='doc_owner')
   name = models.CharField(default='', max_length=255)
   name = models.CharField(default='', max_length=255)
   description = models.TextField(default='')
   description = models.TextField(default='')
 
 
@@ -967,7 +972,8 @@ class Document2QueryMixin(object):
       documents = documents.filter(type__in=types)
       documents = documents.filter(type__in=types)
 
 
     if search_text:
     if search_text:
-      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) | Q(search__icontains=search_text))
+      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) |
+                                   Q(search__icontains=search_text))
 
 
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
       documents = documents.order_by(order_by)
       documents = documents.order_by(order_by)
@@ -976,7 +982,7 @@ class Document2QueryMixin(object):
 
 
 
 
 class Document2QuerySet(QuerySet, Document2QueryMixin):
 class Document2QuerySet(QuerySet, Document2QueryMixin):
-    pass
+  pass
 
 
 
 
 class Document2Manager(models.Manager, Document2QueryMixin):
 class Document2Manager(models.Manager, Document2QueryMixin):
@@ -1605,8 +1611,8 @@ class Directory(Document2):
     # Get documents that are direct children, or shared with but not owned by the current user
     # Get documents that are direct children, or shared with but not owned by the current user
     documents = Document2.objects.filter(
     documents = Document2.objects.filter(
         Q(parent_directory=self) |
         Q(parent_directory=self) |
-        ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
-          ~Q(owner=user) )
+        ((Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
+          ~Q(owner=user))
       )
       )
 
 
     documents = documents.exclude(is_history=True).exclude(is_managed=True)
     documents = documents.exclude(is_history=True).exclude(is_managed=True)
@@ -1759,7 +1765,8 @@ class ClusterConfig(object):
         default_interpreter = []
         default_interpreter = []
         default_app = apps[user_default_app['app']]
         default_app = apps[user_default_app['app']]
         if default_app.get('interpreters'):
         if default_app.get('interpreters'):
-          interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
+          interpreters = [interpreter for interpreter in default_app['interpreters']
+                          if interpreter['type'] == user_default_app['interpreter']]
           if interpreters:
           if interpreters:
             default_interpreter = interpreters
             default_interpreter = interpreters
     except UserPreferences.DoesNotExist:
     except UserPreferences.DoesNotExist:
@@ -1909,6 +1916,7 @@ class ClusterConfig(object):
       hdfs_connectors.append(_('Files'))
       hdfs_connectors.append(_('Files'))
 
 
     for hdfs_connector in hdfs_connectors:
     for hdfs_connector in hdfs_connectors:
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else self.user.get_home_directory().encode('utf-8')
       interpreters.append({
       interpreters.append({
         'type': 'hdfs',
         'type': 'hdfs',
         'displayName': hdfs_connector,
         'displayName': hdfs_connector,
@@ -1916,37 +1924,39 @@ class ClusterConfig(object):
         'tooltip': hdfs_connector,
         'tooltip': hdfs_connector,
         'page': '/filebrowser/' + (
         'page': '/filebrowser/' + (
           not self.user.is_anonymous() and
           not self.user.is_anonymous() and
-          'view=' + urllib_quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
+          'view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
         )
         )
       })
       })
 
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else 'S3A://'.encode('utf-8')
       interpreters.append({
       interpreters.append({
         'type': 's3',
         'type': 's3',
         'displayName': _('S3'),
         'displayName': _('S3'),
         'buttonName': _('Browse'),
         'buttonName': _('Browse'),
         'tooltip': _('S3'),
         'tooltip': _('S3'),
-        'page': '/filebrowser/view=' + urllib_quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
       })
 
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else 'adl:/'.encode('utf-8')
       interpreters.append({
       interpreters.append({
         'type': 'adls',
         'type': 'adls',
         'displayName': _('ADLS'),
         'displayName': _('ADLS'),
         'buttonName': _('Browse'),
         'buttonName': _('Browse'),
         'tooltip': _('ADLS'),
         'tooltip': _('ADLS'),
-        'page': '/filebrowser/view=' + urllib_quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
       })
 
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
       from azure.abfs.__init__ import get_home_dir_for_ABFS
       from azure.abfs.__init__ import get_home_dir_for_ABFS
-
+      home_path = REMOTE_STORAGE_HOME.get() if REMOTE_STORAGE_HOME.get() else get_home_dir_for_ABFS().encode('utf-8')
       interpreters.append({
       interpreters.append({
         'type': 'abfs',
         'type': 'abfs',
         'displayName': _('ABFS'),
         'displayName': _('ABFS'),
         'buttonName': _('Browse'),
         'buttonName': _('Browse'),
         'tooltip': _('ABFS'),
         'tooltip': _('ABFS'),
-        'page': '/filebrowser/view=' + urllib_quote(get_home_dir_for_ABFS().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
       })
 
 
     if 'metastore' in self.apps:
     if 'metastore' in self.apps:
@@ -2112,7 +2122,8 @@ class ClusterConfig(object):
       return None
       return None
 
 
   def get_hive_metastore_interpreters(self):
   def get_hive_metastore_interpreters(self):
-    return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
+    return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user)
+            if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
 
 
 
 
 class Cluster(object):
 class Cluster(object):