소스 검색

[raz] Do not enable S3 in Azure env (#2489)

- Take into account `fs.s3a.bucket..` when enabling S3.
- Change get_raz_default_endpoint -> get_raz_s3_default_bucket.
- Fix pylint long line issues.
Harsh Gupta 4 년 전
부모
커밋
ad8358d427

+ 7 - 6
desktop/core/src/desktop/models.py

@@ -40,7 +40,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.urls import reverse, NoReverseMatch
 from django.urls import reverse, NoReverseMatch
 
 
 from dashboard.conf import get_engines, HAS_REPORT_ENABLED, IS_ENABLED as DASHBOARD_ENABLED
 from dashboard.conf import get_engines, HAS_REPORT_ENABLED, IS_ENABLED as DASHBOARD_ENABLED
-from hadoop.core_site import get_raz_api_url, get_raz_default_endpoint
+from hadoop.core_site import get_raz_api_url, get_raz_s3_default_bucket
 from kafka.conf import has_kafka
 from kafka.conf import has_kafka
 from indexer.conf import ENABLE_DIRECT_UPLOAD
 from indexer.conf import ENABLE_DIRECT_UPLOAD
 from metadata.conf import get_optimizer_mode
 from metadata.conf import get_optimizer_mode
@@ -926,7 +926,8 @@ class FilesystemException(Exception):
 
 
 class Document2QueryMixin(object):
 class Document2QueryMixin(object):
 
 
-  def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False, include_shared_links=False, allow_distinct=True):
+  def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False,
+                include_shared_links=False, allow_distinct=True):
     """
     """
     Returns all documents that are owned or shared with the user.
     Returns all documents that are owned or shared with the user.
     :param perms: both, shared, owned. Defaults to both.
     :param perms: both, shared, owned. Defaults to both.
@@ -1729,8 +1730,8 @@ def get_remote_home_storage():
   remote_home_storage = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else None
   remote_home_storage = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else None
 
 
   if not remote_home_storage:
   if not remote_home_storage:
-    if get_raz_api_url() and get_raz_default_endpoint():
-      remote_home_storage = 's3a://%(bucket)s' % get_raz_default_endpoint()
+    if get_raz_api_url() and get_raz_s3_default_bucket():
+      remote_home_storage = 's3a://%(bucket)s' % get_raz_s3_default_bucket()
 
 
   return remote_home_storage
   return remote_home_storage
 
 
@@ -1849,7 +1850,7 @@ class ClusterConfig(object):
           'name': interpreter['name'],
           'name': interpreter['name'],
           'type': interpreter['type'],  # Connector v1
           'type': interpreter['type'],  # Connector v1
           'id': interpreter['type'],
           'id': interpreter['type'],
-          'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and interpreter['dialect'] == 'hive' else  interpreter['name'],
+        'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and interpreter['dialect'] == 'hive' else interpreter['name'],
           'buttonName': _('Query'),
           'buttonName': _('Query'),
           'tooltip': _('%s Query') % interpreter['type'].title(),
           'tooltip': _('%s Query') % interpreter['type'].title(),
           'optimizer': get_optimizer_mode(),
           'optimizer': get_optimizer_mode(),
@@ -1905,7 +1906,7 @@ class ClusterConfig(object):
           'name': interpreter['name'],
           'name': interpreter['name'],
           'type': interpreter['type'],
           'type': interpreter['type'],
           'id': interpreter['type'],
           'id': interpreter['type'],
-          'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and interpreter['dialect'] == 'hive' else  interpreter['name'],
+        'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and interpreter['dialect'] == 'hive' else interpreter['name'],
           'buttonName': _('Query'),
           'buttonName': _('Query'),
           'tooltip': _('%s Query') % interpreter['type'].title(),
           'tooltip': _('%s Query') % interpreter['type'].title(),
           'page': '/editor/?type=%(type)s' % interpreter,
           'page': '/editor/?type=%(type)s' % interpreter,

+ 9 - 5
desktop/libs/aws/src/aws/conf.py

@@ -24,7 +24,7 @@ import requests
 
 
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool, coerce_password_from_script
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool, coerce_password_from_script
 from desktop.lib.idbroker import conf as conf_idbroker
 from desktop.lib.idbroker import conf as conf_idbroker
-from hadoop.core_site import get_s3a_access_key, get_s3a_secret_key, get_s3a_session_token, get_raz_api_url, get_raz_default_endpoint
+from hadoop.core_site import get_s3a_access_key, get_s3a_secret_key, get_s3a_session_token, get_raz_api_url, get_raz_s3_default_bucket
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   from django.utils.translation import gettext_lazy as _, gettext as _t
   from django.utils.translation import gettext_lazy as _, gettext as _t
@@ -109,7 +109,7 @@ def get_default_host():
   '''Returns the S3 host when Raz is configured'''
   '''Returns the S3 host when Raz is configured'''
 
 
   if get_raz_api_url():
   if get_raz_api_url():
-    endpoint = get_raz_default_endpoint()
+    endpoint = get_raz_s3_default_bucket()
     if endpoint:
     if endpoint:
       return endpoint.get('host')
       return endpoint.get('host')
 
 
@@ -287,8 +287,11 @@ def is_enabled():
   return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or \
   return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or \
       has_iam_metadata() or \
       has_iam_metadata() or \
       conf_idbroker.is_idbroker_enabled('s3a') or \
       conf_idbroker.is_idbroker_enabled('s3a') or \
-      (RAZ.IS_ENABLED.get() and 'default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw()) or \
-      IS_SELF_SIGNING_ENABLED.get()
+      IS_SELF_SIGNING_ENABLED.get() or \
+      (
+        RAZ.IS_ENABLED.get() and bool(get_raz_s3_default_bucket()) and \
+        'default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw()
+      )
 
 
 
 
 def is_ec2_instance():
 def is_ec2_instance():
@@ -349,7 +352,8 @@ def has_s3_access(user):
   from desktop.conf import RAZ  # Must be imported dynamically in order to have proper value
   from desktop.conf import RAZ  # Must be imported dynamically in order to have proper value
 
 
   return user.is_authenticated and user.is_active and (
   return user.is_authenticated and user.is_active and (
-    is_admin(user) or user.has_hue_permission(action="s3_access", app="filebrowser") or RAZ.IS_ENABLED.get()
+    is_admin(user) or user.has_hue_permission(action="s3_access", app="filebrowser") or \
+    (RAZ.IS_ENABLED.get() and bool(get_raz_s3_default_bucket()))
   )
   )
 
 
 
 

+ 1 - 3
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -140,11 +140,9 @@ def get_raz_cluster_name():
   """
   """
   return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '') or get_conf().get(_CNF_ADLS_RAZ_CLUSTER_NAME, '')
   return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '') or get_conf().get(_CNF_ADLS_RAZ_CLUSTER_NAME, '')
 
 
-def get_raz_default_endpoint():
+def get_raz_s3_default_bucket():
   """
   """
   Get the name of the default S3 bucket of Raz
   Get the name of the default S3 bucket of Raz
-
-  S3 only. Add check for Azure when supported.
   """
   """
 
 
   for key, val in get_conf().items():
   for key, val in get_conf().items():

+ 1 - 1
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -92,7 +92,7 @@ def test_core_site():
 
 
     assert_equal(core_site.get_raz_api_url(), 'https://gehue-adls-master:6082/')
     assert_equal(core_site.get_raz_api_url(), 'https://gehue-adls-master:6082/')
     assert_equal(core_site.get_raz_cluster_name(), 'gehue-adls')
     assert_equal(core_site.get_raz_cluster_name(), 'gehue-adls')
-    assert_equal(core_site.get_raz_default_endpoint(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'gethue-dev'})
+    assert_equal(core_site.get_raz_s3_default_bucket(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'gethue-dev'})
 
 
     assert_equal(core_site.get_default_fs(), 'abfs://data@gethuedevstorage.dfs.core.windows.net/hue-adls')
     assert_equal(core_site.get_default_fs(), 'abfs://data@gethuedevstorage.dfs.core.windows.net/hue-adls')