Browse Source

[raz] Add series of test to check the auto configuration

Romain Rigaux 4 years ago
parent
commit
0130cd154a

+ 1 - 1
apps/filebrowser/src/filebrowser/conf.py

@@ -75,4 +75,4 @@ REMOTE_STORAGE_HOME = Config(
   key="remote_storage_home",
   type=str,
   default=None,
-  help="Optionally set this if you want a different home directory path. e.g. s3a://gethue.")
+  help="Optionally set this if you want a different home directory path. e.g. s3a://gethue.")

+ 7 - 3
desktop/core/src/desktop/conf.py

@@ -28,7 +28,6 @@ from collections import OrderedDict
 
 from django.db import connection
 
-from hadoop.core_site import get_raz_api_url
 from metadata.metadata_sites import get_navigator_audit_log_dir, get_navigator_audit_max_file_size
 
 from desktop import appmanager
@@ -2093,6 +2092,11 @@ CONNECTORS = UnspecifiedConfigSection(
   )
 )
 
+def has_raz_url():
+  """Check if we can guess if Raz is configured"""
+  from hadoop.core_site import get_raz_api_url  # Avoid circular import
+  return get_raz_api_url()
+
 
 RAZ = ConfigSection(
   key='raz',
@@ -2102,13 +2106,13 @@ RAZ = ConfigSection(
       key='is_enabled',
       help=_('Turns on the integration as ready to use'),
       type=coerce_bool,
-      dynamic_default=get_raz_api_url,
+      dynamic_default=has_raz_url,
     ),
     API_URL=Config(
         key='api_url',
         help=_('Endpoint to contact'),
         type=str,
-        dynamic_default=get_raz_api_url,
+        dynamic_default=has_raz_url,
     ),
     API_AUTHENTICATION=Config(
         key='api_authentication',

+ 10 - 4
desktop/core/src/desktop/models.py

@@ -1725,6 +1725,15 @@ class Document2Permission(models.Model):
 def get_cluster_config(user):
   return Cluster(user).get_app_config().get_config()
 
+def get_remote_home_storage():
+  remote_home_storage = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else None
+
+  if not remote_home_storage:
+    if get_raz_api_url() and get_raz_default_endpoint():
+      remote_home_storage = 's3a://%(bucket)s' % get_raz_default_endpoint()
+
+  return remote_home_storage
+
 
 class ClusterConfig(object):
   """
@@ -1956,10 +1965,7 @@ class ClusterConfig(object):
       hdfs_connectors.append(_('Files'))
 
 
-    remote_home_storage = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else None
-    if not remote_home_storage:
-      if get_raz_api_url.get() and get_raz_default_endpoint():
-        remote_home_storage = 's3a://%(bucket)s' % get_raz_default_endpoint()
+    remote_home_storage = get_remote_home_storage()
 
     for hdfs_connector in hdfs_connectors:
       force_home = remote_home_storage and not remote_home_storage.startswith('/')

+ 13 - 13
desktop/libs/aws/src/aws/conf.py

@@ -105,6 +105,14 @@ def get_default_region():
   return get_region(conf=AWS_ACCOUNTS['default']) if 'default' in AWS_ACCOUNTS else get_region()
 
 
+def get_default_host():
+  '''Returns the S3 host when Raz is configued'''
+
+  if get_raz_api_url():
+    endpoint = get_raz_default_endpoint()
+    if endpoint:
+      return endpoint.get('host')
+
 def get_region(conf=None):
   global REGION_CACHED
 
@@ -112,10 +120,10 @@ def get_region(conf=None):
     return REGION_CACHED
   region = ''
 
-  if conf:
+  if conf or get_default_host():
     # First check the host/endpoint configuration
-    if conf.HOST.get():
-      endpoint = conf.HOST.get()
+    endpoint = get_default_host() or conf.HOST.get()
+    if endpoint:
       if re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE):
         region = re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE).group('region')
       elif re.search(HYPHEN_ENDPOINT_RE, endpoint, re.IGNORECASE):
@@ -169,14 +177,6 @@ IS_SELF_SIGNING_ENABLED = Config(
   default=False,
 )
 
-def get_default_host():
-  '''Returns the S3 host when Raz is configued'''
-
-  if get_raz_api_url():
-    endpoint = get_raz_default_endpoint()
-    if endpoint:
-      return endpoint.get('host')
-
 def get_default_get_environment_credentials():
   '''Allow to check if environment credentials are present or not'''
   return not get_raz_api_url()
@@ -221,12 +221,12 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
       ALLOW_ENVIRONMENT_CREDENTIALS=Config(
         help=_('Allow to use environment sources of credentials (environment variables, EC2 profile).'),
         key='allow_environment_credentials',
-        dynamic_default=get_default_get_environment_credentials(),
+        default=True,
         type=coerce_bool
       ),
       REGION=Config(
         key='region',
-        dynamic_default=get_default_host,
+        default=None,
         type=str
       ),
       HOST=Config(

+ 90 - 0
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from hadoop import conf
+import logging
+import os
+import sys
+import tempfile
+
+from nose.tools import assert_true, assert_equal, assert_false, assert_not_equal, assert_raises
+
+from desktop.models import get_remote_home_storage
+from aws.conf import get_region
+
+from hadoop import core_site
+
+if sys.version_info[0] > 2:
+  open_file = open
+else:
+  open_file = file
+
+LOG = logging.getLogger(__name__)
+
+
+def test_core_site():
+  hadoop_home = tempfile.mkdtemp()
+  finish = []
+
+  try:
+    xml = """<?xml version="1.0"?>
+<configuration>
+  <property>
+    <name>fs.s3a.custom.signers</name>
+    <value>RazS3SignerPlugin:org.apache.ranger.raz.hook.s3.RazS3SignerPlugin:org.apache.ranger.raz.hook.s3.RazS3SignerPluginInitializer</value>
+  </property>
+  <property>
+    <name>fs.s3a.s3.signing-algorithm</name>
+    <value>RazS3SignerPlugin</value>
+  </property>
+  <property>
+    <name>fs.s3a.delegation.token.binding</name>
+    <value>org.apache.ranger.raz.hook.s3.RazDelegationTokenBinding</value>
+  </property>
+  <property>
+    <name>fs.s3a.ext.raz.rest.host.url</name>
+    <value>https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/</value>
+  </property>
+  <property>
+    <name>fs.s3a.ext.raz.s3.access.cluster.name</name>
+    <value>prakashdh67</value>
+  </property>
+  <property>
+    <name>fs.s3a.bucket.prakashmowdev1.endpoint</name>
+    <value>s3.us-west-2.amazonaws.com</value>
+  </property>
+</configuration>
+    """
+    open_file(os.path.join(hadoop_home, 'core-site.xml'), 'w').write(xml)
+
+    finish = (
+      conf.HDFS_CLUSTERS.set_for_testing({'default': {}}),
+      conf.HDFS_CLUSTERS['default'].HADOOP_CONF_DIR.set_for_testing(hadoop_home)
+    )
+    core_site.reset()
+
+    assert_equal(core_site.get_raz_api_url(), 'https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/')
+    assert_equal(core_site.get_raz_cluster_name(), 'prakashdh67')
+    assert_equal(core_site.get_raz_default_endpoint(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'prakashmowdev1'})
+
+    assert_equal(get_remote_home_storage(), 's3a://prakashmowdev1')
+    assert_equal(get_region(), 'us-west-2')
+  finally:
+    core_site.reset()
+    for f in finish:
+      f()