Browse Source

[raz] Auto detect ADLS cluster name from config file (#2391)

- Also, anonymise the test data of core-site.xml and remove stale docstrings
Harsh Gupta 4 years ago
parent
commit
4568d96e03

+ 2 - 5
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -48,6 +48,7 @@ _CNF_S3A_RAZ_CLUSTER_NAME = 'fs.s3a.ext.raz.s3.access.cluster.name'
 _CNF_S3A_RAZ_BUCKET_ENDPOINT = 'fs.s3a.bucket.(?P<bucket>[^.]+).endpoint'
 
 _CNF_ADLS_RAZ_API_URL = 'fs.azure.ext.raz.rest.host.url'
+_CNF_ADLS_RAZ_CLUSTER_NAME = 'fs.azure.ext.raz.adls.access.cluster.name'
 
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
@@ -122,8 +123,6 @@ def get_s3a_session_token():
 def get_raz_api_url():
   """
   Get Raz API.
-
-  S3 only. Add check for Azure when supported.
   """
   s3a_raz_url = get_conf().get(_CNF_S3A_RAZ_API_URL)
   adls_raz_url = get_conf().get(_CNF_ADLS_RAZ_API_URL)
@@ -136,10 +135,8 @@ def get_raz_api_url():
 def get_raz_cluster_name():
   """
   Get the name of the Cluster where Raz is running.
-
-  S3 only. Add check for Azure when supported.
   """
-  return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '')
+  return get_conf().get(_CNF_S3A_RAZ_CLUSTER_NAME, '') or get_conf().get(_CNF_ADLS_RAZ_CLUSTER_NAME, '')
 
 def get_raz_default_endpoint():
   """

+ 12 - 8
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -58,20 +58,24 @@ def test_core_site():
   </property>
   <property>
     <name>fs.s3a.ext.raz.rest.host.url</name>
-    <value>https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/</value>
+    <value>https://gehue-adls-master:6082/</value>
   </property>
   <property>
     <name>fs.s3a.ext.raz.s3.access.cluster.name</name>
-    <value>prakashdh67</value>
+    <value>gehue-adls</value>
   </property>
   <property>
-    <name>fs.s3a.bucket.prakashmowdev1.endpoint</name>
+    <name>fs.s3a.bucket.gethue-dev.endpoint</name>
     <value>s3.us-west-2.amazonaws.com</value>
   </property>
   <property>    
     <name>fs.azure.ext.raz.rest.host.url</name>    
-    <value>https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/</value>  
+    <value>https://gehue-adls-master:6082/</value>  
   </property> 
+  <property>
+    <name>fs.azure.ext.raz.adls.access.cluster.name</name>
+    <value>gehue-adls</value>
+  </property>
 </configuration>
     """
     open_file(os.path.join(hadoop_home, 'core-site.xml'), 'w').write(xml)
@@ -82,11 +86,11 @@ def test_core_site():
     )
     core_site.reset()
 
-    assert_equal(core_site.get_raz_api_url(), 'https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/')
-    assert_equal(core_site.get_raz_cluster_name(), 'prakashdh67')
-    assert_equal(core_site.get_raz_default_endpoint(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'prakashmowdev1'})
+    assert_equal(core_site.get_raz_api_url(), 'https://gehue-adls-master:6082/')
+    assert_equal(core_site.get_raz_cluster_name(), 'gehue-adls')
+    assert_equal(core_site.get_raz_default_endpoint(), {'host': 's3.us-west-2.amazonaws.com', 'bucket': 'gethue-dev'})
 
-    assert_equal(get_remote_home_storage(), 's3a://prakashmowdev1')
+    assert_equal(get_remote_home_storage(), 's3a://gethue-dev')
     assert_equal(get_region(), 'us-west-2')
   finally:
     core_site.reset()