Przeglądaj źródła

[raz] Retrieve Raz server URL via ADLS config property in core-site.xml

Harshg999 4 lat temu
rodzic
commit
7152fe5a1f

+ 9 - 1
desktop/libs/hadoop/src/hadoop/core_site.py

@@ -47,6 +47,8 @@ _CNF_S3A_RAZ_API_URL = 'fs.s3a.ext.raz.rest.host.url'
 _CNF_S3A_RAZ_CLUSTER_NAME = 'fs.s3a.ext.raz.s3.access.cluster.name'
 _CNF_S3A_RAZ_BUCKET_ENDPOINT = 'fs.s3a.bucket.(?P<bucket>[^.]+).endpoint'
 
+_CNF_ADLS_RAZ_API_URL = 'fs.azure.ext.raz.rest.host.url'
+
 _CNF_ADLS_CLIENT_ID = 'dfs.adls.oauth2.client.id'
 _CNF_ADLS_AUTHENTICATION_CODE = 'dfs.adls.oauth2.credential'
 _CNF_ADLS_REFRESH_URL = 'dfs.adls.oauth2.refresh.url'
@@ -123,7 +125,13 @@ def get_raz_api_url():
 
   S3 only. Add check for Azure when supported.
   """
-  return get_conf().get(_CNF_S3A_RAZ_API_URL)
+  s3a_raz_url = get_conf().get(_CNF_S3A_RAZ_API_URL)
+  adls_raz_url = get_conf().get(_CNF_ADLS_RAZ_API_URL)
+
+  if s3a_raz_url != adls_raz_url:
+    LOG.warning('Raz API: S3A and ADLS URLs are different')
+
+  return s3a_raz_url or adls_raz_url
 
 def get_raz_cluster_name():
   """

+ 4 - 0
desktop/libs/hadoop/src/hadoop/core_site_tests.py

@@ -68,6 +68,10 @@ def test_core_site():
     <name>fs.s3a.bucket.prakashmowdev1.endpoint</name>
     <value>s3.us-west-2.amazonaws.com</value>
   </property>
+  <property>    
+    <name>fs.azure.ext.raz.rest.host.url</name>    
+    <value>https://prakashdh67-master10.prakashr.xcu2-8y8x.dev.cldr.work:6082/</value>  
+  </property> 
 </configuration>
     """
     open_file(os.path.join(hadoop_home, 'core-site.xml'), 'w').write(xml)