Browse Source

[abfs] First skeleton of RAZ client with tests

Romain Rigaux 4 years ago
parent
commit
bdd61d2606

+ 2 - 2
desktop/core/src/desktop/lib/raz/clients.py

@@ -44,13 +44,13 @@ class S3RazClient():
 
 class AdlsRazClient():
 
-  def __init__(self, storage_account):
+  def __init__(self):
     if RAZ.API_AUTHENTICATION.get() == 'kerberos':
       auth = HTTPKerberosAuth()
     else:
       auth = None
 
-    self.ranger = RangerRazAdls(RAZ.API_URL(), auth)
+    self.ranger = RangerRazAdls(RAZ.API_URL.get(), auth)
 
   def get_url(self, storage_account, container, relative_path, perm='read'):
     # e.g. get_url('<storage_account>', '<container>', '<relative_path>', 'read')

+ 55 - 0
desktop/core/src/desktop/lib/rest/raz_http_client.py

@@ -0,0 +1,55 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+
+from desktop import conf
+from desktop.lib.raz.clients import AdlsRazClient
+from desktop.lib.rest.http_client import HttpClient
+
+
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+else:
+  from urllib import quote as urllib_quote
+
+
+LOG = logging.getLogger(__name__)
+
+
+class RazHttpClient(HttpClient):
+
+  def __init__(self):
+    # Note: there is no concept of base_url and credentials anymore
+    # Maybe create here: http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG)
+    pass
+
+  def execute(self, http_method, path, params=None, data=None, headers=None, allow_redirects=False, urlencode=True,
+              files=None, stream=False, clear_cookies=False, timeout=conf.REST_CONN_TIMEOUT.get()):
+
+    raz_client = AdlsRazClient()
+
+    container = 'hue'
+    storage_account = 'gethue.dfs.core.windows.net'
+
+    # https://[storageaccountname].blob.core.windows.net/[containername]/[blobname]?sv=2014-02-14&sr=b&
+    #   sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r
+    tmp_url = raz_client.get_url(storage_account, container, relative_path=path, perm='read')
+
+    # TODO: get clean `path` etc
+
+    return super(RazHttpClient, self).execute(http_method=http_method, path=tmp_url)

+ 49 - 0
desktop/core/src/desktop/lib/rest/raz_http_client_test.py

@@ -0,0 +1,49 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+
+from nose.tools import assert_equal, assert_false, assert_true
+
+from desktop.lib.rest.raz_http_client import RazHttpClient
+
+
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock
+else:
+  from mock import patch, Mock
+
+
+class TestRazHttpClient():
+
+  def test_get_file(self):
+    with patch('desktop.lib.rest.raz_http_client.AdlsRazClient.get_url') as get_url:
+      with patch('desktop.lib.rest.raz_http_client.HttpClient.execute') as execute:
+
+        get_url.return_value = 'https://gethue.blob.core.windows.net/hue/data/customer.csv?sv=2014-02-14&sr=b&' + \
+          'sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r'
+        execute.return_value = 'my file'
+
+        client = RazHttpClient()
+        f = client.execute(http_method='GET', path='/gethue/data/customer.csv')
+
+        assert_equal('my file', f)
+        get_url.assert_called_with('gethue.dfs.core.windows.net', 'hue', relative_path='/gethue/data/customer.csv', perm='read')
+        execute.assert_called_with(http_method='GET', path='https://gethue.blob.core.windows.net/hue/data/customer.csv?' + \
+          'sv=2014-02-14&sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&' + \
+          'st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r'
+        )

+ 6 - 1
desktop/libs/azure/src/azure/abfs/abfs.py

@@ -33,7 +33,9 @@ from posixpath import join
 from hadoop.hdfs_site import get_umask_mode
 from hadoop.fs.exceptions import WebHdfsException
 
+from desktop.conf import RAZ
 from desktop.lib.rest import http_client, resource
+from desktop.lib.rest.raz_http_client import RazHttpClient
 
 import azure.abfs.__init__ as Init_ABFS
 from azure.abfs.abfsfile import ABFSFile
@@ -121,7 +123,10 @@ class ABFS(object):
     )
 
   def get_client(self, url):
-    return resource.Resource(http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG))
+    if RAZ.IS_ENABLED.get():
+      return resource.Resource(RazHttpClient())
+    else:
+      return resource.Resource(http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG))
 
   def _getheaders(self):
     return {