Преглед изворни кода

[raz] Fix rename and cases of % in path for RAZ ABFS (#3173)

What changes were proposed in this pull request?

- Fixing creation of directories and other directory operation having % in name/path for ABFS. For this, we have to double-encode such path. This is required because we do the calls to ABFS in single encoding and the underlying HTTPClient takes care of unquoting it once already.
- For the RAZ side, we make sure that we unquote such paths once before making the RAZ request to get the SAS token.

- We are also making sure that we send correct path in-case of actual whitespaces vs actual %20 present in directory paths.
- This means that we only double encode actual %20 paths and not the paths which contain a whitespace (because encoding a whitespace gives %20).

- Fix rename in Py2 setup for non-ascii and % in path scenarios for ABFS.
- In case of Py2 setup, we are UTF-8 converting the unicode to str for correct URL quoting.

How was this patch tested?

- Manually E2E tested in RAZ enabled cluster.
- Added new unit tests for different scenarios and updated some existing tests.
Harsh Gupta пре 2 година
родитељ
комит
eda7fbebfc

+ 7 - 2
desktop/core/src/desktop/lib/rest/raz_http_client.py

@@ -24,8 +24,10 @@ from desktop.lib.exceptions_renderable import PopupException
 
 if sys.version_info[0] > 2:
   from django.utils.translation import gettext as _
+  from urllib.parse import quote as lib_urlquote
 else:
   from django.utils.translation import ugettext as _
+  from urllib import quote as lib_urlquote
 
 
 LOG = logging.getLogger(__name__)
@@ -46,8 +48,12 @@ class RazHttpClient(HttpClient):
     https://{storageaccountname}.dfs.core.windows.net/{container}/{path}?sv=2014-02-14&sr=b&
     sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r
     """
+    if sys.version_info[0] < 3 and isinstance(path, unicode):
+      path = path.encode('utf-8')
 
-    url = self._make_url(path, params)
+    do_urlencode = True if params and 'directory' in params and '%' in params['directory'] else False
+
+    url = self._make_url(lib_urlquote(path), params, do_urlencode=do_urlencode)
 
     # For root stats, the root path needs to end with '/' before adding the query params.
     if params and 'action' in params and params['action'] == 'getAccessControl':
@@ -56,7 +62,6 @@ class RazHttpClient(HttpClient):
       url = ''.join(partition_url)
 
     sas_token = self.get_sas_token(http_method, self.username, url, params, headers)
-
     signed_url = url + ('?' if '?' not in url else '&') + sas_token
 
     # self._make_url is called in base class execute method as well,

+ 92 - 5
desktop/core/src/desktop/lib/rest/raz_http_client_test.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -61,16 +62,92 @@ class TestRazHttpClient():
             timeout=120
         )
 
-        # Check for path having whitespaces (#20)
+        # Check for file path having whitespaces
         f = client.execute(http_method='GET', path='/gethue/data/banks (1).csv', params={'action': 'getStatus'})
 
-        url = 'https://gethue.dfs.core.windows.net/gethue/data/banks%20(1).csv?action=getStatus'
+        url = 'https://gethue.dfs.core.windows.net/gethue/data/banks%20%281%29.csv?action=getStatus'
         assert_equal('my_file_content', f)
         raz_get_url.assert_called_with(action='GET', path=url, headers=None)
         raz_http_execute.assert_called_with(
             http_method='GET',
-            path='/gethue/data/banks%20(1).csv?action=getStatus&sv=2014-02-14&sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D' \
-              '&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r',
+            path='/gethue/data/banks%20%281%29.csv?action=getStatus&sv=2014-02-14&sr=b&' \
+              'sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r',
+            data=None,
+            headers=None,
+            allow_redirects=False,
+            urlencode=False,
+            files=None,
+            stream=False,
+            clear_cookies=False,
+            timeout=120
+        )
+
+
+  def test_directory_paths(self):
+    with patch('desktop.lib.rest.raz_http_client.AdlsRazClient.get_url') as raz_get_url:
+      with patch('desktop.lib.rest.raz_http_client.HttpClient.execute') as raz_http_execute:
+        raz_get_url.return_value = {
+          'token': 'sv=2014-02-14&sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D' \
+            '&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r'
+        }
+        client = RazHttpClient(username='test', base_url='https://gethue.dfs.core.windows.net')
+
+        # List call for non-ascii directory name (/user/Tжейкоб)
+        if sys.version_info[0] > 2:
+          params = {'directory': 'user/T\u0436\u0435\u0438\u0306\u043a\u043e\u0431', 'resource': 'filesystem'}
+        else:
+          params = {u'directory': u'user/T\u0436\u0435\u0438\u0306\u043a\u043e\u0431', u'resource': u'filesystem'}
+
+        f = client.execute(
+          http_method='GET',
+          path='/test',
+          params=params
+        )
+        url = 'https://gethue.dfs.core.windows.net/test?directory=user/T%D0%B6%D0%B5%D0%B8%CC%86%D0%BA%D0%BE%D0%B1&resource=filesystem'
+
+        raz_get_url.assert_called_with(action='GET', path=url, headers=None)
+        raz_http_execute.assert_called_with(
+            http_method='GET',
+            path='/test?directory=user/T%D0%B6%D0%B5%D0%B8%CC%86%D0%BA%D0%BE%D0%B1&resource=filesystem&sv=2014-02-14&' \
+              'sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r',
+            data=None,
+            headers=None,
+            allow_redirects=False,
+            urlencode=False,
+            files=None,
+            stream=False,
+            clear_cookies=False,
+            timeout=120
+        )
+
+        # List call for directory name having whitespaces (/user/test dir)
+        f = client.execute(http_method='GET', path='/test', params={'directory': 'user/test user', 'resource': 'filesystem'})
+        url = 'https://gethue.dfs.core.windows.net/test?directory=user/test%20user&resource=filesystem'
+
+        raz_get_url.assert_called_with(action='GET', path=url, headers=None)
+        raz_http_execute.assert_called_with(
+            http_method='GET',
+            path='/test?directory=user/test%20user&resource=filesystem&sv=2014-02-14&' \
+              'sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r',
+            data=None,
+            headers=None,
+            allow_redirects=False,
+            urlencode=False,
+            files=None,
+            stream=False,
+            clear_cookies=False,
+            timeout=120
+        )
+
+        # List call for directory name having %20 like characters (/user/ab%20cd)
+        f = client.execute(http_method='GET', path='/test', params={'directory': 'user/ab%20cd', 'resource': 'filesystem'})
+        url = 'https://gethue.dfs.core.windows.net/test?directory=user%2Fab%2520cd&resource=filesystem'
+
+        raz_get_url.assert_called_with(action='GET', path=url, headers=None)
+        raz_http_execute.assert_called_with(
+            http_method='GET',
+            path='/test?directory=user%2Fab%2520cd&resource=filesystem&sv=2014-02-14&' \
+              'sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r',
             data=None,
             headers=None,
             allow_redirects=False,
@@ -81,7 +158,17 @@ class TestRazHttpClient():
             timeout=120
         )
 
-        # Check for root path stats
+
+  def test_root_path_stats(self):
+    with patch('desktop.lib.rest.raz_http_client.AdlsRazClient.get_url') as raz_get_url:
+      with patch('desktop.lib.rest.raz_http_client.HttpClient.execute') as raz_http_execute:
+
+        raz_get_url.return_value = {
+          'token': 'sv=2014-02-14&sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D' \
+            '&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r'
+        }
+
+        client = RazHttpClient(username='test', base_url='https://gethue.dfs.core.windows.net')
         f = client.execute(http_method='HEAD', path='/gethue', params={'action': 'getAccessControl'})
         url = 'https://gethue.dfs.core.windows.net/gethue/?action=getAccessControl'
 

+ 5 - 1
desktop/libs/azure/src/azure/abfs/abfs.py

@@ -587,7 +587,11 @@ class ABFS(object):
     """
     Renames a file
     """
-    headers = {'x-ms-rename-source': '/' + urllib_quote(Init_ABFS.strip_scheme(old))}
+    rename_source = Init_ABFS.strip_scheme(old)
+    if sys.version_info[0] < 3 and isinstance(rename_source, unicode):
+      rename_source = rename_source.encode('utf-8')
+
+    headers = {'x-ms-rename-source': '/' + urllib_quote(rename_source)}
 
     try:
       self._create_path(new, headers=headers, overwrite=True)