Prechádzať zdrojové kódy

[raz] Retry some failed ABFS calls once again (#3114)

- Sometimes correct ABFS calls with correct params and having a SAS token from RAZ failed with a 403 signature mismatch error intermittently. This gives a bad user experience for someone browsing the ABFS file browser.
- We are now retrying such operation for one more time as a workaround time so that they execute successfully (since this issue is intermittent).
- However, the real reason of call failure might be because of some problem with the generated SAS token from RAZ.
- Added units test for retrying operations as well.
Harsh Gupta 3 rokov pred
rodič
commit
c6a80cafbe

+ 40 - 13
desktop/core/src/desktop/lib/rest/raz_http_client.py

@@ -38,7 +38,7 @@ class RazHttpClient(HttpClient):
     self.username = username
 
   def execute(self, http_method, path, params=None, data=None, headers=None, allow_redirects=False, urlencode=True,
-              files=None, stream=False, clear_cookies=False, timeout=conf.REST_CONN_TIMEOUT.get()):
+              files=None, stream=False, clear_cookies=False, timeout=conf.REST_CONN_TIMEOUT.get(), retry=1):
     """
     From an object URL we get back the SAS token as a GET param string, e.g.:
     https://{storageaccountname}.dfs.core.windows.net/{container}/{path}
@@ -63,18 +63,45 @@ class RazHttpClient(HttpClient):
     # so we remove https://{storageaccountname}.dfs.core.windows.net from the signed url here.
     signed_path = signed_url.partition('.dfs.core.windows.net')[2]
 
-    return super(RazHttpClient, self).execute(
-        http_method=http_method,
-        path=signed_path,
-        data=data,
-        headers=headers,
-        allow_redirects=allow_redirects,
-        urlencode=False,
-        files=files,
-        stream=stream,
-        clear_cookies=clear_cookies,
-        timeout=timeout
-    )
+    try:
+      # Sometimes correct call with SAS token fails, so we retry some operations once again.
+      if retry >= 0:
+        return super(RazHttpClient, self).execute(
+            http_method=http_method,
+            path=signed_path,
+            data=data,
+            headers=headers,
+            allow_redirects=allow_redirects,
+            urlencode=False,
+            files=files,
+            stream=stream,
+            clear_cookies=clear_cookies,
+            timeout=timeout
+        )
+    except Exception as e:
+      LOG.debug('ABFS Exception: ' + str(e))
+
+      # Only retrying safe operations once.
+      if http_method in ('HEAD', 'GET') and e.code == 403: 
+        LOG.debug('Retrying same operation again for path: %s' % path)
+        retry -= 1
+        return self.execute(
+            http_method=http_method, 
+            path=path, 
+            params=params, 
+            data=data, 
+            headers=headers, 
+            allow_redirects=allow_redirects, 
+            urlencode=urlencode, 
+            files=files, 
+            stream=stream, 
+            clear_cookies=clear_cookies, 
+            timeout=timeout, 
+            retry=retry
+        )
+      else:
+        # Re-raise all other exceptions to be handled later for other operations such as rename.
+        raise e
 
   def get_sas_token(self, http_method, username, url, params=None, headers=None):
     raz_client = AdlsRazClient(username=username)

+ 29 - 0
desktop/core/src/desktop/lib/rest/raz_http_client_test.py

@@ -21,6 +21,7 @@ from nose.tools import assert_equal, assert_false, assert_true, assert_raises
 from desktop.lib.rest.raz_http_client import RazHttpClient
 from desktop.lib.exceptions_renderable import PopupException
 
+from hadoop.fs.exceptions import WebHdfsException
 
 if sys.version_info[0] > 2:
   from unittest.mock import patch, Mock
@@ -100,6 +101,34 @@ class TestRazHttpClient():
         )
 
 
+  def test_retry_operations(self):
+    with patch('desktop.lib.rest.raz_http_client.AdlsRazClient.get_url') as raz_get_url:
+      with patch('desktop.lib.rest.raz_http_client.HttpClient.execute') as raz_http_execute:
+
+        raz_get_url.return_value = {
+          'token': 'sv=2014-02-14&sr=b&sig=pJL%2FWyed41tptiwBM5ymYre4qF8wzrO05tS5MCjkutc%3D' \
+            '&st=2015-01-02T01%3A40%3A51Z&se=2015-01-02T02%3A00%3A51Z&sp=r'
+        }
+        raz_http_execute.side_effect = WebHdfsException(Mock(response=Mock(status_code=403, text='Signature Mismatch')))
+
+        client = RazHttpClient(username='test', base_url='https://gethue.dfs.core.windows.net')
+        response = client.execute(http_method='HEAD', path='/gethue/user/demo', params={'action': 'getStatus'})
+        url = 'https://gethue.dfs.core.windows.net/gethue/user/demo?action=getStatus'
+
+        raz_get_url.assert_called_with(action='HEAD', path=url, headers=None)
+        # Although we are mocking that both times ABFS sends 403 exception but still it retries only twice as per expectation.
+        assert_equal(raz_http_execute.call_count, 2)
+
+        # When ABFS raises exception with code other than 403.
+        raz_http_execute.side_effect = WebHdfsException(Mock(response=Mock(status_code=404, text='Error resource not found')))
+        client = RazHttpClient(username='test', base_url='https://gethue.dfs.core.windows.net')
+        url = 'https://gethue.dfs.core.windows.net/gethue/user/demo?action=getStatus'
+
+        # Exception got re-raised for later use.
+        assert_raises(WebHdfsException, client.execute, http_method='HEAD', path='/gethue/user/demo', params={'action': 'getStatus'})
+        raz_get_url.assert_called_with(action='HEAD', path=url, headers=None)
+
+
   def test_handle_raz_adls_response(self):
     with patch('desktop.lib.rest.raz_http_client.AdlsRazClient.get_url') as raz_get_url: