Browse Source

HUE-8737 [core] Futurize desktop/libs/aws for Python 3.5

Ying Chen 6 năm trước cách đây
mục cha
commit
058fee64e0

+ 1 - 1
desktop/libs/aws/src/aws/__init__.py

@@ -35,7 +35,7 @@ def _init_clients():
   if CLIENT_CACHE is not None:
     return
   CLIENT_CACHE = {}
-  for identifier in conf.AWS_ACCOUNTS.keys():
+  for identifier in list(conf.AWS_ACCOUNTS.keys()):
     CLIENT_CACHE[identifier] = _make_client(identifier)
   # If default configuration not initialized, initialize client connection with IAM metadata
   if 'default' not in CLIENT_CACHE and conf.has_iam_metadata():

+ 3 - 1
desktop/libs/aws/src/aws/client.py

@@ -15,6 +15,8 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import str
+from builtins import object
 import logging
 import os
 
@@ -115,7 +117,7 @@ class Client(object):
       else:
         kwargs.update({'host': 's3.amazonaws.com'})
         connection = boto.s3.connection.S3Connection(**kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
 

+ 3 - 3
desktop/libs/aws/src/aws/conf.py

@@ -204,7 +204,7 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
 
 
 def is_enabled():
-  return ('default' in AWS_ACCOUNTS.keys() and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata()
+  return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata()
 
 
 def has_iam_metadata():
@@ -214,7 +214,7 @@ def has_iam_metadata():
     if os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'read').read()[:3] == 'ec2':
       metadata = boto.utils.get_instance_metadata(timeout=1, num_retries=1)
       return 'iam' in metadata
-  except Exception, e:
+  except Exception as e:
     LOG.exception("Encountered error when checking IAM metadata: %s" % e)
   return False
 
@@ -231,7 +231,7 @@ def config_validator(user):
     try:
       conn = aws.get_client('default').get_s3_connection()
       conn.get_canonical_user_id()
-    except Exception, e:
+    except Exception as e:
       LOG.exception('AWS failed configuration check.')
       res.append(('aws', _t('Failed to connect to S3, check your AWS credentials.')))
 

+ 2 - 1
desktop/libs/aws/src/aws/s3/__init__.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import map
 import calendar
 import errno
 import logging
@@ -98,7 +99,7 @@ def join(*comp_list):
       return '/%s/%s' % parse_uri(uri)[:2]
     except ValueError:
       return '/' if is_root(uri) else uri
-  joined = posixpath.join(*map(_prep, comp_list))
+  joined = posixpath.join(*list(map(_prep, comp_list)))
   if joined and joined[0] == '/':
     joined = 's3a:/%s' % joined
   return joined

+ 17 - 15
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -16,6 +16,8 @@
 
 from __future__ import absolute_import
 
+from builtins import str
+from builtins import object
 import itertools
 import logging
 import os
@@ -53,7 +55,7 @@ def auth_error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except (S3ResponseError, IOError), e:
+    except (S3ResponseError, IOError) as e:
       if 'Forbidden' in str(e) or (hasattr(e, 'status') and e.status == 403):
         path = kwargs.get('path')
         if not path and len(args) > 1:
@@ -67,7 +69,7 @@ def auth_error_handler(view_fn):
         if isinstance(e, S3ResponseError):
           msg = e.message or e.reason
         raise S3FileSystemException(msg)
-    except Exception, e:
+    except Exception as e:
       raise e
   return decorator
 
@@ -87,9 +89,9 @@ class S3FileSystem(object):
   def _get_or_create_bucket(self, name):
     try:
       bucket = self._get_bucket(name)
-    except BotoClientError, e:
+    except BotoClientError as e:
       raise S3FileSystemException(_('Failed to create bucket named "%s": %s') % (name, e.reason))
-    except S3ResponseError, e:
+    except S3ResponseError as e:
       if e.status == 403 or e.status == 301:
         raise S3FileSystemException(_('User is not authorized to access bucket named "%s". '
           'If you are attempting to create a bucket, this bucket name is already reserved.') % name)
@@ -124,7 +126,7 @@ class S3FileSystem(object):
         key.delete()
       self._s3_connection.delete_bucket(name)
       LOG.info('Successfully deleted bucket name "%s" and all its contents.' % name)
-    except S3ResponseError, e:
+    except S3ResponseError as e:
       if e.status == 403:
         raise S3FileSystemException(_('User is not authorized to access bucket named "%s". '
           'If you are attempting to create a bucket, this bucket name is already reserved.') % name)
@@ -136,9 +138,9 @@ class S3FileSystem(object):
     bucket = self._get_bucket(bucket_name)
     try:
       return bucket.get_key(key_name, validate=validate)
-    except BotoClientError, e:
+    except BotoClientError as e:
       raise S3FileSystemException(_('Failed to access path at "%s": %s') % (path, e.reason))
-    except S3ResponseError, e:
+    except S3ResponseError as e:
       if e.status in (301, 400):
         raise S3FileSystemException(_('Failed to access path: "%s" '
           'Check that you have access to read this bucket and that the region is correct: %s') % (path, e.message or e.reason))
@@ -159,7 +161,7 @@ class S3FileSystem(object):
 
     try:
       key = self._get_key(path, validate=True)
-    except BotoClientError, e:
+    except BotoClientError as e:
       raise S3FileSystemException(_('Failed to access path "%s": %s') % (path, e.reason))
     except S3ResponseError as e:
       if e.status == 404:
@@ -270,11 +272,11 @@ class S3FileSystem(object):
     if s3.is_root(path):
       try:
         return sorted([S3Stat.from_bucket(b) for b in self._s3_connection.get_all_buckets()], key=lambda x: x.name)
-      except S3FileSystemException, e:
+      except S3FileSystemException as e:
         raise e
-      except S3ResponseError, e:
+      except S3ResponseError as e:
         raise S3FileSystemException(_('Failed to retrieve buckets: %s') % e.reason)
-      except Exception, e:
+      except Exception as e:
         raise S3FileSystemException(_('Failed to retrieve buckets: %s') % e)
 
     bucket_name, prefix = s3.parse_uri(path)[:2]
@@ -350,11 +352,11 @@ class S3FileSystem(object):
 
     try:
       self._get_or_create_bucket(bucket_name)
-    except S3FileSystemException, e:
+    except S3FileSystemException as e:
       raise e
-    except S3ResponseError, e:
+    except S3ResponseError as e:
       raise S3FileSystemException(_('Failed to create S3 bucket "%s": %s: %s') % (bucket_name, e.reason, e.body))
-    except Exception, e:
+    except Exception as e:
       raise S3FileSystemException(_('Failed to create S3 bucket "%s": %s') % (bucket_name, e))
 
     stats = self._stats(path)
@@ -494,7 +496,7 @@ class S3FileSystem(object):
         self.remove(path=tmp_path)
       else:
         self.open(path)
-    except Exception, e:
+    except Exception as e:
       LOG.warn('S3 check_access encountered error verifying %s permission at path "%s": %s' % (permission, path, str(e)))
       return False
     return True

+ 3 - 2
desktop/libs/aws/src/aws/s3/s3stat.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import object
 import stat
 import posixpath
 
@@ -22,8 +23,8 @@ from aws.s3 import s3datetime_to_timestamp
 
 
 class S3Stat(object):
-  DIR_MODE = 0777 | stat.S_IFDIR
-  FILE_MODE = 0666 | stat.S_IFREG
+  DIR_MODE = 0o777 | stat.S_IFDIR
+  FILE_MODE = 0o666 | stat.S_IFREG
 
   def __init__(self, name, path, isDir, size, mtime):
     self.name = name

+ 3 - 2
desktop/libs/aws/src/aws/s3/s3stat_test.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import object
 import stat
 
 from nose.tools import eq_
@@ -25,7 +26,7 @@ from aws.s3.s3stat import S3Stat
 def test_derivable_properties():
   s = S3Stat('foo', 's3a://bar/foo', False, 40, 1424983327)
   eq_('FILE', s.type)
-  eq_(0666 | stat.S_IFREG, s.mode)
+  eq_(0o666 | stat.S_IFREG, s.mode)
   eq_('', s.user)
   eq_('', s.group)
   eq_(1424983327, s.atime)
@@ -33,7 +34,7 @@ def test_derivable_properties():
 
   s = S3Stat('bar', 's3a://bar', True, 0, 1424983327)
   eq_('DIRECTORY', s.type)
-  eq_(0777 | stat.S_IFDIR, s.mode)
+  eq_(0o777 | stat.S_IFDIR, s.mode)
 
 
 def test_from_bucket():

+ 1 - 0
desktop/libs/aws/src/aws/s3/s3test_utils.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import range
 import logging
 import os
 import random

+ 11 - 4
desktop/libs/aws/src/aws/s3/upload.py

@@ -21,8 +21,15 @@ Classes for a custom upload handler to stream into S3.
 See http://docs.djangoproject.com/en/1.9/topics/http/file-uploads/
 """
 
+from future import standard_library
+standard_library.install_aliases()
 import logging
-import StringIO
+import sys
+
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.files.uploadhandler import FileUploadHandler, SkipFile, StopFutureHandlers, StopUpload, UploadFileException
@@ -81,7 +88,7 @@ class S3FileUploadHandler(FileUploadHandler):
         self._mp = self._bucket.initiate_multipart_upload(self.target_path)
         self.file = SimpleUploadedFile(name=file_name, content='')
         raise StopFutureHandlers()
-      except (S3FileUploadError, S3FileSystemException), e:
+      except (S3FileUploadError, S3FileSystemException) as e:
         LOG.error("Encountered error in S3UploadHandler check_access: %s" % e)
         self.request.META['upload_failed'] = e
         raise StopUpload()
@@ -95,7 +102,7 @@ class S3FileUploadHandler(FileUploadHandler):
         self._mp.upload_part_from_file(fp=fp, part_num=self._part_num)
         self._part_num += 1
         return None
-      except Exception, e:
+      except Exception as e:
         self._mp.cancel_upload()
         LOG.exception('Failed to upload file to S3 at %s: %s' % (self.target_path, e))
         raise StopUpload()
@@ -144,7 +151,7 @@ class S3FileUploadHandler(FileUploadHandler):
 
 
   def _get_file_part(self, raw_data):
-    fp = StringIO.StringIO()
+    fp = string_io()
     fp.write(raw_data)
     fp.seek(0)
     return fp