provider.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
  2. # Copyright 2010 Google Inc.
  3. # Copyright (c) 2010, Eucalyptus Systems, Inc.
  4. # Copyright (c) 2011, Nexenta Systems Inc.
  5. # All rights reserved.
  6. #
  7. # Permission is hereby granted, free of charge, to any person obtaining a
  8. # copy of this software and associated documentation files (the
  9. # "Software"), to deal in the Software without restriction, including
  10. # without limitation the rights to use, copy, modify, merge, publish, dis-
  11. # tribute, sublicense, and/or sell copies of the Software, and to permit
  12. # persons to whom the Software is furnished to do so, subject to the fol-
  13. # lowing conditions:
  14. #
  15. # The above copyright notice and this permission notice shall be included
  16. # in all copies or substantial portions of the Software.
  17. #
  18. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19. # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  20. # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  21. # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  22. # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24. # IN THE SOFTWARE.
  25. """
  26. This class encapsulates the provider-specific header differences.
  27. """
  28. import os
  29. from boto.compat import six
  30. from datetime import datetime
  31. import boto
  32. from boto import config
  33. from boto.compat import expanduser
  34. from boto.pyami.config import Config
  35. from boto.gs.acl import ACL
  36. from boto.gs.acl import CannedACLStrings as CannedGSACLStrings
  37. from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings
  38. from boto.s3.acl import Policy
  39. HEADER_PREFIX_KEY = 'header_prefix'
  40. METADATA_PREFIX_KEY = 'metadata_prefix'
  41. AWS_HEADER_PREFIX = 'x-amz-'
  42. GOOG_HEADER_PREFIX = 'x-goog-'
  43. ACL_HEADER_KEY = 'acl-header'
  44. AUTH_HEADER_KEY = 'auth-header'
  45. COPY_SOURCE_HEADER_KEY = 'copy-source-header'
  46. COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header'
  47. COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header'
  48. DELETE_MARKER_HEADER_KEY = 'delete-marker-header'
  49. DATE_HEADER_KEY = 'date-header'
  50. METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header'
  51. RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header'
  52. SECURITY_TOKEN_HEADER_KEY = 'security-token-header'
  53. STORAGE_CLASS_HEADER_KEY = 'storage-class'
  54. MFA_HEADER_KEY = 'mfa-header'
  55. SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header'
  56. VERSION_ID_HEADER_KEY = 'version-id-header'
  57. RESTORE_HEADER_KEY = 'restore-header'
  58. STORAGE_COPY_ERROR = 'StorageCopyError'
  59. STORAGE_CREATE_ERROR = 'StorageCreateError'
  60. STORAGE_DATA_ERROR = 'StorageDataError'
  61. STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError'
  62. STORAGE_RESPONSE_ERROR = 'StorageResponseError'
  63. NO_CREDENTIALS_PROVIDED = object()
  64. class ProfileNotFoundError(ValueError):
  65. pass
  66. class Provider(object):
  67. CredentialMap = {
  68. 'aws': ('aws_access_key_id', 'aws_secret_access_key',
  69. 'aws_security_token', 'aws_profile'),
  70. 'google': ('gs_access_key_id', 'gs_secret_access_key',
  71. None, None),
  72. }
  73. AclClassMap = {
  74. 'aws': Policy,
  75. 'google': ACL
  76. }
  77. CannedAclsMap = {
  78. 'aws': CannedS3ACLStrings,
  79. 'google': CannedGSACLStrings
  80. }
  81. HostKeyMap = {
  82. 'aws': 's3',
  83. 'google': 'gs'
  84. }
  85. ChunkedTransferSupport = {
  86. 'aws': False,
  87. 'google': True
  88. }
  89. MetadataServiceSupport = {
  90. 'aws': True,
  91. 'google': False
  92. }
  93. # If you update this map please make sure to put "None" for the
  94. # right-hand-side for any headers that don't apply to a provider, rather
  95. # than simply leaving that header out (which would cause KeyErrors).
  96. HeaderInfoMap = {
  97. 'aws': {
  98. HEADER_PREFIX_KEY: AWS_HEADER_PREFIX,
  99. METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-',
  100. ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl',
  101. AUTH_HEADER_KEY: 'AWS',
  102. COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source',
  103. COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX +
  104. 'copy-source-version-id',
  105. COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX +
  106. 'copy-source-range',
  107. DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date',
  108. DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker',
  109. METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX +
  110. 'metadata-directive',
  111. RESUMABLE_UPLOAD_HEADER_KEY: None,
  112. SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token',
  113. SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX +
  114. 'server-side-encryption',
  115. VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id',
  116. STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class',
  117. MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa',
  118. RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore',
  119. },
  120. 'google': {
  121. HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX,
  122. METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-',
  123. ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl',
  124. AUTH_HEADER_KEY: 'GOOG1',
  125. COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source',
  126. COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX +
  127. 'copy-source-version-id',
  128. COPY_SOURCE_RANGE_HEADER_KEY: None,
  129. DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date',
  130. DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker',
  131. METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX +
  132. 'metadata-directive',
  133. RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable',
  134. SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token',
  135. SERVER_SIDE_ENCRYPTION_KEY: None,
  136. # Note that this version header is not to be confused with
  137. # the Google Cloud Storage 'x-goog-api-version' header.
  138. VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id',
  139. STORAGE_CLASS_HEADER_KEY: None,
  140. MFA_HEADER_KEY: None,
  141. RESTORE_HEADER_KEY: None,
  142. }
  143. }
  144. ErrorMap = {
  145. 'aws': {
  146. STORAGE_COPY_ERROR: boto.exception.S3CopyError,
  147. STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
  148. STORAGE_DATA_ERROR: boto.exception.S3DataError,
  149. STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
  150. STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
  151. },
  152. 'google': {
  153. STORAGE_COPY_ERROR: boto.exception.GSCopyError,
  154. STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
  155. STORAGE_DATA_ERROR: boto.exception.GSDataError,
  156. STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
  157. STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
  158. }
  159. }
  160. def __init__(self, name, access_key=None, secret_key=None,
  161. security_token=None, profile_name=None):
  162. self.host = None
  163. self.port = None
  164. self.host_header = None
  165. self.access_key = access_key
  166. self.secret_key = secret_key
  167. self.security_token = security_token
  168. self.profile_name = profile_name
  169. self.name = name
  170. self.acl_class = self.AclClassMap[self.name]
  171. self.canned_acls = self.CannedAclsMap[self.name]
  172. self._credential_expiry_time = None
  173. # Load shared credentials file if it exists
  174. shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
  175. self.shared_credentials = Config(do_load=False)
  176. if os.path.isfile(shared_path):
  177. self.shared_credentials.load_from_path(shared_path)
  178. self.get_credentials(access_key, secret_key, security_token, profile_name)
  179. self.configure_headers()
  180. self.configure_errors()
  181. # Allow config file to override default host and port.
  182. host_opt_name = '%s_host' % self.HostKeyMap[self.name]
  183. if config.has_option('Credentials', host_opt_name):
  184. self.host = config.get('Credentials', host_opt_name)
  185. port_opt_name = '%s_port' % self.HostKeyMap[self.name]
  186. if config.has_option('Credentials', port_opt_name):
  187. self.port = config.getint('Credentials', port_opt_name)
  188. host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
  189. if config.has_option('Credentials', host_header_opt_name):
  190. self.host_header = config.get('Credentials', host_header_opt_name)
  191. def get_access_key(self):
  192. if self._credentials_need_refresh():
  193. self._populate_keys_from_metadata_server()
  194. return self._access_key
  195. def set_access_key(self, value):
  196. self._access_key = value
  197. access_key = property(get_access_key, set_access_key)
  198. def get_secret_key(self):
  199. if self._credentials_need_refresh():
  200. self._populate_keys_from_metadata_server()
  201. return self._secret_key
  202. def set_secret_key(self, value):
  203. self._secret_key = value
  204. secret_key = property(get_secret_key, set_secret_key)
  205. def get_security_token(self):
  206. if self._credentials_need_refresh():
  207. self._populate_keys_from_metadata_server()
  208. return self._security_token
  209. def set_security_token(self, value):
  210. self._security_token = value
  211. security_token = property(get_security_token, set_security_token)
  212. def _credentials_need_refresh(self):
  213. if self._credential_expiry_time is None:
  214. return False
  215. else:
  216. # The credentials should be refreshed if they're going to expire
  217. # in less than 5 minutes.
  218. delta = self._credential_expiry_time - datetime.utcnow()
  219. # python2.6 does not have timedelta.total_seconds() so we have
  220. # to calculate this ourselves. This is straight from the
  221. # datetime docs.
  222. seconds_left = (
  223. (delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
  224. * 10 ** 6) / 10 ** 6)
  225. if seconds_left < (5 * 60):
  226. boto.log.debug("Credentials need to be refreshed.")
  227. return True
  228. else:
  229. return False
  230. def get_credentials(self, access_key=None, secret_key=None,
  231. security_token=None, profile_name=None):
  232. access_key_name, secret_key_name, security_token_name, \
  233. profile_name_name = self.CredentialMap[self.name]
  234. # Load profile from shared environment variable if it was not
  235. # already passed in and the environment variable exists
  236. if profile_name is None and profile_name_name is not None and \
  237. profile_name_name.upper() in os.environ:
  238. profile_name = os.environ[profile_name_name.upper()]
  239. shared = self.shared_credentials
  240. if access_key is not None:
  241. self.access_key = access_key
  242. boto.log.debug("Using access key provided by client.")
  243. elif access_key_name.upper() in os.environ:
  244. self.access_key = os.environ[access_key_name.upper()]
  245. boto.log.debug("Using access key found in environment variable.")
  246. elif profile_name is not None:
  247. if shared.has_option(profile_name, access_key_name):
  248. self.access_key = shared.get(profile_name, access_key_name)
  249. boto.log.debug("Using access key found in shared credential "
  250. "file for profile %s." % profile_name)
  251. elif config.has_option("profile %s" % profile_name,
  252. access_key_name):
  253. self.access_key = config.get("profile %s" % profile_name,
  254. access_key_name)
  255. boto.log.debug("Using access key found in config file: "
  256. "profile %s." % profile_name)
  257. else:
  258. raise ProfileNotFoundError('Profile "%s" not found!' %
  259. profile_name)
  260. elif shared.has_option('default', access_key_name):
  261. self.access_key = shared.get('default', access_key_name)
  262. boto.log.debug("Using access key found in shared credential file.")
  263. elif config.has_option('Credentials', access_key_name):
  264. self.access_key = config.get('Credentials', access_key_name)
  265. boto.log.debug("Using access key found in config file.")
  266. if secret_key is not None:
  267. self.secret_key = secret_key
  268. boto.log.debug("Using secret key provided by client.")
  269. elif secret_key_name.upper() in os.environ:
  270. self.secret_key = os.environ[secret_key_name.upper()]
  271. boto.log.debug("Using secret key found in environment variable.")
  272. elif profile_name is not None:
  273. if shared.has_option(profile_name, secret_key_name):
  274. self.secret_key = shared.get(profile_name, secret_key_name)
  275. boto.log.debug("Using secret key found in shared credential "
  276. "file for profile %s." % profile_name)
  277. elif config.has_option("profile %s" % profile_name, secret_key_name):
  278. self.secret_key = config.get("profile %s" % profile_name,
  279. secret_key_name)
  280. boto.log.debug("Using secret key found in config file: "
  281. "profile %s." % profile_name)
  282. else:
  283. raise ProfileNotFoundError('Profile "%s" not found!' %
  284. profile_name)
  285. elif shared.has_option('default', secret_key_name):
  286. self.secret_key = shared.get('default', secret_key_name)
  287. boto.log.debug("Using secret key found in shared credential file.")
  288. elif config.has_option('Credentials', secret_key_name):
  289. self.secret_key = config.get('Credentials', secret_key_name)
  290. boto.log.debug("Using secret key found in config file.")
  291. elif config.has_option('Credentials', 'keyring'):
  292. keyring_name = config.get('Credentials', 'keyring')
  293. try:
  294. import keyring
  295. except ImportError:
  296. boto.log.error("The keyring module could not be imported. "
  297. "For keyring support, install the keyring "
  298. "module.")
  299. raise
  300. self.secret_key = keyring.get_password(
  301. keyring_name, self.access_key)
  302. boto.log.debug("Using secret key found in keyring.")
  303. if security_token is not None:
  304. self.security_token = security_token
  305. boto.log.debug("Using security token provided by client.")
  306. elif ((security_token_name is not None) and
  307. (access_key is None) and (secret_key is None)):
  308. # Only provide a token from the environment/config if the
  309. # caller did not specify a key and secret. Otherwise an
  310. # environment/config token could be paired with a
  311. # different set of credentials provided by the caller
  312. if security_token_name.upper() in os.environ:
  313. self.security_token = os.environ[security_token_name.upper()]
  314. boto.log.debug("Using security token found in environment"
  315. " variable.")
  316. elif shared.has_option(profile_name or 'default',
  317. security_token_name):
  318. self.security_token = shared.get(profile_name or 'default',
  319. security_token_name)
  320. boto.log.debug("Using security token found in shared "
  321. "credential file.")
  322. elif profile_name is not None:
  323. if config.has_option("profile %s" % profile_name,
  324. security_token_name):
  325. boto.log.debug("config has option")
  326. self.security_token = config.get("profile %s" % profile_name,
  327. security_token_name)
  328. boto.log.debug("Using security token found in config file: "
  329. "profile %s." % profile_name)
  330. elif config.has_option('Credentials', security_token_name):
  331. self.security_token = config.get('Credentials',
  332. security_token_name)
  333. boto.log.debug("Using security token found in config file.")
  334. if ((self._access_key is None or self._secret_key is None) and
  335. self.MetadataServiceSupport[self.name]):
  336. self._populate_keys_from_metadata_server()
  337. self._secret_key = self._convert_key_to_str(self._secret_key)
  338. def _populate_keys_from_metadata_server(self):
  339. # get_instance_metadata is imported here because of a circular
  340. # dependency.
  341. boto.log.debug("Retrieving credentials from metadata server.")
  342. from boto.utils import get_instance_metadata
  343. timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
  344. attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
  345. # The num_retries arg is actually the total number of attempts made,
  346. # so the config options is named *_num_attempts to make this more
  347. # clear to users.
  348. metadata = get_instance_metadata(
  349. timeout=timeout, num_retries=attempts,
  350. data='meta-data/iam/security-credentials/')
  351. if metadata:
  352. # I'm assuming there's only one role on the instance profile.
  353. security = list(metadata.values())[0]
  354. self._access_key = security['AccessKeyId']
  355. self._secret_key = self._convert_key_to_str(security['SecretAccessKey'])
  356. self._security_token = security['Token']
  357. expires_at = security['Expiration']
  358. self._credential_expiry_time = datetime.strptime(
  359. expires_at, "%Y-%m-%dT%H:%M:%SZ")
  360. boto.log.debug("Retrieved credentials will expire in %s at: %s",
  361. self._credential_expiry_time - datetime.now(), expires_at)
  362. def _convert_key_to_str(self, key):
  363. if isinstance(key, six.text_type):
  364. # the secret key must be bytes and not unicode to work
  365. # properly with hmac.new (see http://bugs.python.org/issue5285)
  366. return str(key)
  367. return key
  368. def configure_headers(self):
  369. header_info_map = self.HeaderInfoMap[self.name]
  370. self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
  371. self.header_prefix = header_info_map[HEADER_PREFIX_KEY]
  372. self.acl_header = header_info_map[ACL_HEADER_KEY]
  373. self.auth_header = header_info_map[AUTH_HEADER_KEY]
  374. self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY]
  375. self.copy_source_version_id = header_info_map[
  376. COPY_SOURCE_VERSION_ID_HEADER_KEY]
  377. self.copy_source_range_header = header_info_map[
  378. COPY_SOURCE_RANGE_HEADER_KEY]
  379. self.date_header = header_info_map[DATE_HEADER_KEY]
  380. self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY]
  381. self.metadata_directive_header = (
  382. header_info_map[METADATA_DIRECTIVE_HEADER_KEY])
  383. self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY]
  384. self.resumable_upload_header = (
  385. header_info_map[RESUMABLE_UPLOAD_HEADER_KEY])
  386. self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY]
  387. self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY]
  388. self.version_id = header_info_map[VERSION_ID_HEADER_KEY]
  389. self.mfa_header = header_info_map[MFA_HEADER_KEY]
  390. self.restore_header = header_info_map[RESTORE_HEADER_KEY]
  391. def configure_errors(self):
  392. error_map = self.ErrorMap[self.name]
  393. self.storage_copy_error = error_map[STORAGE_COPY_ERROR]
  394. self.storage_create_error = error_map[STORAGE_CREATE_ERROR]
  395. self.storage_data_error = error_map[STORAGE_DATA_ERROR]
  396. self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR]
  397. self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR]
  398. def get_provider_name(self):
  399. return self.HostKeyMap[self.name]
  400. def supports_chunked_transfer(self):
  401. return self.ChunkedTransferSupport[self.name]
  402. # Static utility method for getting default Provider.
  403. def get_default():
  404. return Provider('aws')