auth.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044
  1. # Copyright 2010 Google Inc.
  2. # Copyright (c) 2011 Mitch Garnaat http://garnaat.org/
  3. # Copyright (c) 2011, Eucalyptus Systems, Inc.
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a
  6. # copy of this software and associated documentation files (the
  7. # "Software"), to deal in the Software without restriction, including
  8. # without limitation the rights to use, copy, modify, merge, publish, dis-
  9. # tribute, sublicense, and/or sell copies of the Software, and to permit
  10. # persons to whom the Software is furnished to do so, subject to the fol-
  11. # lowing conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included
  14. # in all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17. # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  18. # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  19. # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  20. # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22. # IN THE SOFTWARE.
  23. """
  24. Handles authentication required to AWS and GS
  25. """
  26. import base64
  27. import boto
  28. import boto.auth_handler
  29. import boto.exception
  30. import boto.plugin
  31. import boto.utils
  32. import copy
  33. import datetime
  34. from email.utils import formatdate
  35. import hmac
  36. import os
  37. import posixpath
  38. from boto.compat import urllib, encodebytes, parse_qs_safe
  39. from boto.auth_handler import AuthHandler
  40. from boto.exception import BotoClientError
  41. try:
  42. from hashlib import sha1 as sha
  43. from hashlib import sha256 as sha256
  44. except ImportError:
  45. import sha
  46. sha256 = None
  47. # Region detection strings to determine if SigV4 should be used
  48. # by default.
  49. SIGV4_DETECT = [
  50. '.cn-',
  51. # In eu-central and ap-northeast-2 we support both host styles for S3
  52. '.eu-central',
  53. '-eu-central',
  54. '.ap-northeast-2',
  55. '-ap-northeast-2',
  56. '.ap-south-1',
  57. '-ap-south-1'
  58. ]
  59. class HmacKeys(object):
  60. """Key based Auth handler helper."""
  61. def __init__(self, host, config, provider):
  62. if provider.access_key is None or provider.secret_key is None:
  63. raise boto.auth_handler.NotReadyToAuthenticate()
  64. self.host = host
  65. self.update_provider(provider)
  66. def update_provider(self, provider):
  67. self._provider = provider
  68. self._hmac = hmac.new(self._provider.secret_key.encode('utf-8'),
  69. digestmod=sha)
  70. if sha256:
  71. self._hmac_256 = hmac.new(self._provider.secret_key.encode('utf-8'),
  72. digestmod=sha256)
  73. else:
  74. self._hmac_256 = None
  75. def algorithm(self):
  76. if self._hmac_256:
  77. return 'HmacSHA256'
  78. else:
  79. return 'HmacSHA1'
  80. def _get_hmac(self):
  81. if self._hmac_256:
  82. digestmod = sha256
  83. else:
  84. digestmod = sha
  85. return hmac.new(self._provider.secret_key.encode('utf-8'),
  86. digestmod=digestmod)
  87. def sign_string(self, string_to_sign):
  88. new_hmac = self._get_hmac()
  89. new_hmac.update(string_to_sign.encode('utf-8'))
  90. return encodebytes(new_hmac.digest()).decode('utf-8').strip()
  91. def __getstate__(self):
  92. pickled_dict = copy.copy(self.__dict__)
  93. del pickled_dict['_hmac']
  94. del pickled_dict['_hmac_256']
  95. return pickled_dict
  96. def __setstate__(self, dct):
  97. self.__dict__ = dct
  98. self.update_provider(self._provider)
  99. class AnonAuthHandler(AuthHandler, HmacKeys):
  100. """
  101. Implements Anonymous requests.
  102. """
  103. capability = ['anon']
  104. def __init__(self, host, config, provider):
  105. super(AnonAuthHandler, self).__init__(host, config, provider)
  106. def add_auth(self, http_request, **kwargs):
  107. pass
  108. class HmacAuthV1Handler(AuthHandler, HmacKeys):
  109. """ Implements the HMAC request signing used by S3 and GS."""
  110. capability = ['hmac-v1', 's3']
  111. def __init__(self, host, config, provider):
  112. AuthHandler.__init__(self, host, config, provider)
  113. HmacKeys.__init__(self, host, config, provider)
  114. self._hmac_256 = None
  115. def update_provider(self, provider):
  116. super(HmacAuthV1Handler, self).update_provider(provider)
  117. self._hmac_256 = None
  118. def add_auth(self, http_request, **kwargs):
  119. headers = http_request.headers
  120. method = http_request.method
  121. auth_path = http_request.auth_path
  122. if 'Date' not in headers:
  123. headers['Date'] = formatdate(usegmt=True)
  124. if self._provider.security_token:
  125. key = self._provider.security_token_header
  126. headers[key] = self._provider.security_token
  127. string_to_sign = boto.utils.canonical_string(method, auth_path,
  128. headers, None,
  129. self._provider)
  130. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  131. b64_hmac = self.sign_string(string_to_sign)
  132. auth_hdr = self._provider.auth_header
  133. auth = ("%s %s:%s" % (auth_hdr, self._provider.access_key, b64_hmac))
  134. boto.log.debug('Signature:\n%s' % auth)
  135. headers['Authorization'] = auth
  136. class HmacAuthV2Handler(AuthHandler, HmacKeys):
  137. """
  138. Implements the simplified HMAC authorization used by CloudFront.
  139. """
  140. capability = ['hmac-v2', 'cloudfront']
  141. def __init__(self, host, config, provider):
  142. AuthHandler.__init__(self, host, config, provider)
  143. HmacKeys.__init__(self, host, config, provider)
  144. self._hmac_256 = None
  145. def update_provider(self, provider):
  146. super(HmacAuthV2Handler, self).update_provider(provider)
  147. self._hmac_256 = None
  148. def add_auth(self, http_request, **kwargs):
  149. headers = http_request.headers
  150. if 'Date' not in headers:
  151. headers['Date'] = formatdate(usegmt=True)
  152. if self._provider.security_token:
  153. key = self._provider.security_token_header
  154. headers[key] = self._provider.security_token
  155. b64_hmac = self.sign_string(headers['Date'])
  156. auth_hdr = self._provider.auth_header
  157. headers['Authorization'] = ("%s %s:%s" %
  158. (auth_hdr,
  159. self._provider.access_key, b64_hmac))
  160. class HmacAuthV3Handler(AuthHandler, HmacKeys):
  161. """Implements the new Version 3 HMAC authorization used by Route53."""
  162. capability = ['hmac-v3', 'route53', 'ses']
  163. def __init__(self, host, config, provider):
  164. AuthHandler.__init__(self, host, config, provider)
  165. HmacKeys.__init__(self, host, config, provider)
  166. def add_auth(self, http_request, **kwargs):
  167. headers = http_request.headers
  168. if 'Date' not in headers:
  169. headers['Date'] = formatdate(usegmt=True)
  170. if self._provider.security_token:
  171. key = self._provider.security_token_header
  172. headers[key] = self._provider.security_token
  173. b64_hmac = self.sign_string(headers['Date'])
  174. s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key
  175. s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac)
  176. headers['X-Amzn-Authorization'] = s
  177. class HmacAuthV3HTTPHandler(AuthHandler, HmacKeys):
  178. """
  179. Implements the new Version 3 HMAC authorization used by DynamoDB.
  180. """
  181. capability = ['hmac-v3-http']
  182. def __init__(self, host, config, provider):
  183. AuthHandler.__init__(self, host, config, provider)
  184. HmacKeys.__init__(self, host, config, provider)
  185. def headers_to_sign(self, http_request):
  186. """
  187. Select the headers from the request that need to be included
  188. in the StringToSign.
  189. """
  190. headers_to_sign = {'Host': self.host}
  191. for name, value in http_request.headers.items():
  192. lname = name.lower()
  193. if lname.startswith('x-amz'):
  194. headers_to_sign[name] = value
  195. return headers_to_sign
  196. def canonical_headers(self, headers_to_sign):
  197. """
  198. Return the headers that need to be included in the StringToSign
  199. in their canonical form by converting all header keys to lower
  200. case, sorting them in alphabetical order and then joining
  201. them into a string, separated by newlines.
  202. """
  203. l = sorted(['%s:%s' % (n.lower().strip(),
  204. headers_to_sign[n].strip()) for n in headers_to_sign])
  205. return '\n'.join(l)
  206. def string_to_sign(self, http_request):
  207. """
  208. Return the canonical StringToSign as well as a dict
  209. containing the original version of all headers that
  210. were included in the StringToSign.
  211. """
  212. headers_to_sign = self.headers_to_sign(http_request)
  213. canonical_headers = self.canonical_headers(headers_to_sign)
  214. string_to_sign = '\n'.join([http_request.method,
  215. http_request.auth_path,
  216. '',
  217. canonical_headers,
  218. '',
  219. http_request.body])
  220. return string_to_sign, headers_to_sign
  221. def add_auth(self, req, **kwargs):
  222. """
  223. Add AWS3 authentication to a request.
  224. :type req: :class`boto.connection.HTTPRequest`
  225. :param req: The HTTPRequest object.
  226. """
  227. # This could be a retry. Make sure the previous
  228. # authorization header is removed first.
  229. if 'X-Amzn-Authorization' in req.headers:
  230. del req.headers['X-Amzn-Authorization']
  231. req.headers['X-Amz-Date'] = formatdate(usegmt=True)
  232. if self._provider.security_token:
  233. req.headers['X-Amz-Security-Token'] = self._provider.security_token
  234. string_to_sign, headers_to_sign = self.string_to_sign(req)
  235. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  236. hash_value = sha256(string_to_sign.encode('utf-8')).digest()
  237. b64_hmac = self.sign_string(hash_value)
  238. s = "AWS3 AWSAccessKeyId=%s," % self._provider.access_key
  239. s += "Algorithm=%s," % self.algorithm()
  240. s += "SignedHeaders=%s," % ';'.join(headers_to_sign)
  241. s += "Signature=%s" % b64_hmac
  242. req.headers['X-Amzn-Authorization'] = s
  243. class HmacAuthV4Handler(AuthHandler, HmacKeys):
  244. """
  245. Implements the new Version 4 HMAC authorization.
  246. """
  247. capability = ['hmac-v4']
  248. def __init__(self, host, config, provider,
  249. service_name=None, region_name=None):
  250. AuthHandler.__init__(self, host, config, provider)
  251. HmacKeys.__init__(self, host, config, provider)
  252. # You can set the service_name and region_name to override the
  253. # values which would otherwise come from the endpoint, e.g.
  254. # <service>.<region>.amazonaws.com.
  255. self.service_name = service_name
  256. self.region_name = region_name
  257. def _sign(self, key, msg, hex=False):
  258. if not isinstance(key, bytes):
  259. key = key.encode('utf-8')
  260. if hex:
  261. sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
  262. else:
  263. sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
  264. return sig
  265. def headers_to_sign(self, http_request):
  266. """
  267. Select the headers from the request that need to be included
  268. in the StringToSign.
  269. """
  270. host_header_value = self.host_header(self.host, http_request)
  271. if http_request.headers.get('Host'):
  272. host_header_value = http_request.headers['Host']
  273. headers_to_sign = {'Host': host_header_value}
  274. for name, value in http_request.headers.items():
  275. lname = name.lower()
  276. if lname.startswith('x-amz'):
  277. if isinstance(value, bytes):
  278. value = value.decode('utf-8')
  279. headers_to_sign[name] = value
  280. return headers_to_sign
  281. def host_header(self, host, http_request):
  282. port = http_request.port
  283. secure = http_request.protocol == 'https'
  284. if ((port == 80 and not secure) or (port == 443 and secure)):
  285. return host
  286. return '%s:%s' % (host, port)
  287. def query_string(self, http_request):
  288. parameter_names = sorted(http_request.params.keys())
  289. pairs = []
  290. for pname in parameter_names:
  291. pval = boto.utils.get_utf8_value(http_request.params[pname])
  292. pairs.append(urllib.parse.quote(pname, safe='') + '=' +
  293. urllib.parse.quote(pval, safe='-_~'))
  294. return '&'.join(pairs)
  295. def canonical_query_string(self, http_request):
  296. # POST requests pass parameters in through the
  297. # http_request.body field.
  298. if http_request.method == 'POST':
  299. return ""
  300. l = []
  301. for param in sorted(http_request.params):
  302. value = boto.utils.get_utf8_value(http_request.params[param])
  303. l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
  304. urllib.parse.quote(value, safe='-_.~')))
  305. return '&'.join(l)
  306. def canonical_headers(self, headers_to_sign):
  307. """
  308. Return the headers that need to be included in the StringToSign
  309. in their canonical form by converting all header keys to lower
  310. case, sorting them in alphabetical order and then joining
  311. them into a string, separated by newlines.
  312. """
  313. canonical = []
  314. for header in headers_to_sign:
  315. c_name = header.lower().strip()
  316. raw_value = str(headers_to_sign[header])
  317. if '"' in raw_value:
  318. c_value = raw_value.strip()
  319. else:
  320. c_value = ' '.join(raw_value.strip().split())
  321. canonical.append('%s:%s' % (c_name, c_value))
  322. return '\n'.join(sorted(canonical))
  323. def signed_headers(self, headers_to_sign):
  324. l = ['%s' % n.lower().strip() for n in headers_to_sign]
  325. l = sorted(l)
  326. return ';'.join(l)
  327. def canonical_uri(self, http_request):
  328. path = http_request.auth_path
  329. # Normalize the path
  330. # in windows normpath('/') will be '\\' so we chane it back to '/'
  331. normalized = posixpath.normpath(path).replace('\\', '/')
  332. # Then urlencode whatever's left.
  333. encoded = urllib.parse.quote(normalized)
  334. if len(path) > 1 and path.endswith('/'):
  335. encoded += '/'
  336. return encoded
  337. def payload(self, http_request):
  338. body = http_request.body
  339. # If the body is a file like object, we can use
  340. # boto.utils.compute_hash, which will avoid reading
  341. # the entire body into memory.
  342. if hasattr(body, 'seek') and hasattr(body, 'read'):
  343. return boto.utils.compute_hash(body, hash_algorithm=sha256)[0]
  344. elif not isinstance(body, bytes):
  345. body = body.encode('utf-8')
  346. return sha256(body).hexdigest()
  347. def canonical_request(self, http_request):
  348. cr = [http_request.method.upper()]
  349. cr.append(self.canonical_uri(http_request))
  350. cr.append(self.canonical_query_string(http_request))
  351. headers_to_sign = self.headers_to_sign(http_request)
  352. cr.append(self.canonical_headers(headers_to_sign) + '\n')
  353. cr.append(self.signed_headers(headers_to_sign))
  354. cr.append(self.payload(http_request))
  355. return '\n'.join(cr)
  356. def scope(self, http_request):
  357. scope = [self._provider.access_key]
  358. scope.append(http_request.timestamp)
  359. scope.append(http_request.region_name)
  360. scope.append(http_request.service_name)
  361. scope.append('aws4_request')
  362. return '/'.join(scope)
  363. def split_host_parts(self, host):
  364. return host.split('.')
  365. def determine_region_name(self, host):
  366. parts = self.split_host_parts(host)
  367. if self.region_name is not None:
  368. region_name = self.region_name
  369. elif len(parts) > 1:
  370. if parts[1] == 'us-gov':
  371. region_name = 'us-gov-west-1'
  372. else:
  373. if len(parts) == 3:
  374. region_name = 'us-east-1'
  375. else:
  376. region_name = parts[1]
  377. else:
  378. region_name = parts[0]
  379. return region_name
  380. def determine_service_name(self, host):
  381. parts = self.split_host_parts(host)
  382. if self.service_name is not None:
  383. service_name = self.service_name
  384. else:
  385. service_name = parts[0]
  386. return service_name
  387. def credential_scope(self, http_request):
  388. scope = []
  389. http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
  390. scope.append(http_request.timestamp)
  391. # The service_name and region_name either come from:
  392. # * The service_name/region_name attrs or (if these values are None)
  393. # * parsed from the endpoint <service>.<region>.amazonaws.com.
  394. region_name = self.determine_region_name(http_request.host)
  395. service_name = self.determine_service_name(http_request.host)
  396. http_request.service_name = service_name
  397. http_request.region_name = region_name
  398. scope.append(http_request.region_name)
  399. scope.append(http_request.service_name)
  400. scope.append('aws4_request')
  401. return '/'.join(scope)
  402. def string_to_sign(self, http_request, canonical_request):
  403. """
  404. Return the canonical StringToSign as well as a dict
  405. containing the original version of all headers that
  406. were included in the StringToSign.
  407. """
  408. sts = ['AWS4-HMAC-SHA256']
  409. sts.append(http_request.headers['X-Amz-Date'])
  410. sts.append(self.credential_scope(http_request))
  411. sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
  412. return '\n'.join(sts)
  413. def signature(self, http_request, string_to_sign):
  414. key = self._provider.secret_key
  415. k_date = self._sign(('AWS4' + key).encode('utf-8'),
  416. http_request.timestamp)
  417. k_region = self._sign(k_date, http_request.region_name)
  418. k_service = self._sign(k_region, http_request.service_name)
  419. k_signing = self._sign(k_service, 'aws4_request')
  420. return self._sign(k_signing, string_to_sign, hex=True)
  421. def add_auth(self, req, **kwargs):
  422. """
  423. Add AWS4 authentication to a request.
  424. :type req: :class`boto.connection.HTTPRequest`
  425. :param req: The HTTPRequest object.
  426. """
  427. # This could be a retry. Make sure the previous
  428. # authorization header is removed first.
  429. if 'X-Amzn-Authorization' in req.headers:
  430. del req.headers['X-Amzn-Authorization']
  431. now = datetime.datetime.utcnow()
  432. req.headers['X-Amz-Date'] = now.strftime('%Y%m%dT%H%M%SZ')
  433. if self._provider.security_token:
  434. req.headers['X-Amz-Security-Token'] = self._provider.security_token
  435. qs = self.query_string(req)
  436. qs_to_post = qs
  437. # We do not want to include any params that were mangled into
  438. # the params if performing s3-sigv4 since it does not
  439. # belong in the body of a post for some requests. Mangled
  440. # refers to items in the query string URL being added to the
  441. # http response params. However, these params get added to
  442. # the body of the request, but the query string URL does not
  443. # belong in the body of the request. ``unmangled_resp`` is the
  444. # response that happened prior to the mangling. This ``unmangled_req``
  445. # kwarg will only appear for s3-sigv4.
  446. if 'unmangled_req' in kwargs:
  447. qs_to_post = self.query_string(kwargs['unmangled_req'])
  448. if qs_to_post and req.method == 'POST':
  449. # Stash request parameters into post body
  450. # before we generate the signature.
  451. req.body = qs_to_post
  452. req.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  453. req.headers['Content-Length'] = str(len(req.body))
  454. else:
  455. # Safe to modify req.path here since
  456. # the signature will use req.auth_path.
  457. req.path = req.path.split('?')[0]
  458. if qs:
  459. # Don't insert the '?' unless there's actually a query string
  460. req.path = req.path + '?' + qs
  461. canonical_request = self.canonical_request(req)
  462. boto.log.debug('CanonicalRequest:\n%s' % canonical_request)
  463. string_to_sign = self.string_to_sign(req, canonical_request)
  464. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  465. signature = self.signature(req, string_to_sign)
  466. boto.log.debug('Signature:\n%s' % signature)
  467. headers_to_sign = self.headers_to_sign(req)
  468. l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(req)]
  469. l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign))
  470. l.append('Signature=%s' % signature)
  471. req.headers['Authorization'] = ','.join(l)
  472. class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler):
  473. """
  474. Implements a variant of Version 4 HMAC authorization specific to S3.
  475. """
  476. capability = ['hmac-v4-s3']
  477. def __init__(self, *args, **kwargs):
  478. super(S3HmacAuthV4Handler, self).__init__(*args, **kwargs)
  479. if self.region_name:
  480. self.region_name = self.clean_region_name(self.region_name)
  481. def clean_region_name(self, region_name):
  482. if region_name.startswith('s3-'):
  483. return region_name[3:]
  484. return region_name
  485. def canonical_uri(self, http_request):
  486. # S3 does **NOT** do path normalization that SigV4 typically does.
  487. # Urlencode the path, **NOT** ``auth_path`` (because vhosting).
  488. path = urllib.parse.urlparse(http_request.path)
  489. # Because some quoting may have already been applied, let's back it out.
  490. unquoted = urllib.parse.unquote(path.path)
  491. # Requote, this time addressing all characters.
  492. encoded = urllib.parse.quote(unquoted, safe='/~')
  493. return encoded
  494. def canonical_query_string(self, http_request):
  495. # Note that we just do not return an empty string for
  496. # POST request. Query strings in url are included in canonical
  497. # query string.
  498. l = []
  499. for param in sorted(http_request.params):
  500. value = boto.utils.get_utf8_value(http_request.params[param])
  501. l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
  502. urllib.parse.quote(value, safe='-_.~')))
  503. return '&'.join(l)
  504. def host_header(self, host, http_request):
  505. port = http_request.port
  506. secure = http_request.protocol == 'https'
  507. if ((port == 80 and not secure) or (port == 443 and secure)):
  508. return http_request.host
  509. return '%s:%s' % (http_request.host, port)
  510. def headers_to_sign(self, http_request):
  511. """
  512. Select the headers from the request that need to be included
  513. in the StringToSign.
  514. """
  515. host_header_value = self.host_header(self.host, http_request)
  516. headers_to_sign = {'Host': host_header_value}
  517. for name, value in http_request.headers.items():
  518. lname = name.lower()
  519. # Hooray for the only difference! The main SigV4 signer only does
  520. # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything
  521. # signed, except for authorization itself.
  522. if lname not in ['authorization']:
  523. headers_to_sign[name] = value
  524. return headers_to_sign
  525. def determine_region_name(self, host):
  526. # S3's different format(s) of representing region/service from the
  527. # rest of AWS makes this hurt too.
  528. #
  529. # Possible domain formats:
  530. # - s3.amazonaws.com (Classic)
  531. # - s3-us-west-2.amazonaws.com (Specific region)
  532. # - bukkit.s3.amazonaws.com (Vhosted Classic)
  533. # - bukkit.s3-ap-northeast-1.amazonaws.com (Vhosted specific region)
  534. # - s3.cn-north-1.amazonaws.com.cn - (Beijing region)
  535. # - bukkit.s3.cn-north-1.amazonaws.com.cn - (Vhosted Beijing region)
  536. parts = self.split_host_parts(host)
  537. if self.region_name is not None:
  538. region_name = self.region_name
  539. else:
  540. # Classic URLs - s3-us-west-2.amazonaws.com
  541. if len(parts) == 3:
  542. region_name = self.clean_region_name(parts[0])
  543. # Special-case for Classic.
  544. if region_name == 's3':
  545. region_name = 'us-east-1'
  546. else:
  547. # Iterate over the parts in reverse order.
  548. for offset, part in enumerate(reversed(parts)):
  549. part = part.lower()
  550. # Look for the first thing starting with 's3'.
  551. # Until there's a ``.s3`` TLD, we should be OK. :P
  552. if part == 's3':
  553. # If it's by itself, the region is the previous part.
  554. region_name = parts[-offset]
  555. # Unless it's Vhosted classic
  556. if region_name == 'amazonaws':
  557. region_name = 'us-east-1'
  558. break
  559. elif part.startswith('s3-'):
  560. region_name = self.clean_region_name(part)
  561. break
  562. return region_name
  563. def determine_service_name(self, host):
  564. # Should this signing mechanism ever be used for anything else, this
  565. # will fail. Consider utilizing the logic from the parent class should
  566. # you find yourself here.
  567. return 's3'
  568. def mangle_path_and_params(self, req):
  569. """
  570. Returns a copy of the request object with fixed ``auth_path/params``
  571. attributes from the original.
  572. """
  573. modified_req = copy.copy(req)
  574. # Unlike the most other services, in S3, ``req.params`` isn't the only
  575. # source of query string parameters.
  576. # Because of the ``query_args``, we may already have a query string
  577. # **ON** the ``path/auth_path``.
  578. # Rip them apart, so the ``auth_path/params`` can be signed
  579. # appropriately.
  580. parsed_path = urllib.parse.urlparse(modified_req.auth_path)
  581. modified_req.auth_path = parsed_path.path
  582. if modified_req.params is None:
  583. modified_req.params = {}
  584. else:
  585. # To keep the original request object untouched. We must make
  586. # a copy of the params dictionary. Because the copy of the
  587. # original request directly refers to the params dictionary
  588. # of the original request.
  589. copy_params = req.params.copy()
  590. modified_req.params = copy_params
  591. raw_qs = parsed_path.query
  592. existing_qs = parse_qs_safe(
  593. raw_qs,
  594. keep_blank_values=True
  595. )
  596. # ``parse_qs`` will return lists. Don't do that unless there's a real,
  597. # live list provided.
  598. for key, value in existing_qs.items():
  599. if isinstance(value, (list, tuple)):
  600. if len(value) == 1:
  601. existing_qs[key] = value[0]
  602. modified_req.params.update(existing_qs)
  603. return modified_req
  604. def payload(self, http_request):
  605. if http_request.headers.get('x-amz-content-sha256'):
  606. return http_request.headers['x-amz-content-sha256']
  607. return super(S3HmacAuthV4Handler, self).payload(http_request)
  608. def add_auth(self, req, **kwargs):
  609. if 'x-amz-content-sha256' not in req.headers:
  610. if '_sha256' in req.headers:
  611. req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256')
  612. else:
  613. req.headers['x-amz-content-sha256'] = self.payload(req)
  614. updated_req = self.mangle_path_and_params(req)
  615. return super(S3HmacAuthV4Handler, self).add_auth(updated_req,
  616. unmangled_req=req,
  617. **kwargs)
  618. def presign(self, req, expires, iso_date=None):
  619. """
  620. Presign a request using SigV4 query params. Takes in an HTTP request
  621. and an expiration time in seconds and returns a URL.
  622. http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
  623. """
  624. if iso_date is None:
  625. iso_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
  626. region = self.determine_region_name(req.host)
  627. service = self.determine_service_name(req.host)
  628. params = {
  629. 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
  630. 'X-Amz-Credential': '%s/%s/%s/%s/aws4_request' % (
  631. self._provider.access_key,
  632. iso_date[:8],
  633. region,
  634. service
  635. ),
  636. 'X-Amz-Date': iso_date,
  637. 'X-Amz-Expires': expires,
  638. 'X-Amz-SignedHeaders': 'host'
  639. }
  640. if self._provider.security_token:
  641. params['X-Amz-Security-Token'] = self._provider.security_token
  642. headers_to_sign = self.headers_to_sign(req)
  643. l = sorted(['%s' % n.lower().strip() for n in headers_to_sign])
  644. params['X-Amz-SignedHeaders'] = ';'.join(l)
  645. req.params.update(params)
  646. cr = self.canonical_request(req)
  647. # We need to replace the payload SHA with a constant
  648. cr = '\n'.join(cr.split('\n')[:-1]) + '\nUNSIGNED-PAYLOAD'
  649. # Date header is expected for string_to_sign, but unused otherwise
  650. req.headers['X-Amz-Date'] = iso_date
  651. sts = self.string_to_sign(req, cr)
  652. signature = self.signature(req, sts)
  653. # Add signature to params now that we have it
  654. req.params['X-Amz-Signature'] = signature
  655. return '%s://%s%s?%s' % (req.protocol, req.host, req.path,
  656. urllib.parse.urlencode(req.params))
  657. class STSAnonHandler(AuthHandler):
  658. """
  659. Provides pure query construction (no actual signing).
  660. Used for making anonymous STS request for operations like
  661. ``assume_role_with_web_identity``.
  662. """
  663. capability = ['sts-anon']
  664. def _escape_value(self, value):
  665. # This is changed from a previous version because this string is
  666. # being passed to the query string and query strings must
  667. # be url encoded. In particular STS requires the saml_response to
  668. # be urlencoded when calling assume_role_with_saml.
  669. return urllib.parse.quote(value)
  670. def _build_query_string(self, params):
  671. keys = list(params.keys())
  672. keys.sort(key=lambda x: x.lower())
  673. pairs = []
  674. for key in keys:
  675. val = boto.utils.get_utf8_value(params[key])
  676. pairs.append(key + '=' + self._escape_value(val.decode('utf-8')))
  677. return '&'.join(pairs)
  678. def add_auth(self, http_request, **kwargs):
  679. headers = http_request.headers
  680. qs = self._build_query_string(
  681. http_request.params
  682. )
  683. boto.log.debug('query_string in body: %s' % qs)
  684. headers['Content-Type'] = 'application/x-www-form-urlencoded'
  685. # This will be a POST so the query string should go into the body
  686. # as opposed to being in the uri
  687. http_request.body = qs
  688. class QuerySignatureHelper(HmacKeys):
  689. """
  690. Helper for Query signature based Auth handler.
  691. Concrete sub class need to implement _calc_sigature method.
  692. """
  693. def add_auth(self, http_request, **kwargs):
  694. headers = http_request.headers
  695. params = http_request.params
  696. params['AWSAccessKeyId'] = self._provider.access_key
  697. params['SignatureVersion'] = self.SignatureVersion
  698. params['Timestamp'] = boto.utils.get_ts()
  699. qs, signature = self._calc_signature(
  700. http_request.params, http_request.method,
  701. http_request.auth_path, http_request.host)
  702. boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
  703. if http_request.method == 'POST':
  704. headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  705. http_request.body = qs + '&Signature=' + urllib.parse.quote_plus(signature)
  706. http_request.headers['Content-Length'] = str(len(http_request.body))
  707. else:
  708. http_request.body = ''
  709. # if this is a retried request, the qs from the previous try will
  710. # already be there, we need to get rid of that and rebuild it
  711. http_request.path = http_request.path.split('?')[0]
  712. http_request.path = (http_request.path + '?' + qs +
  713. '&Signature=' + urllib.parse.quote_plus(signature))
  714. class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler):
  715. """Provides Signature V0 Signing"""
  716. SignatureVersion = 0
  717. capability = ['sign-v0']
  718. def _calc_signature(self, params, *args):
  719. boto.log.debug('using _calc_signature_0')
  720. hmac = self._get_hmac()
  721. s = params['Action'] + params['Timestamp']
  722. hmac.update(s.encode('utf-8'))
  723. keys = params.keys()
  724. keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
  725. pairs = []
  726. for key in keys:
  727. val = boto.utils.get_utf8_value(params[key])
  728. pairs.append(key + '=' + urllib.parse.quote(val))
  729. qs = '&'.join(pairs)
  730. return (qs, base64.b64encode(hmac.digest()))
  731. class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler):
  732. """
  733. Provides Query Signature V1 Authentication.
  734. """
  735. SignatureVersion = 1
  736. capability = ['sign-v1', 'mturk']
  737. def __init__(self, *args, **kw):
  738. QuerySignatureHelper.__init__(self, *args, **kw)
  739. AuthHandler.__init__(self, *args, **kw)
  740. self._hmac_256 = None
  741. def _calc_signature(self, params, *args):
  742. boto.log.debug('using _calc_signature_1')
  743. hmac = self._get_hmac()
  744. keys = list(params.keys())
  745. keys.sort(key=lambda x: x.lower())
  746. pairs = []
  747. for key in keys:
  748. hmac.update(key.encode('utf-8'))
  749. val = boto.utils.get_utf8_value(params[key])
  750. hmac.update(val)
  751. pairs.append(key + '=' + urllib.parse.quote(val))
  752. qs = '&'.join(pairs)
  753. return (qs, base64.b64encode(hmac.digest()))
  754. class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler):
  755. """Provides Query Signature V2 Authentication."""
  756. SignatureVersion = 2
  757. capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs',
  758. 'sdb', 'iam', 'rds', 'sns', 'sqs', 'cloudformation']
  759. def _calc_signature(self, params, verb, path, server_name):
  760. boto.log.debug('using _calc_signature_2')
  761. string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path)
  762. hmac = self._get_hmac()
  763. params['SignatureMethod'] = self.algorithm()
  764. if self._provider.security_token:
  765. params['SecurityToken'] = self._provider.security_token
  766. keys = sorted(params.keys())
  767. pairs = []
  768. for key in keys:
  769. val = boto.utils.get_utf8_value(params[key])
  770. pairs.append(urllib.parse.quote(key, safe='') + '=' +
  771. urllib.parse.quote(val, safe='-_~'))
  772. qs = '&'.join(pairs)
  773. boto.log.debug('query string: %s' % qs)
  774. string_to_sign += qs
  775. boto.log.debug('string_to_sign: %s' % string_to_sign)
  776. hmac.update(string_to_sign.encode('utf-8'))
  777. b64 = base64.b64encode(hmac.digest())
  778. boto.log.debug('len(b64)=%d' % len(b64))
  779. boto.log.debug('base64 encoded digest: %s' % b64)
  780. return (qs, b64)
  781. class POSTPathQSV2AuthHandler(QuerySignatureV2AuthHandler, AuthHandler):
  782. """
  783. Query Signature V2 Authentication relocating signed query
  784. into the path and allowing POST requests with Content-Types.
  785. """
  786. capability = ['mws']
  787. def add_auth(self, req, **kwargs):
  788. req.params['AWSAccessKeyId'] = self._provider.access_key
  789. req.params['SignatureVersion'] = self.SignatureVersion
  790. req.params['Timestamp'] = boto.utils.get_ts()
  791. qs, signature = self._calc_signature(req.params, req.method,
  792. req.auth_path, req.host)
  793. boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
  794. if req.method == 'POST':
  795. req.headers['Content-Length'] = str(len(req.body))
  796. req.headers['Content-Type'] = req.headers.get('Content-Type',
  797. 'text/plain')
  798. else:
  799. req.body = ''
  800. # if this is a retried req, the qs from the previous try will
  801. # already be there, we need to get rid of that and rebuild it
  802. req.path = req.path.split('?')[0]
  803. req.path = (req.path + '?' + qs +
  804. '&Signature=' + urllib.parse.quote_plus(signature))
  805. def get_auth_handler(host, config, provider, requested_capability=None):
  806. """Finds an AuthHandler that is ready to authenticate.
  807. Lists through all the registered AuthHandlers to find one that is willing
  808. to handle for the requested capabilities, config and provider.
  809. :type host: string
  810. :param host: The name of the host
  811. :type config:
  812. :param config:
  813. :type provider:
  814. :param provider:
  815. Returns:
  816. An implementation of AuthHandler.
  817. Raises:
  818. boto.exception.NoAuthHandlerFound
  819. """
  820. ready_handlers = []
  821. auth_handlers = boto.plugin.get_plugin(AuthHandler, requested_capability)
  822. for handler in auth_handlers:
  823. try:
  824. ready_handlers.append(handler(host, config, provider))
  825. except boto.auth_handler.NotReadyToAuthenticate:
  826. pass
  827. if not ready_handlers:
  828. checked_handlers = auth_handlers
  829. names = [handler.__name__ for handler in checked_handlers]
  830. raise boto.exception.NoAuthHandlerFound(
  831. 'No handler was ready to authenticate. %d handlers were checked.'
  832. ' %s '
  833. 'Check your credentials' % (len(names), str(names)))
  834. # We select the last ready auth handler that was loaded, to allow users to
  835. # customize how auth works in environments where there are shared boto
  836. # config files (e.g., /etc/boto.cfg and ~/.boto): The more general,
  837. # system-wide shared configs should be loaded first, and the user's
  838. # customizations loaded last. That way, for example, the system-wide
  839. # config might include a plugin_directory that includes a service account
  840. # auth plugin shared by all users of a Google Compute Engine instance
  841. # (allowing sharing of non-user data between various services), and the
  842. # user could override this with a .boto config that includes user-specific
  843. # credentials (for access to user data).
  844. return ready_handlers[-1]
  845. def detect_potential_sigv4(func):
  846. def _wrapper(self):
  847. if os.environ.get('EC2_USE_SIGV4', False):
  848. return ['hmac-v4']
  849. if boto.config.get('ec2', 'use-sigv4', False):
  850. return ['hmac-v4']
  851. if hasattr(self, 'region'):
  852. # If you're making changes here, you should also check
  853. # ``boto/iam/connection.py``, as several things there are also
  854. # endpoint-related.
  855. if getattr(self.region, 'endpoint', ''):
  856. for test in SIGV4_DETECT:
  857. if test in self.region.endpoint:
  858. return ['hmac-v4']
  859. return func(self)
  860. return _wrapper
  861. def detect_potential_s3sigv4(func):
  862. def _wrapper(self):
  863. if os.environ.get('S3_USE_SIGV4', False):
  864. return ['hmac-v4-s3']
  865. if boto.config.get('s3', 'use-sigv4', False):
  866. return ['hmac-v4-s3']
  867. if hasattr(self, 'host'):
  868. # If you're making changes here, you should also check
  869. # ``boto/iam/connection.py``, as several things there are also
  870. # endpoint-related.
  871. for test in SIGV4_DETECT:
  872. if test in self.host:
  873. return ['hmac-v4-s3']
  874. return func(self)
  875. return _wrapper