auth.py 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040
  1. # Copyright 2010 Google Inc.
  2. # Copyright (c) 2011 Mitch Garnaat http://garnaat.org/
  3. # Copyright (c) 2011, Eucalyptus Systems, Inc.
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a
  6. # copy of this software and associated documentation files (the
  7. # "Software"), to deal in the Software without restriction, including
  8. # without limitation the rights to use, copy, modify, merge, publish, dis-
  9. # tribute, sublicense, and/or sell copies of the Software, and to permit
  10. # persons to whom the Software is furnished to do so, subject to the fol-
  11. # lowing conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included
  14. # in all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17. # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  18. # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  19. # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  20. # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22. # IN THE SOFTWARE.
  23. """
  24. Handles authentication required to AWS and GS
  25. """
  26. import base64
  27. import boto
  28. import boto.auth_handler
  29. import boto.exception
  30. import boto.plugin
  31. import boto.utils
  32. import copy
  33. import datetime
  34. from email.utils import formatdate
  35. import hmac
  36. import os
  37. import posixpath
  38. from boto.compat import urllib, encodebytes
  39. from boto.auth_handler import AuthHandler
  40. from boto.exception import BotoClientError
  41. try:
  42. from hashlib import sha1 as sha
  43. from hashlib import sha256 as sha256
  44. except ImportError:
  45. import sha
  46. sha256 = None
  47. # Region detection strings to determine if SigV4 should be used
  48. # by default.
  49. SIGV4_DETECT = [
  50. '.cn-',
  51. # In eu-central we support both host styles for S3
  52. '.eu-central',
  53. '-eu-central',
  54. ]
  55. class HmacKeys(object):
  56. """Key based Auth handler helper."""
  57. def __init__(self, host, config, provider):
  58. if provider.access_key is None or provider.secret_key is None:
  59. raise boto.auth_handler.NotReadyToAuthenticate()
  60. self.host = host
  61. self.update_provider(provider)
  62. def update_provider(self, provider):
  63. self._provider = provider
  64. self._hmac = hmac.new(self._provider.secret_key.encode('utf-8'),
  65. digestmod=sha)
  66. if sha256:
  67. self._hmac_256 = hmac.new(self._provider.secret_key.encode('utf-8'),
  68. digestmod=sha256)
  69. else:
  70. self._hmac_256 = None
  71. def algorithm(self):
  72. if self._hmac_256:
  73. return 'HmacSHA256'
  74. else:
  75. return 'HmacSHA1'
  76. def _get_hmac(self):
  77. if self._hmac_256:
  78. digestmod = sha256
  79. else:
  80. digestmod = sha
  81. return hmac.new(self._provider.secret_key.encode('utf-8'),
  82. digestmod=digestmod)
  83. def sign_string(self, string_to_sign):
  84. new_hmac = self._get_hmac()
  85. new_hmac.update(string_to_sign.encode('utf-8'))
  86. return encodebytes(new_hmac.digest()).decode('utf-8').strip()
  87. def __getstate__(self):
  88. pickled_dict = copy.copy(self.__dict__)
  89. del pickled_dict['_hmac']
  90. del pickled_dict['_hmac_256']
  91. return pickled_dict
  92. def __setstate__(self, dct):
  93. self.__dict__ = dct
  94. self.update_provider(self._provider)
  95. class AnonAuthHandler(AuthHandler, HmacKeys):
  96. """
  97. Implements Anonymous requests.
  98. """
  99. capability = ['anon']
  100. def __init__(self, host, config, provider):
  101. super(AnonAuthHandler, self).__init__(host, config, provider)
  102. def add_auth(self, http_request, **kwargs):
  103. pass
  104. class HmacAuthV1Handler(AuthHandler, HmacKeys):
  105. """ Implements the HMAC request signing used by S3 and GS."""
  106. capability = ['hmac-v1', 's3']
  107. def __init__(self, host, config, provider):
  108. AuthHandler.__init__(self, host, config, provider)
  109. HmacKeys.__init__(self, host, config, provider)
  110. self._hmac_256 = None
  111. def update_provider(self, provider):
  112. super(HmacAuthV1Handler, self).update_provider(provider)
  113. self._hmac_256 = None
  114. def add_auth(self, http_request, **kwargs):
  115. headers = http_request.headers
  116. method = http_request.method
  117. auth_path = http_request.auth_path
  118. if 'Date' not in headers:
  119. headers['Date'] = formatdate(usegmt=True)
  120. if self._provider.security_token:
  121. key = self._provider.security_token_header
  122. headers[key] = self._provider.security_token
  123. string_to_sign = boto.utils.canonical_string(method, auth_path,
  124. headers, None,
  125. self._provider)
  126. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  127. b64_hmac = self.sign_string(string_to_sign)
  128. auth_hdr = self._provider.auth_header
  129. auth = ("%s %s:%s" % (auth_hdr, self._provider.access_key, b64_hmac))
  130. boto.log.debug('Signature:\n%s' % auth)
  131. headers['Authorization'] = auth
  132. class HmacAuthV2Handler(AuthHandler, HmacKeys):
  133. """
  134. Implements the simplified HMAC authorization used by CloudFront.
  135. """
  136. capability = ['hmac-v2', 'cloudfront']
  137. def __init__(self, host, config, provider):
  138. AuthHandler.__init__(self, host, config, provider)
  139. HmacKeys.__init__(self, host, config, provider)
  140. self._hmac_256 = None
  141. def update_provider(self, provider):
  142. super(HmacAuthV2Handler, self).update_provider(provider)
  143. self._hmac_256 = None
  144. def add_auth(self, http_request, **kwargs):
  145. headers = http_request.headers
  146. if 'Date' not in headers:
  147. headers['Date'] = formatdate(usegmt=True)
  148. if self._provider.security_token:
  149. key = self._provider.security_token_header
  150. headers[key] = self._provider.security_token
  151. b64_hmac = self.sign_string(headers['Date'])
  152. auth_hdr = self._provider.auth_header
  153. headers['Authorization'] = ("%s %s:%s" %
  154. (auth_hdr,
  155. self._provider.access_key, b64_hmac))
  156. class HmacAuthV3Handler(AuthHandler, HmacKeys):
  157. """Implements the new Version 3 HMAC authorization used by Route53."""
  158. capability = ['hmac-v3', 'route53', 'ses']
  159. def __init__(self, host, config, provider):
  160. AuthHandler.__init__(self, host, config, provider)
  161. HmacKeys.__init__(self, host, config, provider)
  162. def add_auth(self, http_request, **kwargs):
  163. headers = http_request.headers
  164. if 'Date' not in headers:
  165. headers['Date'] = formatdate(usegmt=True)
  166. if self._provider.security_token:
  167. key = self._provider.security_token_header
  168. headers[key] = self._provider.security_token
  169. b64_hmac = self.sign_string(headers['Date'])
  170. s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key
  171. s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac)
  172. headers['X-Amzn-Authorization'] = s
  173. class HmacAuthV3HTTPHandler(AuthHandler, HmacKeys):
  174. """
  175. Implements the new Version 3 HMAC authorization used by DynamoDB.
  176. """
  177. capability = ['hmac-v3-http']
  178. def __init__(self, host, config, provider):
  179. AuthHandler.__init__(self, host, config, provider)
  180. HmacKeys.__init__(self, host, config, provider)
  181. def headers_to_sign(self, http_request):
  182. """
  183. Select the headers from the request that need to be included
  184. in the StringToSign.
  185. """
  186. headers_to_sign = {'Host': self.host}
  187. for name, value in http_request.headers.items():
  188. lname = name.lower()
  189. if lname.startswith('x-amz'):
  190. headers_to_sign[name] = value
  191. return headers_to_sign
  192. def canonical_headers(self, headers_to_sign):
  193. """
  194. Return the headers that need to be included in the StringToSign
  195. in their canonical form by converting all header keys to lower
  196. case, sorting them in alphabetical order and then joining
  197. them into a string, separated by newlines.
  198. """
  199. l = sorted(['%s:%s' % (n.lower().strip(),
  200. headers_to_sign[n].strip()) for n in headers_to_sign])
  201. return '\n'.join(l)
  202. def string_to_sign(self, http_request):
  203. """
  204. Return the canonical StringToSign as well as a dict
  205. containing the original version of all headers that
  206. were included in the StringToSign.
  207. """
  208. headers_to_sign = self.headers_to_sign(http_request)
  209. canonical_headers = self.canonical_headers(headers_to_sign)
  210. string_to_sign = '\n'.join([http_request.method,
  211. http_request.auth_path,
  212. '',
  213. canonical_headers,
  214. '',
  215. http_request.body])
  216. return string_to_sign, headers_to_sign
  217. def add_auth(self, req, **kwargs):
  218. """
  219. Add AWS3 authentication to a request.
  220. :type req: :class`boto.connection.HTTPRequest`
  221. :param req: The HTTPRequest object.
  222. """
  223. # This could be a retry. Make sure the previous
  224. # authorization header is removed first.
  225. if 'X-Amzn-Authorization' in req.headers:
  226. del req.headers['X-Amzn-Authorization']
  227. req.headers['X-Amz-Date'] = formatdate(usegmt=True)
  228. if self._provider.security_token:
  229. req.headers['X-Amz-Security-Token'] = self._provider.security_token
  230. string_to_sign, headers_to_sign = self.string_to_sign(req)
  231. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  232. hash_value = sha256(string_to_sign.encode('utf-8')).digest()
  233. b64_hmac = self.sign_string(hash_value)
  234. s = "AWS3 AWSAccessKeyId=%s," % self._provider.access_key
  235. s += "Algorithm=%s," % self.algorithm()
  236. s += "SignedHeaders=%s," % ';'.join(headers_to_sign)
  237. s += "Signature=%s" % b64_hmac
  238. req.headers['X-Amzn-Authorization'] = s
  239. class HmacAuthV4Handler(AuthHandler, HmacKeys):
  240. """
  241. Implements the new Version 4 HMAC authorization.
  242. """
  243. capability = ['hmac-v4']
  244. def __init__(self, host, config, provider,
  245. service_name=None, region_name=None):
  246. AuthHandler.__init__(self, host, config, provider)
  247. HmacKeys.__init__(self, host, config, provider)
  248. # You can set the service_name and region_name to override the
  249. # values which would otherwise come from the endpoint, e.g.
  250. # <service>.<region>.amazonaws.com.
  251. self.service_name = service_name
  252. self.region_name = region_name
  253. def _sign(self, key, msg, hex=False):
  254. if not isinstance(key, bytes):
  255. key = key.encode('utf-8')
  256. if hex:
  257. sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
  258. else:
  259. sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
  260. return sig
  261. def headers_to_sign(self, http_request):
  262. """
  263. Select the headers from the request that need to be included
  264. in the StringToSign.
  265. """
  266. host_header_value = self.host_header(self.host, http_request)
  267. if http_request.headers.get('Host'):
  268. host_header_value = http_request.headers['Host']
  269. headers_to_sign = {'Host': host_header_value}
  270. for name, value in http_request.headers.items():
  271. lname = name.lower()
  272. if lname.startswith('x-amz'):
  273. if isinstance(value, bytes):
  274. value = value.decode('utf-8')
  275. headers_to_sign[name] = value
  276. return headers_to_sign
  277. def host_header(self, host, http_request):
  278. port = http_request.port
  279. secure = http_request.protocol == 'https'
  280. if ((port == 80 and not secure) or (port == 443 and secure)):
  281. return host
  282. return '%s:%s' % (host, port)
  283. def query_string(self, http_request):
  284. parameter_names = sorted(http_request.params.keys())
  285. pairs = []
  286. for pname in parameter_names:
  287. pval = boto.utils.get_utf8_value(http_request.params[pname])
  288. pairs.append(urllib.parse.quote(pname, safe='') + '=' +
  289. urllib.parse.quote(pval, safe='-_~'))
  290. return '&'.join(pairs)
  291. def canonical_query_string(self, http_request):
  292. # POST requests pass parameters in through the
  293. # http_request.body field.
  294. if http_request.method == 'POST':
  295. return ""
  296. l = []
  297. for param in sorted(http_request.params):
  298. value = boto.utils.get_utf8_value(http_request.params[param])
  299. l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
  300. urllib.parse.quote(value, safe='-_.~')))
  301. return '&'.join(l)
  302. def canonical_headers(self, headers_to_sign):
  303. """
  304. Return the headers that need to be included in the StringToSign
  305. in their canonical form by converting all header keys to lower
  306. case, sorting them in alphabetical order and then joining
  307. them into a string, separated by newlines.
  308. """
  309. canonical = []
  310. for header in headers_to_sign:
  311. c_name = header.lower().strip()
  312. raw_value = str(headers_to_sign[header])
  313. if '"' in raw_value:
  314. c_value = raw_value.strip()
  315. else:
  316. c_value = ' '.join(raw_value.strip().split())
  317. canonical.append('%s:%s' % (c_name, c_value))
  318. return '\n'.join(sorted(canonical))
  319. def signed_headers(self, headers_to_sign):
  320. l = ['%s' % n.lower().strip() for n in headers_to_sign]
  321. l = sorted(l)
  322. return ';'.join(l)
  323. def canonical_uri(self, http_request):
  324. path = http_request.auth_path
  325. # Normalize the path
  326. # in windows normpath('/') will be '\\' so we chane it back to '/'
  327. normalized = posixpath.normpath(path).replace('\\', '/')
  328. # Then urlencode whatever's left.
  329. encoded = urllib.parse.quote(normalized)
  330. if len(path) > 1 and path.endswith('/'):
  331. encoded += '/'
  332. return encoded
  333. def payload(self, http_request):
  334. body = http_request.body
  335. # If the body is a file like object, we can use
  336. # boto.utils.compute_hash, which will avoid reading
  337. # the entire body into memory.
  338. if hasattr(body, 'seek') and hasattr(body, 'read'):
  339. return boto.utils.compute_hash(body, hash_algorithm=sha256)[0]
  340. elif not isinstance(body, bytes):
  341. body = body.encode('utf-8')
  342. return sha256(body).hexdigest()
  343. def canonical_request(self, http_request):
  344. cr = [http_request.method.upper()]
  345. cr.append(self.canonical_uri(http_request))
  346. cr.append(self.canonical_query_string(http_request))
  347. headers_to_sign = self.headers_to_sign(http_request)
  348. cr.append(self.canonical_headers(headers_to_sign) + '\n')
  349. cr.append(self.signed_headers(headers_to_sign))
  350. cr.append(self.payload(http_request))
  351. return '\n'.join(cr)
  352. def scope(self, http_request):
  353. scope = [self._provider.access_key]
  354. scope.append(http_request.timestamp)
  355. scope.append(http_request.region_name)
  356. scope.append(http_request.service_name)
  357. scope.append('aws4_request')
  358. return '/'.join(scope)
  359. def split_host_parts(self, host):
  360. return host.split('.')
  361. def determine_region_name(self, host):
  362. parts = self.split_host_parts(host)
  363. if self.region_name is not None:
  364. region_name = self.region_name
  365. elif len(parts) > 1:
  366. if parts[1] == 'us-gov':
  367. region_name = 'us-gov-west-1'
  368. else:
  369. if len(parts) == 3:
  370. region_name = 'us-east-1'
  371. else:
  372. region_name = parts[1]
  373. else:
  374. region_name = parts[0]
  375. return region_name
  376. def determine_service_name(self, host):
  377. parts = self.split_host_parts(host)
  378. if self.service_name is not None:
  379. service_name = self.service_name
  380. else:
  381. service_name = parts[0]
  382. return service_name
  383. def credential_scope(self, http_request):
  384. scope = []
  385. http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
  386. scope.append(http_request.timestamp)
  387. # The service_name and region_name either come from:
  388. # * The service_name/region_name attrs or (if these values are None)
  389. # * parsed from the endpoint <service>.<region>.amazonaws.com.
  390. region_name = self.determine_region_name(http_request.host)
  391. service_name = self.determine_service_name(http_request.host)
  392. http_request.service_name = service_name
  393. http_request.region_name = region_name
  394. scope.append(http_request.region_name)
  395. scope.append(http_request.service_name)
  396. scope.append('aws4_request')
  397. return '/'.join(scope)
  398. def string_to_sign(self, http_request, canonical_request):
  399. """
  400. Return the canonical StringToSign as well as a dict
  401. containing the original version of all headers that
  402. were included in the StringToSign.
  403. """
  404. sts = ['AWS4-HMAC-SHA256']
  405. sts.append(http_request.headers['X-Amz-Date'])
  406. sts.append(self.credential_scope(http_request))
  407. sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
  408. return '\n'.join(sts)
  409. def signature(self, http_request, string_to_sign):
  410. key = self._provider.secret_key
  411. k_date = self._sign(('AWS4' + key).encode('utf-8'),
  412. http_request.timestamp)
  413. k_region = self._sign(k_date, http_request.region_name)
  414. k_service = self._sign(k_region, http_request.service_name)
  415. k_signing = self._sign(k_service, 'aws4_request')
  416. return self._sign(k_signing, string_to_sign, hex=True)
  417. def add_auth(self, req, **kwargs):
  418. """
  419. Add AWS4 authentication to a request.
  420. :type req: :class`boto.connection.HTTPRequest`
  421. :param req: The HTTPRequest object.
  422. """
  423. # This could be a retry. Make sure the previous
  424. # authorization header is removed first.
  425. if 'X-Amzn-Authorization' in req.headers:
  426. del req.headers['X-Amzn-Authorization']
  427. now = datetime.datetime.utcnow()
  428. req.headers['X-Amz-Date'] = now.strftime('%Y%m%dT%H%M%SZ')
  429. if self._provider.security_token:
  430. req.headers['X-Amz-Security-Token'] = self._provider.security_token
  431. qs = self.query_string(req)
  432. qs_to_post = qs
  433. # We do not want to include any params that were mangled into
  434. # the params if performing s3-sigv4 since it does not
  435. # belong in the body of a post for some requests. Mangled
  436. # refers to items in the query string URL being added to the
  437. # http response params. However, these params get added to
  438. # the body of the request, but the query string URL does not
  439. # belong in the body of the request. ``unmangled_resp`` is the
  440. # response that happened prior to the mangling. This ``unmangled_req``
  441. # kwarg will only appear for s3-sigv4.
  442. if 'unmangled_req' in kwargs:
  443. qs_to_post = self.query_string(kwargs['unmangled_req'])
  444. if qs_to_post and req.method == 'POST':
  445. # Stash request parameters into post body
  446. # before we generate the signature.
  447. req.body = qs_to_post
  448. req.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  449. req.headers['Content-Length'] = str(len(req.body))
  450. else:
  451. # Safe to modify req.path here since
  452. # the signature will use req.auth_path.
  453. req.path = req.path.split('?')[0]
  454. if qs:
  455. # Don't insert the '?' unless there's actually a query string
  456. req.path = req.path + '?' + qs
  457. canonical_request = self.canonical_request(req)
  458. boto.log.debug('CanonicalRequest:\n%s' % canonical_request)
  459. string_to_sign = self.string_to_sign(req, canonical_request)
  460. boto.log.debug('StringToSign:\n%s' % string_to_sign)
  461. signature = self.signature(req, string_to_sign)
  462. boto.log.debug('Signature:\n%s' % signature)
  463. headers_to_sign = self.headers_to_sign(req)
  464. l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(req)]
  465. l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign))
  466. l.append('Signature=%s' % signature)
  467. req.headers['Authorization'] = ','.join(l)
  468. class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler):
  469. """
  470. Implements a variant of Version 4 HMAC authorization specific to S3.
  471. """
  472. capability = ['hmac-v4-s3']
  473. def __init__(self, *args, **kwargs):
  474. super(S3HmacAuthV4Handler, self).__init__(*args, **kwargs)
  475. if self.region_name:
  476. self.region_name = self.clean_region_name(self.region_name)
  477. def clean_region_name(self, region_name):
  478. if region_name.startswith('s3-'):
  479. return region_name[3:]
  480. return region_name
  481. def canonical_uri(self, http_request):
  482. # S3 does **NOT** do path normalization that SigV4 typically does.
  483. # Urlencode the path, **NOT** ``auth_path`` (because vhosting).
  484. path = urllib.parse.urlparse(http_request.path)
  485. # Because some quoting may have already been applied, let's back it out.
  486. unquoted = urllib.parse.unquote(path.path)
  487. # Requote, this time addressing all characters.
  488. encoded = urllib.parse.quote(unquoted)
  489. return encoded
  490. def canonical_query_string(self, http_request):
  491. # Note that we just do not return an empty string for
  492. # POST request. Query strings in url are included in canonical
  493. # query string.
  494. l = []
  495. for param in sorted(http_request.params):
  496. value = boto.utils.get_utf8_value(http_request.params[param])
  497. l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
  498. urllib.parse.quote(value, safe='-_.~')))
  499. return '&'.join(l)
  500. def host_header(self, host, http_request):
  501. port = http_request.port
  502. secure = http_request.protocol == 'https'
  503. if ((port == 80 and not secure) or (port == 443 and secure)):
  504. return http_request.host
  505. return '%s:%s' % (http_request.host, port)
  506. def headers_to_sign(self, http_request):
  507. """
  508. Select the headers from the request that need to be included
  509. in the StringToSign.
  510. """
  511. host_header_value = self.host_header(self.host, http_request)
  512. headers_to_sign = {'Host': host_header_value}
  513. for name, value in http_request.headers.items():
  514. lname = name.lower()
  515. # Hooray for the only difference! The main SigV4 signer only does
  516. # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything
  517. # signed, except for authorization itself.
  518. if lname not in ['authorization']:
  519. headers_to_sign[name] = value
  520. return headers_to_sign
  521. def determine_region_name(self, host):
  522. # S3's different format(s) of representing region/service from the
  523. # rest of AWS makes this hurt too.
  524. #
  525. # Possible domain formats:
  526. # - s3.amazonaws.com (Classic)
  527. # - s3-us-west-2.amazonaws.com (Specific region)
  528. # - bukkit.s3.amazonaws.com (Vhosted Classic)
  529. # - bukkit.s3-ap-northeast-1.amazonaws.com (Vhosted specific region)
  530. # - s3.cn-north-1.amazonaws.com.cn - (Beijing region)
  531. # - bukkit.s3.cn-north-1.amazonaws.com.cn - (Vhosted Beijing region)
  532. parts = self.split_host_parts(host)
  533. if self.region_name is not None:
  534. region_name = self.region_name
  535. else:
  536. # Classic URLs - s3-us-west-2.amazonaws.com
  537. if len(parts) == 3:
  538. region_name = self.clean_region_name(parts[0])
  539. # Special-case for Classic.
  540. if region_name == 's3':
  541. region_name = 'us-east-1'
  542. else:
  543. # Iterate over the parts in reverse order.
  544. for offset, part in enumerate(reversed(parts)):
  545. part = part.lower()
  546. # Look for the first thing starting with 's3'.
  547. # Until there's a ``.s3`` TLD, we should be OK. :P
  548. if part == 's3':
  549. # If it's by itself, the region is the previous part.
  550. region_name = parts[-offset]
  551. # Unless it's Vhosted classic
  552. if region_name == 'amazonaws':
  553. region_name = 'us-east-1'
  554. break
  555. elif part.startswith('s3-'):
  556. region_name = self.clean_region_name(part)
  557. break
  558. return region_name
  559. def determine_service_name(self, host):
  560. # Should this signing mechanism ever be used for anything else, this
  561. # will fail. Consider utilizing the logic from the parent class should
  562. # you find yourself here.
  563. return 's3'
  564. def mangle_path_and_params(self, req):
  565. """
  566. Returns a copy of the request object with fixed ``auth_path/params``
  567. attributes from the original.
  568. """
  569. modified_req = copy.copy(req)
  570. # Unlike the most other services, in S3, ``req.params`` isn't the only
  571. # source of query string parameters.
  572. # Because of the ``query_args``, we may already have a query string
  573. # **ON** the ``path/auth_path``.
  574. # Rip them apart, so the ``auth_path/params`` can be signed
  575. # appropriately.
  576. parsed_path = urllib.parse.urlparse(modified_req.auth_path)
  577. modified_req.auth_path = parsed_path.path
  578. if modified_req.params is None:
  579. modified_req.params = {}
  580. else:
  581. # To keep the original request object untouched. We must make
  582. # a copy of the params dictionary. Because the copy of the
  583. # original request directly refers to the params dictionary
  584. # of the original request.
  585. copy_params = req.params.copy()
  586. modified_req.params = copy_params
  587. raw_qs = parsed_path.query
  588. existing_qs = urllib.parse.parse_qs(
  589. raw_qs,
  590. keep_blank_values=True
  591. )
  592. # ``parse_qs`` will return lists. Don't do that unless there's a real,
  593. # live list provided.
  594. for key, value in existing_qs.items():
  595. if isinstance(value, (list, tuple)):
  596. if len(value) == 1:
  597. existing_qs[key] = value[0]
  598. modified_req.params.update(existing_qs)
  599. return modified_req
  600. def payload(self, http_request):
  601. if http_request.headers.get('x-amz-content-sha256'):
  602. return http_request.headers['x-amz-content-sha256']
  603. return super(S3HmacAuthV4Handler, self).payload(http_request)
  604. def add_auth(self, req, **kwargs):
  605. if 'x-amz-content-sha256' not in req.headers:
  606. if '_sha256' in req.headers:
  607. req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256')
  608. else:
  609. req.headers['x-amz-content-sha256'] = self.payload(req)
  610. updated_req = self.mangle_path_and_params(req)
  611. return super(S3HmacAuthV4Handler, self).add_auth(updated_req,
  612. unmangled_req=req,
  613. **kwargs)
  614. def presign(self, req, expires, iso_date=None):
  615. """
  616. Presign a request using SigV4 query params. Takes in an HTTP request
  617. and an expiration time in seconds and returns a URL.
  618. http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
  619. """
  620. if iso_date is None:
  621. iso_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
  622. region = self.determine_region_name(req.host)
  623. service = self.determine_service_name(req.host)
  624. params = {
  625. 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
  626. 'X-Amz-Credential': '%s/%s/%s/%s/aws4_request' % (
  627. self._provider.access_key,
  628. iso_date[:8],
  629. region,
  630. service
  631. ),
  632. 'X-Amz-Date': iso_date,
  633. 'X-Amz-Expires': expires,
  634. 'X-Amz-SignedHeaders': 'host'
  635. }
  636. if self._provider.security_token:
  637. params['X-Amz-Security-Token'] = self._provider.security_token
  638. headers_to_sign = self.headers_to_sign(req)
  639. l = sorted(['%s' % n.lower().strip() for n in headers_to_sign])
  640. params['X-Amz-SignedHeaders'] = ';'.join(l)
  641. req.params.update(params)
  642. cr = self.canonical_request(req)
  643. # We need to replace the payload SHA with a constant
  644. cr = '\n'.join(cr.split('\n')[:-1]) + '\nUNSIGNED-PAYLOAD'
  645. # Date header is expected for string_to_sign, but unused otherwise
  646. req.headers['X-Amz-Date'] = iso_date
  647. sts = self.string_to_sign(req, cr)
  648. signature = self.signature(req, sts)
  649. # Add signature to params now that we have it
  650. req.params['X-Amz-Signature'] = signature
  651. return 'https://%s%s?%s' % (req.host, req.path,
  652. urllib.parse.urlencode(req.params))
  653. class STSAnonHandler(AuthHandler):
  654. """
  655. Provides pure query construction (no actual signing).
  656. Used for making anonymous STS request for operations like
  657. ``assume_role_with_web_identity``.
  658. """
  659. capability = ['sts-anon']
  660. def _escape_value(self, value):
  661. # This is changed from a previous version because this string is
  662. # being passed to the query string and query strings must
  663. # be url encoded. In particular STS requires the saml_response to
  664. # be urlencoded when calling assume_role_with_saml.
  665. return urllib.parse.quote(value)
  666. def _build_query_string(self, params):
  667. keys = list(params.keys())
  668. keys.sort(key=lambda x: x.lower())
  669. pairs = []
  670. for key in keys:
  671. val = boto.utils.get_utf8_value(params[key])
  672. pairs.append(key + '=' + self._escape_value(val.decode('utf-8')))
  673. return '&'.join(pairs)
  674. def add_auth(self, http_request, **kwargs):
  675. headers = http_request.headers
  676. qs = self._build_query_string(
  677. http_request.params
  678. )
  679. boto.log.debug('query_string in body: %s' % qs)
  680. headers['Content-Type'] = 'application/x-www-form-urlencoded'
  681. # This will be a POST so the query string should go into the body
  682. # as opposed to being in the uri
  683. http_request.body = qs
  684. class QuerySignatureHelper(HmacKeys):
  685. """
  686. Helper for Query signature based Auth handler.
  687. Concrete sub class need to implement _calc_sigature method.
  688. """
  689. def add_auth(self, http_request, **kwargs):
  690. headers = http_request.headers
  691. params = http_request.params
  692. params['AWSAccessKeyId'] = self._provider.access_key
  693. params['SignatureVersion'] = self.SignatureVersion
  694. params['Timestamp'] = boto.utils.get_ts()
  695. qs, signature = self._calc_signature(
  696. http_request.params, http_request.method,
  697. http_request.auth_path, http_request.host)
  698. boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
  699. if http_request.method == 'POST':
  700. headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  701. http_request.body = qs + '&Signature=' + urllib.parse.quote_plus(signature)
  702. http_request.headers['Content-Length'] = str(len(http_request.body))
  703. else:
  704. http_request.body = ''
  705. # if this is a retried request, the qs from the previous try will
  706. # already be there, we need to get rid of that and rebuild it
  707. http_request.path = http_request.path.split('?')[0]
  708. http_request.path = (http_request.path + '?' + qs +
  709. '&Signature=' + urllib.parse.quote_plus(signature))
  710. class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler):
  711. """Provides Signature V0 Signing"""
  712. SignatureVersion = 0
  713. capability = ['sign-v0']
  714. def _calc_signature(self, params, *args):
  715. boto.log.debug('using _calc_signature_0')
  716. hmac = self._get_hmac()
  717. s = params['Action'] + params['Timestamp']
  718. hmac.update(s.encode('utf-8'))
  719. keys = params.keys()
  720. keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
  721. pairs = []
  722. for key in keys:
  723. val = boto.utils.get_utf8_value(params[key])
  724. pairs.append(key + '=' + urllib.parse.quote(val))
  725. qs = '&'.join(pairs)
  726. return (qs, base64.b64encode(hmac.digest()))
  727. class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler):
  728. """
  729. Provides Query Signature V1 Authentication.
  730. """
  731. SignatureVersion = 1
  732. capability = ['sign-v1', 'mturk']
  733. def __init__(self, *args, **kw):
  734. QuerySignatureHelper.__init__(self, *args, **kw)
  735. AuthHandler.__init__(self, *args, **kw)
  736. self._hmac_256 = None
  737. def _calc_signature(self, params, *args):
  738. boto.log.debug('using _calc_signature_1')
  739. hmac = self._get_hmac()
  740. keys = params.keys()
  741. keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
  742. pairs = []
  743. for key in keys:
  744. hmac.update(key.encode('utf-8'))
  745. val = boto.utils.get_utf8_value(params[key])
  746. hmac.update(val)
  747. pairs.append(key + '=' + urllib.parse.quote(val))
  748. qs = '&'.join(pairs)
  749. return (qs, base64.b64encode(hmac.digest()))
  750. class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler):
  751. """Provides Query Signature V2 Authentication."""
  752. SignatureVersion = 2
  753. capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs',
  754. 'sdb', 'iam', 'rds', 'sns', 'sqs', 'cloudformation']
  755. def _calc_signature(self, params, verb, path, server_name):
  756. boto.log.debug('using _calc_signature_2')
  757. string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path)
  758. hmac = self._get_hmac()
  759. params['SignatureMethod'] = self.algorithm()
  760. if self._provider.security_token:
  761. params['SecurityToken'] = self._provider.security_token
  762. keys = sorted(params.keys())
  763. pairs = []
  764. for key in keys:
  765. val = boto.utils.get_utf8_value(params[key])
  766. pairs.append(urllib.parse.quote(key, safe='') + '=' +
  767. urllib.parse.quote(val, safe='-_~'))
  768. qs = '&'.join(pairs)
  769. boto.log.debug('query string: %s' % qs)
  770. string_to_sign += qs
  771. boto.log.debug('string_to_sign: %s' % string_to_sign)
  772. hmac.update(string_to_sign.encode('utf-8'))
  773. b64 = base64.b64encode(hmac.digest())
  774. boto.log.debug('len(b64)=%d' % len(b64))
  775. boto.log.debug('base64 encoded digest: %s' % b64)
  776. return (qs, b64)
  777. class POSTPathQSV2AuthHandler(QuerySignatureV2AuthHandler, AuthHandler):
  778. """
  779. Query Signature V2 Authentication relocating signed query
  780. into the path and allowing POST requests with Content-Types.
  781. """
  782. capability = ['mws']
  783. def add_auth(self, req, **kwargs):
  784. req.params['AWSAccessKeyId'] = self._provider.access_key
  785. req.params['SignatureVersion'] = self.SignatureVersion
  786. req.params['Timestamp'] = boto.utils.get_ts()
  787. qs, signature = self._calc_signature(req.params, req.method,
  788. req.auth_path, req.host)
  789. boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
  790. if req.method == 'POST':
  791. req.headers['Content-Length'] = str(len(req.body))
  792. req.headers['Content-Type'] = req.headers.get('Content-Type',
  793. 'text/plain')
  794. else:
  795. req.body = ''
  796. # if this is a retried req, the qs from the previous try will
  797. # already be there, we need to get rid of that and rebuild it
  798. req.path = req.path.split('?')[0]
  799. req.path = (req.path + '?' + qs +
  800. '&Signature=' + urllib.parse.quote_plus(signature))
  801. def get_auth_handler(host, config, provider, requested_capability=None):
  802. """Finds an AuthHandler that is ready to authenticate.
  803. Lists through all the registered AuthHandlers to find one that is willing
  804. to handle for the requested capabilities, config and provider.
  805. :type host: string
  806. :param host: The name of the host
  807. :type config:
  808. :param config:
  809. :type provider:
  810. :param provider:
  811. Returns:
  812. An implementation of AuthHandler.
  813. Raises:
  814. boto.exception.NoAuthHandlerFound
  815. """
  816. ready_handlers = []
  817. auth_handlers = boto.plugin.get_plugin(AuthHandler, requested_capability)
  818. for handler in auth_handlers:
  819. try:
  820. ready_handlers.append(handler(host, config, provider))
  821. except boto.auth_handler.NotReadyToAuthenticate:
  822. pass
  823. if not ready_handlers:
  824. checked_handlers = auth_handlers
  825. names = [handler.__name__ for handler in checked_handlers]
  826. raise boto.exception.NoAuthHandlerFound(
  827. 'No handler was ready to authenticate. %d handlers were checked.'
  828. ' %s '
  829. 'Check your credentials' % (len(names), str(names)))
  830. # We select the last ready auth handler that was loaded, to allow users to
  831. # customize how auth works in environments where there are shared boto
  832. # config files (e.g., /etc/boto.cfg and ~/.boto): The more general,
  833. # system-wide shared configs should be loaded first, and the user's
  834. # customizations loaded last. That way, for example, the system-wide
  835. # config might include a plugin_directory that includes a service account
  836. # auth plugin shared by all users of a Google Compute Engine instance
  837. # (allowing sharing of non-user data between various services), and the
  838. # user could override this with a .boto config that includes user-specific
  839. # credentials (for access to user data).
  840. return ready_handlers[-1]
  841. def detect_potential_sigv4(func):
  842. def _wrapper(self):
  843. if os.environ.get('EC2_USE_SIGV4', False):
  844. return ['hmac-v4']
  845. if boto.config.get('ec2', 'use-sigv4', False):
  846. return ['hmac-v4']
  847. if hasattr(self, 'region'):
  848. # If you're making changes here, you should also check
  849. # ``boto/iam/connection.py``, as several things there are also
  850. # endpoint-related.
  851. if getattr(self.region, 'endpoint', ''):
  852. for test in SIGV4_DETECT:
  853. if test in self.region.endpoint:
  854. return ['hmac-v4']
  855. return func(self)
  856. return _wrapper
  857. def detect_potential_s3sigv4(func):
  858. def _wrapper(self):
  859. if os.environ.get('S3_USE_SIGV4', False):
  860. return ['hmac-v4-s3']
  861. if boto.config.get('s3', 'use-sigv4', False):
  862. return ['hmac-v4-s3']
  863. if hasattr(self, 'host'):
  864. # If you're making changes here, you should also check
  865. # ``boto/iam/connection.py``, as several things there are also
  866. # endpoint-related.
  867. for test in SIGV4_DETECT:
  868. if test in self.host:
  869. return ['hmac-v4-s3']
  870. return func(self)
  871. return _wrapper