sessions.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682
  1. # -*- coding: utf-8 -*-
  2. """
  3. requests.session
  4. ~~~~~~~~~~~~~~~~
  5. This module provides a Session object to manage and persist settings across
  6. requests (cookies, auth, proxies).
  7. """
  8. import os
  9. from collections import Mapping
  10. from datetime import datetime
  11. from .auth import _basic_auth_str
  12. from .compat import cookielib, OrderedDict, urljoin, urlparse
  13. from .cookies import (
  14. cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
  15. from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
  16. from .hooks import default_hooks, dispatch_hook
  17. from .utils import to_key_val_list, default_headers, to_native_string
  18. from .exceptions import (
  19. TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
  20. from .packages.urllib3._collections import RecentlyUsedContainer
  21. from .structures import CaseInsensitiveDict
  22. from .adapters import HTTPAdapter
  23. from .utils import (
  24. requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies,
  25. get_auth_from_url
  26. )
  27. from .status_codes import codes
  28. # formerly defined here, reexposed here for backward compatibility
  29. from .models import REDIRECT_STATI
  30. REDIRECT_CACHE_SIZE = 1000
  31. def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
  32. """
  33. Determines appropriate setting for a given request, taking into account the
  34. explicit setting on that request, and the setting in the session. If a
  35. setting is a dictionary, they will be merged together using `dict_class`
  36. """
  37. if session_setting is None:
  38. return request_setting
  39. if request_setting is None:
  40. return session_setting
  41. # Bypass if not a dictionary (e.g. verify)
  42. if not (
  43. isinstance(session_setting, Mapping) and
  44. isinstance(request_setting, Mapping)
  45. ):
  46. return request_setting
  47. merged_setting = dict_class(to_key_val_list(session_setting))
  48. merged_setting.update(to_key_val_list(request_setting))
  49. # Remove keys that are set to None.
  50. for (k, v) in request_setting.items():
  51. if v is None:
  52. del merged_setting[k]
  53. merged_setting = dict((k, v) for (k, v) in merged_setting.items() if v is not None)
  54. return merged_setting
  55. def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
  56. """
  57. Properly merges both requests and session hooks.
  58. This is necessary because when request_hooks == {'response': []}, the
  59. merge breaks Session hooks entirely.
  60. """
  61. if session_hooks is None or session_hooks.get('response') == []:
  62. return request_hooks
  63. if request_hooks is None or request_hooks.get('response') == []:
  64. return session_hooks
  65. return merge_setting(request_hooks, session_hooks, dict_class)
  66. class SessionRedirectMixin(object):
  67. def resolve_redirects(self, resp, req, stream=False, timeout=None,
  68. verify=True, cert=None, proxies=None):
  69. """Receives a Response. Returns a generator of Responses."""
  70. i = 0
  71. hist = [] # keep track of history
  72. while resp.is_redirect:
  73. prepared_request = req.copy()
  74. if i > 0:
  75. # Update history and keep track of redirects.
  76. hist.append(resp)
  77. new_hist = list(hist)
  78. resp.history = new_hist
  79. try:
  80. resp.content # Consume socket so it can be released
  81. except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
  82. resp.raw.read(decode_content=False)
  83. if i >= self.max_redirects:
  84. raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects)
  85. # Release the connection back into the pool.
  86. resp.close()
  87. url = resp.headers['location']
  88. method = req.method
  89. # Handle redirection without scheme (see: RFC 1808 Section 4)
  90. if url.startswith('//'):
  91. parsed_rurl = urlparse(resp.url)
  92. url = '%s:%s' % (parsed_rurl.scheme, url)
  93. # The scheme should be lower case...
  94. parsed = urlparse(url)
  95. url = parsed.geturl()
  96. # Facilitate relative 'location' headers, as allowed by RFC 7231.
  97. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
  98. # Compliant with RFC3986, we percent encode the url.
  99. if not parsed.netloc:
  100. url = urljoin(resp.url, requote_uri(url))
  101. else:
  102. url = requote_uri(url)
  103. prepared_request.url = to_native_string(url)
  104. # Cache the url, unless it redirects to itself.
  105. if resp.is_permanent_redirect and req.url != prepared_request.url:
  106. self.redirect_cache[req.url] = prepared_request.url
  107. # http://tools.ietf.org/html/rfc7231#section-6.4.4
  108. if (resp.status_code == codes.see_other and
  109. method != 'HEAD'):
  110. method = 'GET'
  111. # Do what the browsers do, despite standards...
  112. # First, turn 302s into GETs.
  113. if resp.status_code == codes.found and method != 'HEAD':
  114. method = 'GET'
  115. # Second, if a POST is responded to with a 301, turn it into a GET.
  116. # This bizarre behaviour is explained in Issue 1704.
  117. if resp.status_code == codes.moved and method == 'POST':
  118. method = 'GET'
  119. prepared_request.method = method
  120. # https://github.com/kennethreitz/requests/issues/1084
  121. if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect):
  122. if 'Content-Length' in prepared_request.headers:
  123. del prepared_request.headers['Content-Length']
  124. prepared_request.body = None
  125. headers = prepared_request.headers
  126. try:
  127. del headers['Cookie']
  128. except KeyError:
  129. pass
  130. extract_cookies_to_jar(prepared_request._cookies, prepared_request, resp.raw)
  131. prepared_request._cookies.update(self.cookies)
  132. prepared_request.prepare_cookies(prepared_request._cookies)
  133. # Rebuild auth and proxy information.
  134. proxies = self.rebuild_proxies(prepared_request, proxies)
  135. self.rebuild_auth(prepared_request, resp)
  136. # Override the original request.
  137. req = prepared_request
  138. resp = self.send(
  139. req,
  140. stream=stream,
  141. timeout=timeout,
  142. verify=verify,
  143. cert=cert,
  144. proxies=proxies,
  145. allow_redirects=False,
  146. )
  147. extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)
  148. i += 1
  149. yield resp
  150. def rebuild_auth(self, prepared_request, response):
  151. """
  152. When being redirected we may want to strip authentication from the
  153. request to avoid leaking credentials. This method intelligently removes
  154. and reapplies authentication where possible to avoid credential loss.
  155. """
  156. headers = prepared_request.headers
  157. url = prepared_request.url
  158. if 'Authorization' in headers:
  159. # If we get redirected to a new host, we should strip out any
  160. # authentication headers.
  161. original_parsed = urlparse(response.request.url)
  162. redirect_parsed = urlparse(url)
  163. if (original_parsed.hostname != redirect_parsed.hostname):
  164. del headers['Authorization']
  165. # .netrc might have more auth for us on our new host.
  166. new_auth = get_netrc_auth(url) if self.trust_env else None
  167. if new_auth is not None:
  168. prepared_request.prepare_auth(new_auth)
  169. return
  170. def rebuild_proxies(self, prepared_request, proxies):
  171. """
  172. This method re-evaluates the proxy configuration by considering the
  173. environment variables. If we are redirected to a URL covered by
  174. NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
  175. proxy keys for this URL (in case they were stripped by a previous
  176. redirect).
  177. This method also replaces the Proxy-Authorization header where
  178. necessary.
  179. """
  180. headers = prepared_request.headers
  181. url = prepared_request.url
  182. scheme = urlparse(url).scheme
  183. new_proxies = proxies.copy() if proxies is not None else {}
  184. if self.trust_env and not should_bypass_proxies(url):
  185. environ_proxies = get_environ_proxies(url)
  186. proxy = environ_proxies.get(scheme)
  187. if proxy:
  188. new_proxies.setdefault(scheme, environ_proxies[scheme])
  189. if 'Proxy-Authorization' in headers:
  190. del headers['Proxy-Authorization']
  191. try:
  192. username, password = get_auth_from_url(new_proxies[scheme])
  193. except KeyError:
  194. username, password = None, None
  195. if username and password:
  196. headers['Proxy-Authorization'] = _basic_auth_str(username, password)
  197. return new_proxies
  198. class Session(SessionRedirectMixin):
  199. """A Requests session.
  200. Provides cookie persistence, connection-pooling, and configuration.
  201. Basic Usage::
  202. >>> import requests
  203. >>> s = requests.Session()
  204. >>> s.get('http://httpbin.org/get')
  205. 200
  206. """
  207. __attrs__ = [
  208. 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
  209. 'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
  210. 'max_redirects',
  211. ]
  212. def __init__(self):
  213. #: A case-insensitive dictionary of headers to be sent on each
  214. #: :class:`Request <Request>` sent from this
  215. #: :class:`Session <Session>`.
  216. self.headers = default_headers()
  217. #: Default Authentication tuple or object to attach to
  218. #: :class:`Request <Request>`.
  219. self.auth = None
  220. #: Dictionary mapping protocol to the URL of the proxy (e.g.
  221. #: {'http': 'foo.bar:3128'}) to be used on each
  222. #: :class:`Request <Request>`.
  223. self.proxies = {}
  224. #: Event-handling hooks.
  225. self.hooks = default_hooks()
  226. #: Dictionary of querystring data to attach to each
  227. #: :class:`Request <Request>`. The dictionary values may be lists for
  228. #: representing multivalued query parameters.
  229. self.params = {}
  230. #: Stream response content default.
  231. self.stream = False
  232. #: SSL Verification default.
  233. self.verify = True
  234. #: SSL certificate default.
  235. self.cert = None
  236. #: Maximum number of redirects allowed. If the request exceeds this
  237. #: limit, a :class:`TooManyRedirects` exception is raised.
  238. self.max_redirects = DEFAULT_REDIRECT_LIMIT
  239. #: Should we trust the environment?
  240. self.trust_env = True
  241. #: A CookieJar containing all currently outstanding cookies set on this
  242. #: session. By default it is a
  243. #: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
  244. #: may be any other ``cookielib.CookieJar`` compatible object.
  245. self.cookies = cookiejar_from_dict({})
  246. # Default connection adapters.
  247. self.adapters = OrderedDict()
  248. self.mount('https://', HTTPAdapter())
  249. self.mount('http://', HTTPAdapter())
  250. # Only store 1000 redirects to prevent using infinite memory
  251. self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
  252. def __enter__(self):
  253. return self
  254. def __exit__(self, *args):
  255. self.close()
  256. def prepare_request(self, request):
  257. """Constructs a :class:`PreparedRequest <PreparedRequest>` for
  258. transmission and returns it. The :class:`PreparedRequest` has settings
  259. merged from the :class:`Request <Request>` instance and those of the
  260. :class:`Session`.
  261. :param request: :class:`Request` instance to prepare with this
  262. session's settings.
  263. """
  264. cookies = request.cookies or {}
  265. # Bootstrap CookieJar.
  266. if not isinstance(cookies, cookielib.CookieJar):
  267. cookies = cookiejar_from_dict(cookies)
  268. # Merge with session cookies
  269. merged_cookies = merge_cookies(
  270. merge_cookies(RequestsCookieJar(), self.cookies), cookies)
  271. # Set environment's basic authentication if not explicitly set.
  272. auth = request.auth
  273. if self.trust_env and not auth and not self.auth:
  274. auth = get_netrc_auth(request.url)
  275. p = PreparedRequest()
  276. p.prepare(
  277. method=request.method.upper(),
  278. url=request.url,
  279. files=request.files,
  280. data=request.data,
  281. json=request.json,
  282. headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
  283. params=merge_setting(request.params, self.params),
  284. auth=merge_setting(auth, self.auth),
  285. cookies=merged_cookies,
  286. hooks=merge_hooks(request.hooks, self.hooks),
  287. )
  288. return p
  289. def request(self, method, url,
  290. params=None,
  291. data=None,
  292. headers=None,
  293. cookies=None,
  294. files=None,
  295. auth=None,
  296. timeout=None,
  297. allow_redirects=True,
  298. proxies=None,
  299. hooks=None,
  300. stream=None,
  301. verify=None,
  302. cert=None,
  303. json=None):
  304. """Constructs a :class:`Request <Request>`, prepares it and sends it.
  305. Returns :class:`Response <Response>` object.
  306. :param method: method for the new :class:`Request` object.
  307. :param url: URL for the new :class:`Request` object.
  308. :param params: (optional) Dictionary or bytes to be sent in the query
  309. string for the :class:`Request`.
  310. :param data: (optional) Dictionary or bytes to send in the body of the
  311. :class:`Request`.
  312. :param json: (optional) json to send in the body of the
  313. :class:`Request`.
  314. :param headers: (optional) Dictionary of HTTP Headers to send with the
  315. :class:`Request`.
  316. :param cookies: (optional) Dict or CookieJar object to send with the
  317. :class:`Request`.
  318. :param files: (optional) Dictionary of ``'filename': file-like-objects``
  319. for multipart encoding upload.
  320. :param auth: (optional) Auth tuple or callable to enable
  321. Basic/Digest/Custom HTTP Auth.
  322. :param timeout: (optional) How long to wait for the server to send
  323. data before giving up, as a float, or a (`connect timeout, read
  324. timeout <user/advanced.html#timeouts>`_) tuple.
  325. :type timeout: float or tuple
  326. :param allow_redirects: (optional) Set to True by default.
  327. :type allow_redirects: bool
  328. :param proxies: (optional) Dictionary mapping protocol to the URL of
  329. the proxy.
  330. :param stream: (optional) whether to immediately download the response
  331. content. Defaults to ``False``.
  332. :param verify: (optional) if ``True``, the SSL cert will be verified.
  333. A CA_BUNDLE path can also be provided.
  334. :param cert: (optional) if String, path to ssl client cert file (.pem).
  335. If Tuple, ('cert', 'key') pair.
  336. """
  337. method = to_native_string(method)
  338. # Create the Request.
  339. req = Request(
  340. method = method.upper(),
  341. url = url,
  342. headers = headers,
  343. files = files,
  344. data = data or {},
  345. json = json,
  346. params = params or {},
  347. auth = auth,
  348. cookies = cookies,
  349. hooks = hooks,
  350. )
  351. prep = self.prepare_request(req)
  352. proxies = proxies or {}
  353. settings = self.merge_environment_settings(
  354. prep.url, proxies, stream, verify, cert
  355. )
  356. # Send the request.
  357. send_kwargs = {
  358. 'timeout': timeout,
  359. 'allow_redirects': allow_redirects,
  360. }
  361. send_kwargs.update(settings)
  362. resp = self.send(prep, **send_kwargs)
  363. return resp
  364. def get(self, url, **kwargs):
  365. """Sends a GET request. Returns :class:`Response` object.
  366. :param url: URL for the new :class:`Request` object.
  367. :param \*\*kwargs: Optional arguments that ``request`` takes.
  368. """
  369. kwargs.setdefault('allow_redirects', True)
  370. return self.request('GET', url, **kwargs)
  371. def options(self, url, **kwargs):
  372. """Sends a OPTIONS request. Returns :class:`Response` object.
  373. :param url: URL for the new :class:`Request` object.
  374. :param \*\*kwargs: Optional arguments that ``request`` takes.
  375. """
  376. kwargs.setdefault('allow_redirects', True)
  377. return self.request('OPTIONS', url, **kwargs)
  378. def head(self, url, **kwargs):
  379. """Sends a HEAD request. Returns :class:`Response` object.
  380. :param url: URL for the new :class:`Request` object.
  381. :param \*\*kwargs: Optional arguments that ``request`` takes.
  382. """
  383. kwargs.setdefault('allow_redirects', False)
  384. return self.request('HEAD', url, **kwargs)
  385. def post(self, url, data=None, json=None, **kwargs):
  386. """Sends a POST request. Returns :class:`Response` object.
  387. :param url: URL for the new :class:`Request` object.
  388. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
  389. :param json: (optional) json to send in the body of the :class:`Request`.
  390. :param \*\*kwargs: Optional arguments that ``request`` takes.
  391. """
  392. return self.request('POST', url, data=data, json=json, **kwargs)
  393. def put(self, url, data=None, **kwargs):
  394. """Sends a PUT request. Returns :class:`Response` object.
  395. :param url: URL for the new :class:`Request` object.
  396. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
  397. :param \*\*kwargs: Optional arguments that ``request`` takes.
  398. """
  399. return self.request('PUT', url, data=data, **kwargs)
  400. def patch(self, url, data=None, **kwargs):
  401. """Sends a PATCH request. Returns :class:`Response` object.
  402. :param url: URL for the new :class:`Request` object.
  403. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
  404. :param \*\*kwargs: Optional arguments that ``request`` takes.
  405. """
  406. return self.request('PATCH', url, data=data, **kwargs)
  407. def delete(self, url, **kwargs):
  408. """Sends a DELETE request. Returns :class:`Response` object.
  409. :param url: URL for the new :class:`Request` object.
  410. :param \*\*kwargs: Optional arguments that ``request`` takes.
  411. """
  412. return self.request('DELETE', url, **kwargs)
  413. def send(self, request, **kwargs):
  414. """Send a given PreparedRequest."""
  415. # Set defaults that the hooks can utilize to ensure they always have
  416. # the correct parameters to reproduce the previous request.
  417. kwargs.setdefault('stream', self.stream)
  418. kwargs.setdefault('verify', self.verify)
  419. kwargs.setdefault('cert', self.cert)
  420. kwargs.setdefault('proxies', self.proxies)
  421. # It's possible that users might accidentally send a Request object.
  422. # Guard against that specific failure case.
  423. if not isinstance(request, PreparedRequest):
  424. raise ValueError('You can only send PreparedRequests.')
  425. checked_urls = set()
  426. while request.url in self.redirect_cache:
  427. checked_urls.add(request.url)
  428. new_url = self.redirect_cache.get(request.url)
  429. if new_url in checked_urls:
  430. break
  431. request.url = new_url
  432. # Set up variables needed for resolve_redirects and dispatching of hooks
  433. allow_redirects = kwargs.pop('allow_redirects', True)
  434. stream = kwargs.get('stream')
  435. timeout = kwargs.get('timeout')
  436. verify = kwargs.get('verify')
  437. cert = kwargs.get('cert')
  438. proxies = kwargs.get('proxies')
  439. hooks = request.hooks
  440. # Get the appropriate adapter to use
  441. adapter = self.get_adapter(url=request.url)
  442. # Start time (approximately) of the request
  443. start = datetime.utcnow()
  444. # Send the request
  445. r = adapter.send(request, **kwargs)
  446. # Total elapsed time of the request (approximately)
  447. r.elapsed = datetime.utcnow() - start
  448. # Response manipulation hooks
  449. r = dispatch_hook('response', hooks, r, **kwargs)
  450. # Persist cookies
  451. if r.history:
  452. # If the hooks create history then we want those cookies too
  453. for resp in r.history:
  454. extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
  455. extract_cookies_to_jar(self.cookies, request, r.raw)
  456. # Redirect resolving generator.
  457. gen = self.resolve_redirects(r, request,
  458. stream=stream,
  459. timeout=timeout,
  460. verify=verify,
  461. cert=cert,
  462. proxies=proxies)
  463. # Resolve redirects if allowed.
  464. history = [resp for resp in gen] if allow_redirects else []
  465. # Shuffle things around if there's history.
  466. if history:
  467. # Insert the first (original) request at the start
  468. history.insert(0, r)
  469. # Get the last request made
  470. r = history.pop()
  471. r.history = history
  472. if not stream:
  473. r.content
  474. return r
  475. def merge_environment_settings(self, url, proxies, stream, verify, cert):
  476. """Check the environment and merge it with some settings."""
  477. # Gather clues from the surrounding environment.
  478. if self.trust_env:
  479. # Set environment's proxies.
  480. env_proxies = get_environ_proxies(url) or {}
  481. for (k, v) in env_proxies.items():
  482. proxies.setdefault(k, v)
  483. # Look for requests environment configuration and be compatible
  484. # with cURL.
  485. if verify is True or verify is None:
  486. verify = (os.environ.get('REQUESTS_CA_BUNDLE') or
  487. os.environ.get('CURL_CA_BUNDLE'))
  488. # Merge all the kwargs.
  489. proxies = merge_setting(proxies, self.proxies)
  490. stream = merge_setting(stream, self.stream)
  491. verify = merge_setting(verify, self.verify)
  492. cert = merge_setting(cert, self.cert)
  493. return {'verify': verify, 'proxies': proxies, 'stream': stream,
  494. 'cert': cert}
  495. def get_adapter(self, url):
  496. """Returns the appropriate connnection adapter for the given URL."""
  497. for (prefix, adapter) in self.adapters.items():
  498. if url.lower().startswith(prefix):
  499. return adapter
  500. # Nothing matches :-/
  501. raise InvalidSchema("No connection adapters were found for '%s'" % url)
  502. def close(self):
  503. """Closes all adapters and as such the session"""
  504. for v in self.adapters.values():
  505. v.close()
  506. def mount(self, prefix, adapter):
  507. """Registers a connection adapter to a prefix.
  508. Adapters are sorted in descending order by key length."""
  509. self.adapters[prefix] = adapter
  510. keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
  511. for key in keys_to_move:
  512. self.adapters[key] = self.adapters.pop(key)
  513. def __getstate__(self):
  514. state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
  515. state['redirect_cache'] = dict(self.redirect_cache)
  516. return state
  517. def __setstate__(self, state):
  518. redirect_cache = state.pop('redirect_cache', {})
  519. for attr, value in state.items():
  520. setattr(self, attr, value)
  521. self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
  522. for redirect, to in redirect_cache.items():
  523. self.redirect_cache[redirect] = to
  524. def session():
  525. """Returns a :class:`Session` for context-management."""
  526. return Session()