adapters.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. # -*- coding: utf-8 -*-
  2. """
  3. requests.adapters
  4. ~~~~~~~~~~~~~~~~~
  5. This module contains the transport adapters that Requests uses to define
  6. and maintain connections.
  7. """
  8. import socket
  9. from .models import Response
  10. from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
  11. from .packages.urllib3.response import HTTPResponse
  12. from .packages.urllib3.util import Timeout as TimeoutSauce
  13. from .compat import urlparse, basestring, urldefrag, unquote
  14. from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
  15. except_on_missing_scheme, get_auth_from_url)
  16. from .structures import CaseInsensitiveDict
  17. from .packages.urllib3.exceptions import MaxRetryError
  18. from .packages.urllib3.exceptions import TimeoutError
  19. from .packages.urllib3.exceptions import SSLError as _SSLError
  20. from .packages.urllib3.exceptions import HTTPError as _HTTPError
  21. from .cookies import extract_cookies_to_jar
  22. from .exceptions import ConnectionError, Timeout, SSLError
  23. from .auth import _basic_auth_str
  24. DEFAULT_POOLBLOCK = False
  25. DEFAULT_POOLSIZE = 10
  26. DEFAULT_RETRIES = 0
  27. class BaseAdapter(object):
  28. """The Base Transport Adapter"""
  29. def __init__(self):
  30. super(BaseAdapter, self).__init__()
  31. def send(self):
  32. raise NotImplementedError
  33. def close(self):
  34. raise NotImplementedError
  35. class HTTPAdapter(BaseAdapter):
  36. """The built-in HTTP Adapter for urllib3.
  37. Provides a general-case interface for Requests sessions to contact HTTP and
  38. HTTPS urls by implementing the Transport Adapter interface. This class will
  39. usually be created by the :class:`Session <Session>` class under the
  40. covers.
  41. :param pool_connections: The number of urllib3 connection pools to cache.
  42. :param pool_maxsize: The maximum number of connections to save in the pool.
  43. :param max_retries: The maximum number of retries each connection should attempt.
  44. :param pool_block: Whether the connection pool should block for connections.
  45. Usage::
  46. >>> import requests
  47. >>> s = requests.Session()
  48. >>> a = requests.adapters.HTTPAdapter()
  49. >>> s.mount('http://', a)
  50. """
  51. __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize',
  52. '_pool_block']
  53. def __init__(self, pool_connections=DEFAULT_POOLSIZE,
  54. pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
  55. pool_block=DEFAULT_POOLBLOCK):
  56. self.max_retries = max_retries
  57. self.config = {}
  58. self.proxy_manager = {}
  59. super(HTTPAdapter, self).__init__()
  60. self._pool_connections = pool_connections
  61. self._pool_maxsize = pool_maxsize
  62. self._pool_block = pool_block
  63. self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
  64. def __getstate__(self):
  65. return dict((attr, getattr(self, attr, None)) for attr in
  66. self.__attrs__)
  67. def __setstate__(self, state):
  68. for attr, value in state.items():
  69. setattr(self, attr, value)
  70. self.init_poolmanager(self._pool_connections, self._pool_maxsize,
  71. block=self._pool_block)
  72. def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK):
  73. """Initializes a urllib3 PoolManager. This method should not be called
  74. from user code, and is only exposed for use when subclassing the
  75. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  76. :param connections: The number of urllib3 connection pools to cache.
  77. :param maxsize: The maximum number of connections to save in the pool.
  78. :param block: Block when no free connections are available.
  79. """
  80. # save these values for pickling
  81. self._pool_connections = connections
  82. self._pool_maxsize = maxsize
  83. self._pool_block = block
  84. self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
  85. block=block)
  86. def cert_verify(self, conn, url, verify, cert):
  87. """Verify a SSL certificate. This method should not be called from user
  88. code, and is only exposed for use when subclassing the
  89. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  90. :param conn: The urllib3 connection object associated with the cert.
  91. :param url: The requested URL.
  92. :param verify: Whether we should actually verify the certificate.
  93. :param cert: The SSL certificate to verify.
  94. """
  95. if url.lower().startswith('https') and verify:
  96. cert_loc = None
  97. # Allow self-specified cert location.
  98. if verify is not True:
  99. cert_loc = verify
  100. if not cert_loc:
  101. cert_loc = DEFAULT_CA_BUNDLE_PATH
  102. if not cert_loc:
  103. raise Exception("Could not find a suitable SSL CA certificate bundle.")
  104. conn.cert_reqs = 'CERT_REQUIRED'
  105. conn.ca_certs = cert_loc
  106. else:
  107. conn.cert_reqs = 'CERT_NONE'
  108. conn.ca_certs = None
  109. if cert:
  110. if not isinstance(cert, basestring):
  111. conn.cert_file = cert[0]
  112. conn.key_file = cert[1]
  113. else:
  114. conn.cert_file = cert
  115. def build_response(self, req, resp):
  116. """Builds a :class:`Response <requests.Response>` object from a urllib3
  117. response. This should not be called from user code, and is only exposed
  118. for use when subclassing the
  119. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
  120. :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
  121. :param resp: The urllib3 response object.
  122. """
  123. response = Response()
  124. # Fallback to None if there's no status_code, for whatever reason.
  125. response.status_code = getattr(resp, 'status', None)
  126. # Make headers case-insensitive.
  127. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
  128. # Set encoding.
  129. response.encoding = get_encoding_from_headers(response.headers)
  130. response.raw = resp
  131. response.reason = response.raw.reason
  132. if isinstance(req.url, bytes):
  133. response.url = req.url.decode('utf-8')
  134. else:
  135. response.url = req.url
  136. # Add new cookies from the server.
  137. extract_cookies_to_jar(response.cookies, req, resp)
  138. # Give the Response some context.
  139. response.request = req
  140. response.connection = self
  141. return response
  142. def get_connection(self, url, proxies=None):
  143. """Returns a urllib3 connection for the given URL. This should not be
  144. called from user code, and is only exposed for use when subclassing the
  145. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  146. :param url: The URL to connect to.
  147. :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
  148. """
  149. proxies = proxies or {}
  150. proxy = proxies.get(urlparse(url.lower()).scheme)
  151. if proxy:
  152. except_on_missing_scheme(proxy)
  153. proxy_headers = self.proxy_headers(proxy)
  154. if not proxy in self.proxy_manager:
  155. self.proxy_manager[proxy] = proxy_from_url(
  156. proxy,
  157. proxy_headers=proxy_headers)
  158. conn = self.proxy_manager[proxy].connection_from_url(url)
  159. else:
  160. conn = self.poolmanager.connection_from_url(url.lower())
  161. return conn
  162. def close(self):
  163. """Disposes of any internal state.
  164. Currently, this just closes the PoolManager, which closes pooled
  165. connections.
  166. """
  167. self.poolmanager.clear()
  168. def request_url(self, request, proxies):
  169. """Obtain the url to use when making the final request.
  170. If the message is being sent through a proxy, the full URL has to be
  171. used. Otherwise, we should only use the path portion of the URL.
  172. This should not be called from user code, and is only exposed for use
  173. when subclassing the
  174. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  175. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
  176. :param proxies: A dictionary of schemes to proxy URLs.
  177. """
  178. proxies = proxies or {}
  179. proxy = proxies.get(urlparse(request.url).scheme)
  180. if proxy:
  181. url, _ = urldefrag(request.url)
  182. else:
  183. url = request.path_url
  184. return url
  185. def add_headers(self, request, **kwargs):
  186. """Add any headers needed by the connection. As of v2.0 this does
  187. nothing by default, but is left for overriding by users that subclass
  188. the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  189. This should not be called from user code, and is only exposed for use
  190. when subclassing the
  191. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  192. :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
  193. :param kwargs: The keyword arguments from the call to send().
  194. """
  195. pass
  196. def proxy_headers(self, proxy):
  197. """Returns a dictionary of the headers to add to any request sent
  198. through a proxy. This works with urllib3 magic to ensure that they are
  199. correctly sent to the proxy, rather than in a tunnelled request if
  200. CONNECT is being used.
  201. This should not be called from user code, and is only exposed for use
  202. when subclassing the
  203. :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
  204. :param proxies: The url of the proxy being used for this request.
  205. :param kwargs: Optional additional keyword arguments.
  206. """
  207. headers = {}
  208. username, password = get_auth_from_url(proxy)
  209. if username and password:
  210. # Proxy auth usernames and passwords will be urlencoded, we need
  211. # to decode them.
  212. username = unquote(username)
  213. password = unquote(password)
  214. headers['Proxy-Authorization'] = _basic_auth_str(username,
  215. password)
  216. return headers
  217. def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
  218. """Sends PreparedRequest object. Returns Response object.
  219. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
  220. :param stream: (optional) Whether to stream the request content.
  221. :param timeout: (optional) The timeout on the request.
  222. :param verify: (optional) Whether to verify SSL certificates.
  223. :param vert: (optional) Any user-provided SSL certificate to be trusted.
  224. :param proxies: (optional) The proxies dictionary to apply to the request.
  225. """
  226. conn = self.get_connection(request.url, proxies)
  227. self.cert_verify(conn, request.url, verify, cert)
  228. url = self.request_url(request, proxies)
  229. self.add_headers(request)
  230. chunked = not (request.body is None or 'Content-Length' in request.headers)
  231. if stream:
  232. timeout = TimeoutSauce(connect=timeout)
  233. else:
  234. timeout = TimeoutSauce(connect=timeout, read=timeout)
  235. try:
  236. if not chunked:
  237. resp = conn.urlopen(
  238. method=request.method,
  239. url=url,
  240. body=request.body,
  241. headers=request.headers,
  242. redirect=False,
  243. assert_same_host=False,
  244. preload_content=False,
  245. decode_content=False,
  246. retries=self.max_retries,
  247. timeout=timeout
  248. )
  249. # Send the request.
  250. else:
  251. if hasattr(conn, 'proxy_pool'):
  252. conn = conn.proxy_pool
  253. low_conn = conn._get_conn(timeout=timeout)
  254. low_conn.putrequest(request.method, url, skip_accept_encoding=True)
  255. for header, value in request.headers.items():
  256. low_conn.putheader(header, value)
  257. low_conn.endheaders()
  258. for i in request.body:
  259. low_conn.send(hex(len(i))[2:].encode('utf-8'))
  260. low_conn.send(b'\r\n')
  261. low_conn.send(i)
  262. low_conn.send(b'\r\n')
  263. low_conn.send(b'0\r\n\r\n')
  264. r = low_conn.getresponse()
  265. resp = HTTPResponse.from_httplib(r,
  266. pool=conn,
  267. connection=low_conn,
  268. preload_content=False,
  269. decode_content=False
  270. )
  271. except socket.error as sockerr:
  272. raise ConnectionError(sockerr)
  273. except MaxRetryError as e:
  274. raise ConnectionError(e)
  275. except (_SSLError, _HTTPError) as e:
  276. if isinstance(e, _SSLError):
  277. raise SSLError(e)
  278. elif isinstance(e, TimeoutError):
  279. raise Timeout(e)
  280. else:
  281. raise
  282. r = self.build_response(request, resp)
  283. if not stream:
  284. r.content
  285. return r