models.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730
  1. # -*- coding: utf-8 -*-
  2. """
  3. requests.models
  4. ~~~~~~~~~~~~~~~
  5. This module contains the primary objects that power Requests.
  6. """
  7. import collections
  8. import logging
  9. import datetime
  10. from io import BytesIO, UnsupportedOperation
  11. from .hooks import default_hooks
  12. from .structures import CaseInsensitiveDict
  13. from .auth import HTTPBasicAuth
  14. from .cookies import cookiejar_from_dict, get_cookie_header
  15. from .packages.urllib3.filepost import encode_multipart_formdata
  16. from .packages.urllib3.util import parse_url
  17. from .exceptions import (
  18. HTTPError, RequestException, MissingSchema, InvalidURL,
  19. ChunkedEncodingError)
  20. from .utils import (
  21. guess_filename, get_auth_from_url, requote_uri,
  22. stream_decode_response_unicode, to_key_val_list, parse_header_links,
  23. iter_slices, guess_json_utf, super_len, to_native_string)
  24. from .compat import (
  25. cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
  26. is_py2, chardet, json, builtin_str, basestring, IncompleteRead)
  27. CONTENT_CHUNK_SIZE = 10 * 1024
  28. ITER_CHUNK_SIZE = 512
  29. log = logging.getLogger(__name__)
  30. class RequestEncodingMixin(object):
  31. @property
  32. def path_url(self):
  33. """Build the path URL to use."""
  34. url = []
  35. p = urlsplit(self.url)
  36. path = p.path
  37. if not path:
  38. path = '/'
  39. url.append(path)
  40. query = p.query
  41. if query:
  42. url.append('?')
  43. url.append(query)
  44. return ''.join(url)
  45. @staticmethod
  46. def _encode_params(data):
  47. """Encode parameters in a piece of data.
  48. Will successfully encode parameters when passed as a dict or a list of
  49. 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
  50. if parameters are supplied as a dict.
  51. """
  52. if isinstance(data, (str, bytes)):
  53. return data
  54. elif hasattr(data, 'read'):
  55. return data
  56. elif hasattr(data, '__iter__'):
  57. result = []
  58. for k, vs in to_key_val_list(data):
  59. if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):
  60. vs = [vs]
  61. for v in vs:
  62. if v is not None:
  63. result.append(
  64. (k.encode('utf-8') if isinstance(k, str) else k,
  65. v.encode('utf-8') if isinstance(v, str) else v))
  66. return urlencode(result, doseq=True)
  67. else:
  68. return data
  69. @staticmethod
  70. def _encode_files(files, data):
  71. """Build the body for a multipart/form-data request.
  72. Will successfully encode files when passed as a dict or a list of
  73. 2-tuples. Order is retained if data is a list of 2-tuples but abritrary
  74. if parameters are supplied as a dict.
  75. """
  76. if (not files):
  77. raise ValueError("Files must be provided.")
  78. elif isinstance(data, basestring):
  79. raise ValueError("Data must not be a string.")
  80. new_fields = []
  81. fields = to_key_val_list(data or {})
  82. files = to_key_val_list(files or {})
  83. for field, val in fields:
  84. if isinstance(val, basestring) or not hasattr(val, '__iter__'):
  85. val = [val]
  86. for v in val:
  87. if v is not None:
  88. # Don't call str() on bytestrings: in Py3 it all goes wrong.
  89. if not isinstance(v, bytes):
  90. v = str(v)
  91. new_fields.append(
  92. (field.decode('utf-8') if isinstance(field, bytes) else field,
  93. v.encode('utf-8') if isinstance(v, str) else v))
  94. for (k, v) in files:
  95. # support for explicit filename
  96. ft = None
  97. if isinstance(v, (tuple, list)):
  98. if len(v) == 2:
  99. fn, fp = v
  100. else:
  101. fn, fp, ft = v
  102. else:
  103. fn = guess_filename(v) or k
  104. fp = v
  105. if isinstance(fp, str):
  106. fp = StringIO(fp)
  107. if isinstance(fp, bytes):
  108. fp = BytesIO(fp)
  109. if ft:
  110. new_v = (fn, fp.read(), ft)
  111. else:
  112. new_v = (fn, fp.read())
  113. new_fields.append((k, new_v))
  114. body, content_type = encode_multipart_formdata(new_fields)
  115. return body, content_type
  116. class RequestHooksMixin(object):
  117. def register_hook(self, event, hook):
  118. """Properly register a hook."""
  119. if event not in self.hooks:
  120. raise ValueError('Unsupported event specified, with event name "%s"' % (event))
  121. if isinstance(hook, collections.Callable):
  122. self.hooks[event].append(hook)
  123. elif hasattr(hook, '__iter__'):
  124. self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
  125. def deregister_hook(self, event, hook):
  126. """Deregister a previously registered hook.
  127. Returns True if the hook existed, False if not.
  128. """
  129. try:
  130. self.hooks[event].remove(hook)
  131. return True
  132. except ValueError:
  133. return False
  134. class Request(RequestHooksMixin):
  135. """A user-created :class:`Request <Request>` object.
  136. Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
  137. :param method: HTTP method to use.
  138. :param url: URL to send.
  139. :param headers: dictionary of headers to send.
  140. :param files: dictionary of {filename: fileobject} files to multipart upload.
  141. :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place.
  142. :param params: dictionary of URL parameters to append to the URL.
  143. :param auth: Auth handler or (user, pass) tuple.
  144. :param cookies: dictionary or CookieJar of cookies to attach to this request.
  145. :param hooks: dictionary of callback hooks, for internal usage.
  146. Usage::
  147. >>> import requests
  148. >>> req = requests.Request('GET', 'http://httpbin.org/get')
  149. >>> req.prepare()
  150. <PreparedRequest [GET]>
  151. """
  152. def __init__(self,
  153. method=None,
  154. url=None,
  155. headers=None,
  156. files=None,
  157. data=None,
  158. params=None,
  159. auth=None,
  160. cookies=None,
  161. hooks=None):
  162. # Default empty dicts for dict params.
  163. data = [] if data is None else data
  164. files = [] if files is None else files
  165. headers = {} if headers is None else headers
  166. params = {} if params is None else params
  167. hooks = {} if hooks is None else hooks
  168. self.hooks = default_hooks()
  169. for (k, v) in list(hooks.items()):
  170. self.register_hook(event=k, hook=v)
  171. self.method = method
  172. self.url = url
  173. self.headers = headers
  174. self.files = files
  175. self.data = data
  176. self.params = params
  177. self.auth = auth
  178. self.cookies = cookies
  179. def __repr__(self):
  180. return '<Request [%s]>' % (self.method)
  181. def prepare(self):
  182. """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
  183. p = PreparedRequest()
  184. p.prepare(
  185. method=self.method,
  186. url=self.url,
  187. headers=self.headers,
  188. files=self.files,
  189. data=self.data,
  190. params=self.params,
  191. auth=self.auth,
  192. cookies=self.cookies,
  193. hooks=self.hooks,
  194. )
  195. return p
  196. class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
  197. """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
  198. containing the exact bytes that will be sent to the server.
  199. Generated from either a :class:`Request <Request>` object or manually.
  200. Usage::
  201. >>> import requests
  202. >>> req = requests.Request('GET', 'http://httpbin.org/get')
  203. >>> r = req.prepare()
  204. <PreparedRequest [GET]>
  205. >>> s = requests.Session()
  206. >>> s.send(r)
  207. <Response [200]>
  208. """
  209. def __init__(self):
  210. #: HTTP verb to send to the server.
  211. self.method = None
  212. #: HTTP URL to send the request to.
  213. self.url = None
  214. #: dictionary of HTTP headers.
  215. self.headers = None
  216. #: request body to send to the server.
  217. self.body = None
  218. #: dictionary of callback hooks, for internal usage.
  219. self.hooks = default_hooks()
  220. def prepare(self, method=None, url=None, headers=None, files=None,
  221. data=None, params=None, auth=None, cookies=None, hooks=None):
  222. """Prepares the the entire request with the given parameters."""
  223. self.prepare_method(method)
  224. self.prepare_url(url, params)
  225. self.prepare_headers(headers)
  226. self.prepare_cookies(cookies)
  227. self.prepare_body(data, files)
  228. self.prepare_auth(auth, url)
  229. # Note that prepare_auth must be last to enable authentication schemes
  230. # such as OAuth to work on a fully prepared request.
  231. # This MUST go after prepare_auth. Authenticators could add a hook
  232. self.prepare_hooks(hooks)
  233. def __repr__(self):
  234. return '<PreparedRequest [%s]>' % (self.method)
  235. def copy(self):
  236. p = PreparedRequest()
  237. p.method = self.method
  238. p.url = self.url
  239. p.headers = self.headers
  240. p.body = self.body
  241. p.hooks = self.hooks
  242. return p
  243. def prepare_method(self, method):
  244. """Prepares the given HTTP method."""
  245. self.method = method
  246. if self.method is not None:
  247. self.method = self.method.upper()
  248. def prepare_url(self, url, params):
  249. """Prepares the given HTTP URL."""
  250. #: Accept objects that have string representations.
  251. try:
  252. url = unicode(url)
  253. except NameError:
  254. # We're on Python 3.
  255. url = str(url)
  256. except UnicodeDecodeError:
  257. pass
  258. # Support for unicode domain names and paths.
  259. scheme, auth, host, port, path, query, fragment = parse_url(url)
  260. if not scheme:
  261. raise MissingSchema("Invalid URL %r: No schema supplied" % url)
  262. if not host:
  263. raise InvalidURL("Invalid URL %r: No host supplied" % url)
  264. # Only want to apply IDNA to the hostname
  265. try:
  266. host = host.encode('idna').decode('utf-8')
  267. except UnicodeError:
  268. raise InvalidURL('URL has an invalid label.')
  269. # Carefully reconstruct the network location
  270. netloc = auth or ''
  271. if netloc:
  272. netloc += '@'
  273. netloc += host
  274. if port:
  275. netloc += ':' + str(port)
  276. # Bare domains aren't valid URLs.
  277. if not path:
  278. path = '/'
  279. if is_py2:
  280. if isinstance(scheme, str):
  281. scheme = scheme.encode('utf-8')
  282. if isinstance(netloc, str):
  283. netloc = netloc.encode('utf-8')
  284. if isinstance(path, str):
  285. path = path.encode('utf-8')
  286. if isinstance(query, str):
  287. query = query.encode('utf-8')
  288. if isinstance(fragment, str):
  289. fragment = fragment.encode('utf-8')
  290. enc_params = self._encode_params(params)
  291. if enc_params:
  292. if query:
  293. query = '%s&%s' % (query, enc_params)
  294. else:
  295. query = enc_params
  296. url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
  297. self.url = url
  298. def prepare_headers(self, headers):
  299. """Prepares the given HTTP headers."""
  300. if headers:
  301. self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items())
  302. else:
  303. self.headers = CaseInsensitiveDict()
  304. def prepare_body(self, data, files):
  305. """Prepares the given HTTP body data."""
  306. # Check if file, fo, generator, iterator.
  307. # If not, run through normal process.
  308. # Nottin' on you.
  309. body = None
  310. content_type = None
  311. length = None
  312. is_stream = all([
  313. hasattr(data, '__iter__'),
  314. not isinstance(data, basestring),
  315. not isinstance(data, list),
  316. not isinstance(data, dict)
  317. ])
  318. try:
  319. length = super_len(data)
  320. except (TypeError, AttributeError, UnsupportedOperation):
  321. length = None
  322. if is_stream:
  323. body = data
  324. if files:
  325. raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
  326. if length is not None:
  327. self.headers['Content-Length'] = str(length)
  328. else:
  329. self.headers['Transfer-Encoding'] = 'chunked'
  330. else:
  331. # Multi-part file uploads.
  332. if files:
  333. (body, content_type) = self._encode_files(files, data)
  334. else:
  335. if data:
  336. body = self._encode_params(data)
  337. if isinstance(data, str) or isinstance(data, builtin_str) or hasattr(data, 'read'):
  338. content_type = None
  339. else:
  340. content_type = 'application/x-www-form-urlencoded'
  341. self.prepare_content_length(body)
  342. # Add content-type if it wasn't explicitly provided.
  343. if (content_type) and (not 'content-type' in self.headers):
  344. self.headers['Content-Type'] = content_type
  345. self.body = body
  346. def prepare_content_length(self, body):
  347. if hasattr(body, 'seek') and hasattr(body, 'tell'):
  348. body.seek(0, 2)
  349. self.headers['Content-Length'] = str(body.tell())
  350. body.seek(0, 0)
  351. elif body is not None:
  352. l = super_len(body)
  353. if l:
  354. self.headers['Content-Length'] = str(l)
  355. elif self.method not in ('GET', 'HEAD'):
  356. self.headers['Content-Length'] = '0'
  357. def prepare_auth(self, auth, url=''):
  358. """Prepares the given HTTP auth data."""
  359. # If no Auth is explicitly provided, extract it from the URL first.
  360. if auth is None:
  361. url_auth = get_auth_from_url(self.url)
  362. auth = url_auth if any(url_auth) else None
  363. if auth:
  364. if isinstance(auth, tuple) and len(auth) == 2:
  365. # special-case basic HTTP auth
  366. auth = HTTPBasicAuth(*auth)
  367. # Allow auth to make its changes.
  368. r = auth(self)
  369. # Update self to reflect the auth changes.
  370. self.__dict__.update(r.__dict__)
  371. # Recompute Content-Length
  372. self.prepare_content_length(self.body)
  373. def prepare_cookies(self, cookies):
  374. """Prepares the given HTTP cookie data."""
  375. if isinstance(cookies, cookielib.CookieJar):
  376. cookies = cookies
  377. else:
  378. cookies = cookiejar_from_dict(cookies)
  379. if 'cookie' not in self.headers:
  380. cookie_header = get_cookie_header(cookies, self)
  381. if cookie_header is not None:
  382. self.headers['Cookie'] = cookie_header
  383. def prepare_hooks(self, hooks):
  384. """Prepares the given hooks."""
  385. for event in hooks:
  386. self.register_hook(event, hooks[event])
  387. class Response(object):
  388. """The :class:`Response <Response>` object, which contains a
  389. server's response to an HTTP request.
  390. """
  391. def __init__(self):
  392. super(Response, self).__init__()
  393. self._content = False
  394. self._content_consumed = False
  395. #: Integer Code of responded HTTP Status.
  396. self.status_code = None
  397. #: Case-insensitive Dictionary of Response Headers.
  398. #: For example, ``headers['content-encoding']`` will return the
  399. #: value of a ``'Content-Encoding'`` response header.
  400. self.headers = CaseInsensitiveDict()
  401. #: File-like object representation of response (for advanced usage).
  402. #: Requires that ``stream=True` on the request.
  403. # This requirement does not apply for use internally to Requests.
  404. self.raw = None
  405. #: Final URL location of Response.
  406. self.url = None
  407. #: Encoding to decode with when accessing r.text.
  408. self.encoding = None
  409. #: A list of :class:`Response <Response>` objects from
  410. #: the history of the Request. Any redirect responses will end
  411. #: up here. The list is sorted from the oldest to the most recent request.
  412. self.history = []
  413. self.reason = None
  414. #: A CookieJar of Cookies the server sent back.
  415. self.cookies = cookiejar_from_dict({})
  416. #: The amount of time elapsed between sending the request
  417. #: and the arrival of the response (as a timedelta)
  418. self.elapsed = datetime.timedelta(0)
  419. def __repr__(self):
  420. return '<Response [%s]>' % (self.status_code)
  421. def __bool__(self):
  422. """Returns true if :attr:`status_code` is 'OK'."""
  423. return self.ok
  424. def __nonzero__(self):
  425. """Returns true if :attr:`status_code` is 'OK'."""
  426. return self.ok
  427. def __iter__(self):
  428. """Allows you to use a response as an iterator."""
  429. return self.iter_content(128)
  430. @property
  431. def ok(self):
  432. try:
  433. self.raise_for_status()
  434. except RequestException:
  435. return False
  436. return True
  437. @property
  438. def apparent_encoding(self):
  439. """The apparent encoding, provided by the lovely Charade library
  440. (Thanks, Ian!)."""
  441. return chardet.detect(self.content)['encoding']
  442. def iter_content(self, chunk_size=1, decode_unicode=False):
  443. """Iterates over the response data. When stream=True is set on the
  444. request, this avoids reading the content at once into memory for
  445. large responses. The chunk size is the number of bytes it should
  446. read into memory. This is not necessarily the length of each item
  447. returned as decoding can take place.
  448. """
  449. if self._content_consumed:
  450. # simulate reading small chunks of the content
  451. return iter_slices(self._content, chunk_size)
  452. def generate():
  453. try:
  454. # Special case for urllib3.
  455. try:
  456. for chunk in self.raw.stream(chunk_size,
  457. decode_content=True):
  458. yield chunk
  459. except IncompleteRead as e:
  460. raise ChunkedEncodingError(e)
  461. except AttributeError:
  462. # Standard file-like object.
  463. while 1:
  464. chunk = self.raw.read(chunk_size)
  465. if not chunk:
  466. break
  467. yield chunk
  468. self._content_consumed = True
  469. gen = generate()
  470. if decode_unicode:
  471. gen = stream_decode_response_unicode(gen, self)
  472. return gen
  473. def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
  474. """Iterates over the response data, one line at a time. When
  475. stream=True is set on the request, this avoids reading the
  476. content at once into memory for large responses.
  477. """
  478. pending = None
  479. for chunk in self.iter_content(chunk_size=chunk_size,
  480. decode_unicode=decode_unicode):
  481. if pending is not None:
  482. chunk = pending + chunk
  483. lines = chunk.splitlines()
  484. if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
  485. pending = lines.pop()
  486. else:
  487. pending = None
  488. for line in lines:
  489. yield line
  490. if pending is not None:
  491. yield pending
  492. @property
  493. def content(self):
  494. """Content of the response, in bytes."""
  495. if self._content is False:
  496. # Read the contents.
  497. try:
  498. if self._content_consumed:
  499. raise RuntimeError(
  500. 'The content for this response was already consumed')
  501. if self.status_code == 0:
  502. self._content = None
  503. else:
  504. self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
  505. except AttributeError:
  506. self._content = None
  507. self._content_consumed = True
  508. # don't need to release the connection; that's been handled by urllib3
  509. # since we exhausted the data.
  510. return self._content
  511. @property
  512. def text(self):
  513. """Content of the response, in unicode.
  514. if Response.encoding is None and chardet module is available, encoding
  515. will be guessed.
  516. """
  517. # Try charset from content-type
  518. content = None
  519. encoding = self.encoding
  520. if not self.content:
  521. return str('')
  522. # Fallback to auto-detected encoding.
  523. if self.encoding is None:
  524. encoding = self.apparent_encoding
  525. # Decode unicode from given encoding.
  526. try:
  527. content = str(self.content, encoding, errors='replace')
  528. except (LookupError, TypeError):
  529. # A LookupError is raised if the encoding was not found which could
  530. # indicate a misspelling or similar mistake.
  531. #
  532. # A TypeError can be raised if encoding is None
  533. #
  534. # So we try blindly encoding.
  535. content = str(self.content, errors='replace')
  536. return content
  537. def json(self, **kwargs):
  538. """Returns the json-encoded content of a response, if any.
  539. :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
  540. """
  541. if not self.encoding and len(self.content) > 3:
  542. # No encoding set. JSON RFC 4627 section 3 states we should expect
  543. # UTF-8, -16 or -32. Detect which one to use; If the detection or
  544. # decoding fails, fall back to `self.text` (using chardet to make
  545. # a best guess).
  546. encoding = guess_json_utf(self.content)
  547. if encoding is not None:
  548. return json.loads(self.content.decode(encoding), **kwargs)
  549. return json.loads(self.text or self.content, **kwargs)
  550. @property
  551. def links(self):
  552. """Returns the parsed header links of the response, if any."""
  553. header = self.headers.get('link')
  554. # l = MultiDict()
  555. l = {}
  556. if header:
  557. links = parse_header_links(header)
  558. for link in links:
  559. key = link.get('rel') or link.get('url')
  560. l[key] = link
  561. return l
  562. def raise_for_status(self):
  563. """Raises stored :class:`HTTPError`, if one occurred."""
  564. http_error_msg = ''
  565. if 400 <= self.status_code < 500:
  566. http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
  567. elif 500 <= self.status_code < 600:
  568. http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
  569. if http_error_msg:
  570. raise HTTPError(http_error_msg, response=self)
  571. def close(self):
  572. """Closes the underlying file descriptor and releases the connection
  573. back to the pool.
  574. *Note: Should not normally need to be called explicitly.*
  575. """
  576. return self.raw.release_conn()