decoder.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. """Implementation of JSONDecoder
  2. """
  3. import re
  4. import sys
  5. import struct
  6. from simplejson.scanner import make_scanner
  7. try:
  8. from simplejson._speedups import scanstring as c_scanstring
  9. except ImportError:
  10. c_scanstring = None
  11. __all__ = ['JSONDecoder']
  12. FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  13. def _floatconstants():
  14. _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
  15. if sys.byteorder != 'big':
  16. _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
  17. nan, inf = struct.unpack('dd', _BYTES)
  18. return nan, inf, -inf
  19. NaN, PosInf, NegInf = _floatconstants()
  20. def linecol(doc, pos):
  21. lineno = doc.count('\n', 0, pos) + 1
  22. if lineno == 1:
  23. colno = pos
  24. else:
  25. colno = pos - doc.rindex('\n', 0, pos)
  26. return lineno, colno
  27. def errmsg(msg, doc, pos, end=None):
  28. # Note that this function is called from _speedups
  29. lineno, colno = linecol(doc, pos)
  30. if end is None:
  31. #fmt = '{0}: line {1} column {2} (char {3})'
  32. #return fmt.format(msg, lineno, colno, pos)
  33. fmt = '%s: line %d column %d (char %d)'
  34. return fmt % (msg, lineno, colno, pos)
  35. endlineno, endcolno = linecol(doc, end)
  36. #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
  37. #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
  38. fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
  39. return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
  40. _CONSTANTS = {
  41. '-Infinity': NegInf,
  42. 'Infinity': PosInf,
  43. 'NaN': NaN,
  44. }
  45. STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  46. BACKSLASH = {
  47. '"': u'"', '\\': u'\\', '/': u'/',
  48. 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
  49. }
  50. DEFAULT_ENCODING = "utf-8"
  51. def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
  52. """Scan the string s for a JSON string. End is the index of the
  53. character in s after the quote that started the JSON string.
  54. Unescapes all valid JSON string escape sequences and raises ValueError
  55. on attempt to decode an invalid string. If strict is False then literal
  56. control characters are allowed in the string.
  57. Returns a tuple of the decoded string and the index of the character in s
  58. after the end quote."""
  59. if encoding is None:
  60. encoding = DEFAULT_ENCODING
  61. chunks = []
  62. _append = chunks.append
  63. begin = end - 1
  64. while 1:
  65. chunk = _m(s, end)
  66. if chunk is None:
  67. raise ValueError(
  68. errmsg("Unterminated string starting at", s, begin))
  69. end = chunk.end()
  70. content, terminator = chunk.groups()
  71. # Content is contains zero or more unescaped string characters
  72. if content:
  73. if not isinstance(content, unicode):
  74. content = unicode(content, encoding)
  75. _append(content)
  76. # Terminator is the end of string, a literal control character,
  77. # or a backslash denoting that an escape sequence follows
  78. if terminator == '"':
  79. break
  80. elif terminator != '\\':
  81. if strict:
  82. msg = "Invalid control character %r at" % (terminator,)
  83. #msg = "Invalid control character {0!r} at".format(terminator)
  84. raise ValueError(errmsg(msg, s, end))
  85. else:
  86. _append(terminator)
  87. continue
  88. try:
  89. esc = s[end]
  90. except IndexError:
  91. raise ValueError(
  92. errmsg("Unterminated string starting at", s, begin))
  93. # If not a unicode escape sequence, must be in the lookup table
  94. if esc != 'u':
  95. try:
  96. char = _b[esc]
  97. except KeyError:
  98. msg = "Invalid \\escape: " + repr(esc)
  99. raise ValueError(errmsg(msg, s, end))
  100. end += 1
  101. else:
  102. # Unicode escape sequence
  103. esc = s[end + 1:end + 5]
  104. next_end = end + 5
  105. if len(esc) != 4:
  106. msg = "Invalid \\uXXXX escape"
  107. raise ValueError(errmsg(msg, s, end))
  108. uni = int(esc, 16)
  109. # Check for surrogate pair on UCS-4 systems
  110. if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
  111. msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
  112. if not s[end + 5:end + 7] == '\\u':
  113. raise ValueError(errmsg(msg, s, end))
  114. esc2 = s[end + 7:end + 11]
  115. if len(esc2) != 4:
  116. raise ValueError(errmsg(msg, s, end))
  117. uni2 = int(esc2, 16)
  118. uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
  119. next_end += 6
  120. char = unichr(uni)
  121. end = next_end
  122. # Append the unescaped character
  123. _append(char)
  124. return u''.join(chunks), end
  125. # Use speedup if available
  126. scanstring = c_scanstring or py_scanstring
  127. WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
  128. WHITESPACE_STR = ' \t\n\r'
  129. def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  130. pairs = {}
  131. # Use a slice to prevent IndexError from being raised, the following
  132. # check will raise a more specific ValueError if the string is empty
  133. nextchar = s[end:end + 1]
  134. # Normally we expect nextchar == '"'
  135. if nextchar != '"':
  136. if nextchar in _ws:
  137. end = _w(s, end).end()
  138. nextchar = s[end:end + 1]
  139. # Trivial empty object
  140. if nextchar == '}':
  141. return pairs, end + 1
  142. elif nextchar != '"':
  143. raise ValueError(errmsg("Expecting property name", s, end))
  144. end += 1
  145. while True:
  146. key, end = scanstring(s, end, encoding, strict)
  147. # To skip some function call overhead we optimize the fast paths where
  148. # the JSON key separator is ": " or just ":".
  149. if s[end:end + 1] != ':':
  150. end = _w(s, end).end()
  151. if s[end:end + 1] != ':':
  152. raise ValueError(errmsg("Expecting : delimiter", s, end))
  153. end += 1
  154. try:
  155. if s[end] in _ws:
  156. end += 1
  157. if s[end] in _ws:
  158. end = _w(s, end + 1).end()
  159. except IndexError:
  160. pass
  161. try:
  162. value, end = scan_once(s, end)
  163. except StopIteration:
  164. raise ValueError(errmsg("Expecting object", s, end))
  165. pairs[key] = value
  166. try:
  167. nextchar = s[end]
  168. if nextchar in _ws:
  169. end = _w(s, end + 1).end()
  170. nextchar = s[end]
  171. except IndexError:
  172. nextchar = ''
  173. end += 1
  174. if nextchar == '}':
  175. break
  176. elif nextchar != ',':
  177. raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
  178. try:
  179. nextchar = s[end]
  180. if nextchar in _ws:
  181. end += 1
  182. nextchar = s[end]
  183. if nextchar in _ws:
  184. end = _w(s, end + 1).end()
  185. nextchar = s[end]
  186. except IndexError:
  187. nextchar = ''
  188. end += 1
  189. if nextchar != '"':
  190. raise ValueError(errmsg("Expecting property name", s, end - 1))
  191. if object_hook is not None:
  192. pairs = object_hook(pairs)
  193. return pairs, end
  194. def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  195. values = []
  196. nextchar = s[end:end + 1]
  197. if nextchar in _ws:
  198. end = _w(s, end + 1).end()
  199. nextchar = s[end:end + 1]
  200. # Look-ahead for trivial empty array
  201. if nextchar == ']':
  202. return values, end + 1
  203. _append = values.append
  204. while True:
  205. try:
  206. value, end = scan_once(s, end)
  207. except StopIteration:
  208. raise ValueError(errmsg("Expecting object", s, end))
  209. _append(value)
  210. nextchar = s[end:end + 1]
  211. if nextchar in _ws:
  212. end = _w(s, end + 1).end()
  213. nextchar = s[end:end + 1]
  214. end += 1
  215. if nextchar == ']':
  216. break
  217. elif nextchar != ',':
  218. raise ValueError(errmsg("Expecting , delimiter", s, end))
  219. try:
  220. if s[end] in _ws:
  221. end += 1
  222. if s[end] in _ws:
  223. end = _w(s, end + 1).end()
  224. except IndexError:
  225. pass
  226. return values, end
  227. class JSONDecoder(object):
  228. """Simple JSON <http://json.org> decoder
  229. Performs the following translations in decoding by default:
  230. +---------------+-------------------+
  231. | JSON | Python |
  232. +===============+===================+
  233. | object | dict |
  234. +---------------+-------------------+
  235. | array | list |
  236. +---------------+-------------------+
  237. | string | unicode |
  238. +---------------+-------------------+
  239. | number (int) | int, long |
  240. +---------------+-------------------+
  241. | number (real) | float |
  242. +---------------+-------------------+
  243. | true | True |
  244. +---------------+-------------------+
  245. | false | False |
  246. +---------------+-------------------+
  247. | null | None |
  248. +---------------+-------------------+
  249. It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
  250. their corresponding ``float`` values, which is outside the JSON spec.
  251. """
  252. def __init__(self, encoding=None, object_hook=None, parse_float=None,
  253. parse_int=None, parse_constant=None, strict=True):
  254. """``encoding`` determines the encoding used to interpret any ``str``
  255. objects decoded by this instance (utf-8 by default). It has no
  256. effect when decoding ``unicode`` objects.
  257. Note that currently only encodings that are a superset of ASCII work,
  258. strings of other encodings should be passed in as ``unicode``.
  259. ``object_hook``, if specified, will be called with the result
  260. of every JSON object decoded and its return value will be used in
  261. place of the given ``dict``. This can be used to provide custom
  262. deserializations (e.g. to support JSON-RPC class hinting).
  263. ``parse_float``, if specified, will be called with the string
  264. of every JSON float to be decoded. By default this is equivalent to
  265. float(num_str). This can be used to use another datatype or parser
  266. for JSON floats (e.g. decimal.Decimal).
  267. ``parse_int``, if specified, will be called with the string
  268. of every JSON int to be decoded. By default this is equivalent to
  269. int(num_str). This can be used to use another datatype or parser
  270. for JSON integers (e.g. float).
  271. ``parse_constant``, if specified, will be called with one of the
  272. following strings: -Infinity, Infinity, NaN.
  273. This can be used to raise an exception if invalid JSON numbers
  274. are encountered.
  275. """
  276. self.encoding = encoding
  277. self.object_hook = object_hook
  278. self.parse_float = parse_float or float
  279. self.parse_int = parse_int or int
  280. self.parse_constant = parse_constant or _CONSTANTS.__getitem__
  281. self.strict = strict
  282. self.parse_object = JSONObject
  283. self.parse_array = JSONArray
  284. self.parse_string = scanstring
  285. self.scan_once = make_scanner(self)
  286. def decode(self, s, _w=WHITESPACE.match):
  287. """Return the Python representation of ``s`` (a ``str`` or ``unicode``
  288. instance containing a JSON document)
  289. """
  290. obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  291. end = _w(s, end).end()
  292. if end != len(s):
  293. raise ValueError(errmsg("Extra data", s, end, len(s)))
  294. return obj
  295. def raw_decode(self, s, idx=0):
  296. """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
  297. with a JSON document) and return a 2-tuple of the Python
  298. representation and the index in ``s`` where the document ended.
  299. This can be used to decode a JSON document from a string that may
  300. have extraneous data at the end.
  301. """
  302. try:
  303. obj, end = self.scan_once(s, idx)
  304. except StopIteration:
  305. raise ValueError("No JSON object could be decoded")
  306. return obj, end