lexer.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. # lexer.py
  2. # Copyright (C) 2006, 2007, 2008, 2009, 2010 Michael Bayer mike_mp@zzzcomputing.com
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """provides the Lexer class for parsing template strings into parse trees."""
  7. import re, codecs
  8. from mako import parsetree, exceptions, util
  9. from mako.pygen import adjust_whitespace
  10. _regexp_cache = {}
  11. class Lexer(object):
  12. def __init__(self, text, filename=None,
  13. disable_unicode=False,
  14. input_encoding=None, preprocessor=None):
  15. self.text = text
  16. self.filename = filename
  17. self.template = parsetree.TemplateNode(self.filename)
  18. self.matched_lineno = 1
  19. self.matched_charpos = 0
  20. self.lineno = 1
  21. self.match_position = 0
  22. self.tag = []
  23. self.control_line = []
  24. self.disable_unicode = disable_unicode
  25. self.encoding = input_encoding
  26. if util.py3k and disable_unicode:
  27. raise exceptions.UnsupportedError(
  28. "Mako for Python 3 does not "
  29. "support disabling Unicode")
  30. if preprocessor is None:
  31. self.preprocessor = []
  32. elif not hasattr(preprocessor, '__iter__'):
  33. self.preprocessor = [preprocessor]
  34. else:
  35. self.preprocessor = preprocessor
  36. @property
  37. def exception_kwargs(self):
  38. return {'source':self.text,
  39. 'lineno':self.matched_lineno,
  40. 'pos':self.matched_charpos,
  41. 'filename':self.filename}
  42. def match(self, regexp, flags=None):
  43. """compile the given regexp, cache the reg, and call match_reg()."""
  44. try:
  45. reg = _regexp_cache[(regexp, flags)]
  46. except KeyError:
  47. if flags:
  48. reg = re.compile(regexp, flags)
  49. else:
  50. reg = re.compile(regexp)
  51. _regexp_cache[(regexp, flags)] = reg
  52. return self.match_reg(reg)
  53. def match_reg(self, reg):
  54. """match the given regular expression object to the current text position.
  55. if a match occurs, update the current text and line position.
  56. """
  57. mp = self.match_position
  58. match = reg.match(self.text, self.match_position)
  59. if match:
  60. (start, end) = match.span()
  61. if end == start:
  62. self.match_position = end + 1
  63. else:
  64. self.match_position = end
  65. self.matched_lineno = self.lineno
  66. lines = re.findall(r"\n", self.text[mp:self.match_position])
  67. cp = mp - 1
  68. while (cp >= 0 and cp<self.textlength and self.text[cp] != '\n'):
  69. cp -=1
  70. self.matched_charpos = mp - cp
  71. self.lineno += len(lines)
  72. #print "MATCHED:", match.group(0), "LINE START:",
  73. # self.matched_lineno, "LINE END:", self.lineno
  74. #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], (match and "TRUE" or "FALSE")
  75. return match
  76. def parse_until_text(self, *text):
  77. startpos = self.match_position
  78. while True:
  79. match = self.match(r'#.*\n')
  80. if match:
  81. continue
  82. match = self.match(r'(\"\"\"|\'\'\'|\"|\')')
  83. if match:
  84. m = self.match(r'.*?%s' % match.group(1), re.S)
  85. if not m:
  86. raise exceptions.SyntaxException(
  87. "Unmatched '%s'" %
  88. match.group(1),
  89. **self.exception_kwargs)
  90. else:
  91. match = self.match(r'(%s)' % r'|'.join(text))
  92. if match:
  93. return \
  94. self.text[startpos:self.match_position-len(match.group(1))],\
  95. match.group(1)
  96. else:
  97. match = self.match(r".*?(?=\"|\'|#|%s)" % r'|'.join(text), re.S)
  98. if not match:
  99. raise exceptions.SyntaxException(
  100. "Expected: %s" %
  101. ','.join(text),
  102. **self.exception_kwargs)
  103. def append_node(self, nodecls, *args, **kwargs):
  104. kwargs.setdefault('source', self.text)
  105. kwargs.setdefault('lineno', self.matched_lineno)
  106. kwargs.setdefault('pos', self.matched_charpos)
  107. kwargs['filename'] = self.filename
  108. node = nodecls(*args, **kwargs)
  109. if len(self.tag):
  110. self.tag[-1].nodes.append(node)
  111. else:
  112. self.template.nodes.append(node)
  113. if isinstance(node, parsetree.Tag):
  114. if len(self.tag):
  115. node.parent = self.tag[-1]
  116. self.tag.append(node)
  117. elif isinstance(node, parsetree.ControlLine):
  118. if node.isend:
  119. self.control_line.pop()
  120. elif node.is_primary:
  121. self.control_line.append(node)
  122. elif len(self.control_line) and \
  123. not self.control_line[-1].is_ternary(node.keyword):
  124. raise exceptions.SyntaxException(
  125. "Keyword '%s' not a legal ternary for keyword '%s'" %
  126. (node.keyword, self.control_line[-1].keyword),
  127. **self.exception_kwargs)
  128. _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n')
  129. def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
  130. """given string/unicode or bytes/string, determine encoding
  131. from magic encoding comment, return body as unicode
  132. or raw if decode_raw=False
  133. """
  134. if isinstance(text, unicode):
  135. m = self._coding_re.match(text)
  136. encoding = m and m.group(1) or known_encoding or 'ascii'
  137. return encoding, text
  138. if text.startswith(codecs.BOM_UTF8):
  139. text = text[len(codecs.BOM_UTF8):]
  140. parsed_encoding = 'utf-8'
  141. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  142. if m is not None and m.group(1) != 'utf-8':
  143. raise exceptions.CompileException(
  144. "Found utf-8 BOM in file, with conflicting "
  145. "magic encoding comment of '%s'" % m.group(1),
  146. text.decode('utf-8', 'ignore'),
  147. 0, 0, filename)
  148. else:
  149. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  150. if m:
  151. parsed_encoding = m.group(1)
  152. else:
  153. parsed_encoding = known_encoding or 'ascii'
  154. if decode_raw:
  155. try:
  156. text = text.decode(parsed_encoding)
  157. except UnicodeDecodeError, e:
  158. raise exceptions.CompileException(
  159. "Unicode decode operation of encoding '%s' failed" %
  160. parsed_encoding,
  161. text.decode('utf-8', 'ignore'),
  162. 0, 0, filename)
  163. return parsed_encoding, text
  164. def parse(self):
  165. self.encoding, self.text = self.decode_raw_stream(self.text,
  166. not self.disable_unicode,
  167. self.encoding,
  168. self.filename,)
  169. for preproc in self.preprocessor:
  170. self.text = preproc(self.text)
  171. # push the match marker past the
  172. # encoding comment.
  173. self.match_reg(self._coding_re)
  174. self.textlength = len(self.text)
  175. while (True):
  176. if self.match_position > self.textlength:
  177. break
  178. if self.match_end():
  179. break
  180. if self.match_expression():
  181. continue
  182. if self.match_control_line():
  183. continue
  184. if self.match_comment():
  185. continue
  186. if self.match_tag_start():
  187. continue
  188. if self.match_tag_end():
  189. continue
  190. if self.match_python_block():
  191. continue
  192. if self.match_text():
  193. continue
  194. if self.match_position > self.textlength:
  195. break
  196. raise exceptions.CompileException("assertion failed")
  197. if len(self.tag):
  198. raise exceptions.SyntaxException("Unclosed tag: <%%%s>" %
  199. self.tag[-1].keyword,
  200. **self.exception_kwargs)
  201. if len(self.control_line):
  202. raise exceptions.SyntaxException("Unterminated control keyword: '%s'" %
  203. self.control_line[-1].keyword,
  204. self.text,
  205. self.control_line[-1].lineno,
  206. self.control_line[-1].pos, self.filename)
  207. return self.template
  208. def match_tag_start(self):
  209. match = self.match(r'''
  210. \<% # opening tag
  211. ([\w\.\:]+) # keyword
  212. ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = sign, string expression
  213. \s* # more whitespace
  214. (/)?> # closing
  215. ''',
  216. re.I | re.S | re.X)
  217. if match:
  218. keyword, attr, isend = match.group(1), match.group(2), match.group(3)
  219. self.keyword = keyword
  220. attributes = {}
  221. if attr:
  222. for att in re.findall(r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr):
  223. key, val1, val2 = att
  224. text = val1 or val2
  225. text = text.replace('\r\n', '\n')
  226. attributes[key] = text
  227. self.append_node(parsetree.Tag, keyword, attributes)
  228. if isend:
  229. self.tag.pop()
  230. else:
  231. if keyword == 'text':
  232. match = self.match(r'(.*?)(?=\</%text>)', re.S)
  233. if not match:
  234. raise exceptions.SyntaxException(
  235. "Unclosed tag: <%%%s>" %
  236. self.tag[-1].keyword,
  237. **self.exception_kwargs)
  238. self.append_node(parsetree.Text, match.group(1))
  239. return self.match_tag_end()
  240. return True
  241. else:
  242. return False
  243. def match_tag_end(self):
  244. match = self.match(r'\</%[\t ]*(.+?)[\t ]*>')
  245. if match:
  246. if not len(self.tag):
  247. raise exceptions.SyntaxException(
  248. "Closing tag without opening tag: </%%%s>" %
  249. match.group(1),
  250. **self.exception_kwargs)
  251. elif self.tag[-1].keyword != match.group(1):
  252. raise exceptions.SyntaxException(
  253. "Closing tag </%%%s> does not match tag: <%%%s>" %
  254. (match.group(1), self.tag[-1].keyword),
  255. **self.exception_kwargs)
  256. self.tag.pop()
  257. return True
  258. else:
  259. return False
  260. def match_end(self):
  261. match = self.match(r'\Z', re.S)
  262. if match:
  263. string = match.group()
  264. if string:
  265. return string
  266. else:
  267. return True
  268. else:
  269. return False
  270. def match_text(self):
  271. match = self.match(r"""
  272. (.*?) # anything, followed by:
  273. (
  274. (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
  275. # comment preceded by a
  276. # consumed newline and whitespace
  277. |
  278. (?=\${) # an expression
  279. |
  280. (?=\#\*) # multiline comment
  281. |
  282. (?=</?[%&]) # a substitution or block or call start or end
  283. # - don't consume
  284. |
  285. (\\\r?\n) # an escaped newline - throw away
  286. |
  287. \Z # end of string
  288. )""", re.X | re.S)
  289. if match:
  290. text = match.group(1)
  291. self.append_node(parsetree.Text, text)
  292. return True
  293. else:
  294. return False
  295. def match_python_block(self):
  296. match = self.match(r"<%(!)?")
  297. if match:
  298. line, pos = self.matched_lineno, self.matched_charpos
  299. text, end = self.parse_until_text(r'%>')
  300. # the trailing newline helps
  301. # compiler.parse() not complain about indentation
  302. text = adjust_whitespace(text) + "\n"
  303. self.append_node(
  304. parsetree.Code,
  305. text,
  306. match.group(1)=='!', lineno=line, pos=pos)
  307. return True
  308. else:
  309. return False
  310. def match_expression(self):
  311. match = self.match(r"\${")
  312. if match:
  313. line, pos = self.matched_lineno, self.matched_charpos
  314. text, end = self.parse_until_text(r'\|', r'}')
  315. if end == '|':
  316. escapes, end = self.parse_until_text(r'}')
  317. else:
  318. escapes = ""
  319. text = text.replace('\r\n', '\n')
  320. self.append_node(
  321. parsetree.Expression,
  322. text, escapes.strip(),
  323. lineno=line, pos=pos)
  324. return True
  325. else:
  326. return False
  327. def match_control_line(self):
  328. match = self.match(r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)(?:\r?\n|\Z)", re.M)
  329. if match:
  330. operator = match.group(1)
  331. text = match.group(2)
  332. if operator == '%':
  333. m2 = re.match(r'(end)?(\w+)\s*(.*)', text)
  334. if not m2:
  335. raise exceptions.SyntaxException(
  336. "Invalid control line: '%s'" %
  337. text,
  338. **self.exception_kwargs)
  339. isend, keyword = m2.group(1, 2)
  340. isend = (isend is not None)
  341. if isend:
  342. if not len(self.control_line):
  343. raise exceptions.SyntaxException(
  344. "No starting keyword '%s' for '%s'" %
  345. (keyword, text),
  346. **self.exception_kwargs)
  347. elif self.control_line[-1].keyword != keyword:
  348. raise exceptions.SyntaxException(
  349. "Keyword '%s' doesn't match keyword '%s'" %
  350. (text, self.control_line[-1].keyword),
  351. **self.exception_kwargs)
  352. self.append_node(parsetree.ControlLine, keyword, isend, text)
  353. else:
  354. self.append_node(parsetree.Comment, text)
  355. return True
  356. else:
  357. return False
  358. def match_comment(self):
  359. """matches the multiline version of a comment"""
  360. match = self.match(r"<%doc>(.*?)</%doc>", re.S)
  361. if match:
  362. self.append_node(parsetree.Comment, match.group(1))
  363. return True
  364. else:
  365. return False