lexer.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443
  1. # mako/lexer.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """provides the Lexer class for parsing template strings into parse trees."""
  7. import re
  8. import codecs
  9. from mako import parsetree, exceptions, compat
  10. from mako.pygen import adjust_whitespace
  11. _regexp_cache = {}
  12. class Lexer(object):
  13. def __init__(self, text, filename=None,
  14. disable_unicode=False,
  15. input_encoding=None, preprocessor=None):
  16. self.text = text
  17. self.filename = filename
  18. self.template = parsetree.TemplateNode(self.filename)
  19. self.matched_lineno = 1
  20. self.matched_charpos = 0
  21. self.lineno = 1
  22. self.match_position = 0
  23. self.tag = []
  24. self.control_line = []
  25. self.ternary_stack = []
  26. self.disable_unicode = disable_unicode
  27. self.encoding = input_encoding
  28. if compat.py3k and disable_unicode:
  29. raise exceptions.UnsupportedError(
  30. "Mako for Python 3 does not "
  31. "support disabling Unicode")
  32. if preprocessor is None:
  33. self.preprocessor = []
  34. elif not hasattr(preprocessor, '__iter__'):
  35. self.preprocessor = [preprocessor]
  36. else:
  37. self.preprocessor = preprocessor
  38. @property
  39. def exception_kwargs(self):
  40. return {'source':self.text,
  41. 'lineno':self.matched_lineno,
  42. 'pos':self.matched_charpos,
  43. 'filename':self.filename}
  44. def match(self, regexp, flags=None):
  45. """compile the given regexp, cache the reg, and call match_reg()."""
  46. try:
  47. reg = _regexp_cache[(regexp, flags)]
  48. except KeyError:
  49. if flags:
  50. reg = re.compile(regexp, flags)
  51. else:
  52. reg = re.compile(regexp)
  53. _regexp_cache[(regexp, flags)] = reg
  54. return self.match_reg(reg)
  55. def match_reg(self, reg):
  56. """match the given regular expression object to the current text
  57. position.
  58. if a match occurs, update the current text and line position.
  59. """
  60. mp = self.match_position
  61. match = reg.match(self.text, self.match_position)
  62. if match:
  63. (start, end) = match.span()
  64. if end == start:
  65. self.match_position = end + 1
  66. else:
  67. self.match_position = end
  68. self.matched_lineno = self.lineno
  69. lines = re.findall(r"\n", self.text[mp:self.match_position])
  70. cp = mp - 1
  71. while (cp >= 0 and cp<self.textlength and self.text[cp] != '\n'):
  72. cp -=1
  73. self.matched_charpos = mp - cp
  74. self.lineno += len(lines)
  75. #print "MATCHED:", match.group(0), "LINE START:",
  76. # self.matched_lineno, "LINE END:", self.lineno
  77. #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], \
  78. # (match and "TRUE" or "FALSE")
  79. return match
  80. def parse_until_text(self, *text):
  81. startpos = self.match_position
  82. text_re = r'|'.join(text)
  83. brace_level = 0
  84. while True:
  85. match = self.match(r'#.*\n')
  86. if match:
  87. continue
  88. match = self.match(r'(\"\"\"|\'\'\'|\"|\')((?<!\\)\\\1|.)*?\1',
  89. re.S)
  90. if match:
  91. continue
  92. match = self.match(r'(%s)' % text_re)
  93. if match:
  94. if match.group(1) == '}' and brace_level > 0:
  95. brace_level -= 1
  96. continue
  97. return \
  98. self.text[startpos:\
  99. self.match_position-len(match.group(1))],\
  100. match.group(1)
  101. match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S)
  102. if match:
  103. brace_level += match.group(1).count('{')
  104. brace_level -= match.group(1).count('}')
  105. continue
  106. raise exceptions.SyntaxException(
  107. "Expected: %s" %
  108. ','.join(text),
  109. **self.exception_kwargs)
  110. def append_node(self, nodecls, *args, **kwargs):
  111. kwargs.setdefault('source', self.text)
  112. kwargs.setdefault('lineno', self.matched_lineno)
  113. kwargs.setdefault('pos', self.matched_charpos)
  114. kwargs['filename'] = self.filename
  115. node = nodecls(*args, **kwargs)
  116. if len(self.tag):
  117. self.tag[-1].nodes.append(node)
  118. else:
  119. self.template.nodes.append(node)
  120. # build a set of child nodes for the control line
  121. # (used for loop variable detection)
  122. # also build a set of child nodes on ternary control lines
  123. # (used for determining if a pass needs to be auto-inserted
  124. if self.control_line:
  125. control_frame = self.control_line[-1]
  126. control_frame.nodes.append(node)
  127. if not (isinstance(node, parsetree.ControlLine) and
  128. control_frame.is_ternary(node.keyword)):
  129. if self.ternary_stack and self.ternary_stack[-1]:
  130. self.ternary_stack[-1][-1].nodes.append(node)
  131. if isinstance(node, parsetree.Tag):
  132. if len(self.tag):
  133. node.parent = self.tag[-1]
  134. self.tag.append(node)
  135. elif isinstance(node, parsetree.ControlLine):
  136. if node.isend:
  137. self.control_line.pop()
  138. self.ternary_stack.pop()
  139. elif node.is_primary:
  140. self.control_line.append(node)
  141. self.ternary_stack.append([])
  142. elif self.control_line and \
  143. self.control_line[-1].is_ternary(node.keyword):
  144. self.ternary_stack[-1].append(node)
  145. elif self.control_line and \
  146. not self.control_line[-1].is_ternary(node.keyword):
  147. raise exceptions.SyntaxException(
  148. "Keyword '%s' not a legal ternary for keyword '%s'" %
  149. (node.keyword, self.control_line[-1].keyword),
  150. **self.exception_kwargs)
  151. _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n')
  152. def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
  153. """given string/unicode or bytes/string, determine encoding
  154. from magic encoding comment, return body as unicode
  155. or raw if decode_raw=False
  156. """
  157. if isinstance(text, compat.text_type):
  158. m = self._coding_re.match(text)
  159. encoding = m and m.group(1) or known_encoding or 'ascii'
  160. return encoding, text
  161. if text.startswith(codecs.BOM_UTF8):
  162. text = text[len(codecs.BOM_UTF8):]
  163. parsed_encoding = 'utf-8'
  164. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  165. if m is not None and m.group(1) != 'utf-8':
  166. raise exceptions.CompileException(
  167. "Found utf-8 BOM in file, with conflicting "
  168. "magic encoding comment of '%s'" % m.group(1),
  169. text.decode('utf-8', 'ignore'),
  170. 0, 0, filename)
  171. else:
  172. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  173. if m:
  174. parsed_encoding = m.group(1)
  175. else:
  176. parsed_encoding = known_encoding or 'ascii'
  177. if decode_raw:
  178. try:
  179. text = text.decode(parsed_encoding)
  180. except UnicodeDecodeError:
  181. raise exceptions.CompileException(
  182. "Unicode decode operation of encoding '%s' failed" %
  183. parsed_encoding,
  184. text.decode('utf-8', 'ignore'),
  185. 0, 0, filename)
  186. return parsed_encoding, text
  187. def parse(self):
  188. self.encoding, self.text = self.decode_raw_stream(self.text,
  189. not self.disable_unicode,
  190. self.encoding,
  191. self.filename,)
  192. for preproc in self.preprocessor:
  193. self.text = preproc(self.text)
  194. # push the match marker past the
  195. # encoding comment.
  196. self.match_reg(self._coding_re)
  197. self.textlength = len(self.text)
  198. while (True):
  199. if self.match_position > self.textlength:
  200. break
  201. if self.match_end():
  202. break
  203. if self.match_expression():
  204. continue
  205. if self.match_control_line():
  206. continue
  207. if self.match_comment():
  208. continue
  209. if self.match_tag_start():
  210. continue
  211. if self.match_tag_end():
  212. continue
  213. if self.match_python_block():
  214. continue
  215. if self.match_text():
  216. continue
  217. if self.match_position > self.textlength:
  218. break
  219. raise exceptions.CompileException("assertion failed")
  220. if len(self.tag):
  221. raise exceptions.SyntaxException("Unclosed tag: <%%%s>" %
  222. self.tag[-1].keyword,
  223. **self.exception_kwargs)
  224. if len(self.control_line):
  225. raise exceptions.SyntaxException(
  226. "Unterminated control keyword: '%s'" %
  227. self.control_line[-1].keyword,
  228. self.text,
  229. self.control_line[-1].lineno,
  230. self.control_line[-1].pos, self.filename)
  231. return self.template
  232. def match_tag_start(self):
  233. match = self.match(r'''
  234. \<% # opening tag
  235. ([\w\.\:]+) # keyword
  236. ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = \
  237. # sign, string expression
  238. \s* # more whitespace
  239. (/)?> # closing
  240. ''',
  241. re.I | re.S | re.X)
  242. if match:
  243. keyword, attr, isend = match.groups()
  244. self.keyword = keyword
  245. attributes = {}
  246. if attr:
  247. for att in re.findall(
  248. r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr):
  249. key, val1, val2 = att
  250. text = val1 or val2
  251. text = text.replace('\r\n', '\n')
  252. attributes[key] = text
  253. self.append_node(parsetree.Tag, keyword, attributes)
  254. if isend:
  255. self.tag.pop()
  256. else:
  257. if keyword == 'text':
  258. match = self.match(r'(.*?)(?=\</%text>)', re.S)
  259. if not match:
  260. raise exceptions.SyntaxException(
  261. "Unclosed tag: <%%%s>" %
  262. self.tag[-1].keyword,
  263. **self.exception_kwargs)
  264. self.append_node(parsetree.Text, match.group(1))
  265. return self.match_tag_end()
  266. return True
  267. else:
  268. return False
  269. def match_tag_end(self):
  270. match = self.match(r'\</%[\t ]*(.+?)[\t ]*>')
  271. if match:
  272. if not len(self.tag):
  273. raise exceptions.SyntaxException(
  274. "Closing tag without opening tag: </%%%s>" %
  275. match.group(1),
  276. **self.exception_kwargs)
  277. elif self.tag[-1].keyword != match.group(1):
  278. raise exceptions.SyntaxException(
  279. "Closing tag </%%%s> does not match tag: <%%%s>" %
  280. (match.group(1), self.tag[-1].keyword),
  281. **self.exception_kwargs)
  282. self.tag.pop()
  283. return True
  284. else:
  285. return False
  286. def match_end(self):
  287. match = self.match(r'\Z', re.S)
  288. if match:
  289. string = match.group()
  290. if string:
  291. return string
  292. else:
  293. return True
  294. else:
  295. return False
  296. def match_text(self):
  297. match = self.match(r"""
  298. (.*?) # anything, followed by:
  299. (
  300. (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
  301. # comment preceded by a
  302. # consumed newline and whitespace
  303. |
  304. (?=\${) # an expression
  305. |
  306. (?=\#\*) # multiline comment
  307. |
  308. (?=</?[%&]) # a substitution or block or call start or end
  309. # - don't consume
  310. |
  311. (\\\r?\n) # an escaped newline - throw away
  312. |
  313. \Z # end of string
  314. )""", re.X | re.S)
  315. if match:
  316. text = match.group(1)
  317. if text:
  318. self.append_node(parsetree.Text, text)
  319. return True
  320. else:
  321. return False
  322. def match_python_block(self):
  323. match = self.match(r"<%(!)?")
  324. if match:
  325. line, pos = self.matched_lineno, self.matched_charpos
  326. text, end = self.parse_until_text(r'%>')
  327. # the trailing newline helps
  328. # compiler.parse() not complain about indentation
  329. text = adjust_whitespace(text) + "\n"
  330. self.append_node(
  331. parsetree.Code,
  332. text,
  333. match.group(1)=='!', lineno=line, pos=pos)
  334. return True
  335. else:
  336. return False
  337. def match_expression(self):
  338. match = self.match(r"\${")
  339. if match:
  340. line, pos = self.matched_lineno, self.matched_charpos
  341. text, end = self.parse_until_text(r'\|', r'}')
  342. if end == '|':
  343. escapes, end = self.parse_until_text(r'}')
  344. else:
  345. escapes = ""
  346. text = text.replace('\r\n', '\n')
  347. self.append_node(
  348. parsetree.Expression,
  349. text, escapes.strip(),
  350. lineno=line, pos=pos)
  351. return True
  352. else:
  353. return False
  354. def match_control_line(self):
  355. match = self.match(
  356. r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)"
  357. r"(?:\r?\n|\Z)", re.M)
  358. if match:
  359. operator = match.group(1)
  360. text = match.group(2)
  361. if operator == '%':
  362. m2 = re.match(r'(end)?(\w+)\s*(.*)', text)
  363. if not m2:
  364. raise exceptions.SyntaxException(
  365. "Invalid control line: '%s'" %
  366. text,
  367. **self.exception_kwargs)
  368. isend, keyword = m2.group(1, 2)
  369. isend = (isend is not None)
  370. if isend:
  371. if not len(self.control_line):
  372. raise exceptions.SyntaxException(
  373. "No starting keyword '%s' for '%s'" %
  374. (keyword, text),
  375. **self.exception_kwargs)
  376. elif self.control_line[-1].keyword != keyword:
  377. raise exceptions.SyntaxException(
  378. "Keyword '%s' doesn't match keyword '%s'" %
  379. (text, self.control_line[-1].keyword),
  380. **self.exception_kwargs)
  381. self.append_node(parsetree.ControlLine, keyword, isend, text)
  382. else:
  383. self.append_node(parsetree.Comment, text)
  384. return True
  385. else:
  386. return False
  387. def match_comment(self):
  388. """matches the multiline version of a comment"""
  389. match = self.match(r"<%doc>(.*?)</%doc>", re.S)
  390. if match:
  391. self.append_node(parsetree.Comment, match.group(1))
  392. return True
  393. else:
  394. return False