lexer.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442
  1. # mako/lexer.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """provides the Lexer class for parsing template strings into parse trees."""
  7. import re, codecs
  8. from mako import parsetree, exceptions, util
  9. from mako.pygen import adjust_whitespace
  10. _regexp_cache = {}
  11. class Lexer(object):
  12. def __init__(self, text, filename=None,
  13. disable_unicode=False,
  14. input_encoding=None, preprocessor=None):
  15. self.text = text
  16. self.filename = filename
  17. self.template = parsetree.TemplateNode(self.filename)
  18. self.matched_lineno = 1
  19. self.matched_charpos = 0
  20. self.lineno = 1
  21. self.match_position = 0
  22. self.tag = []
  23. self.control_line = []
  24. self.ternary_stack = []
  25. self.disable_unicode = disable_unicode
  26. self.encoding = input_encoding
  27. if util.py3k and disable_unicode:
  28. raise exceptions.UnsupportedError(
  29. "Mako for Python 3 does not "
  30. "support disabling Unicode")
  31. if preprocessor is None:
  32. self.preprocessor = []
  33. elif not hasattr(preprocessor, '__iter__'):
  34. self.preprocessor = [preprocessor]
  35. else:
  36. self.preprocessor = preprocessor
  37. @property
  38. def exception_kwargs(self):
  39. return {'source':self.text,
  40. 'lineno':self.matched_lineno,
  41. 'pos':self.matched_charpos,
  42. 'filename':self.filename}
  43. def match(self, regexp, flags=None):
  44. """compile the given regexp, cache the reg, and call match_reg()."""
  45. try:
  46. reg = _regexp_cache[(regexp, flags)]
  47. except KeyError:
  48. if flags:
  49. reg = re.compile(regexp, flags)
  50. else:
  51. reg = re.compile(regexp)
  52. _regexp_cache[(regexp, flags)] = reg
  53. return self.match_reg(reg)
  54. def match_reg(self, reg):
  55. """match the given regular expression object to the current text
  56. position.
  57. if a match occurs, update the current text and line position.
  58. """
  59. mp = self.match_position
  60. match = reg.match(self.text, self.match_position)
  61. if match:
  62. (start, end) = match.span()
  63. if end == start:
  64. self.match_position = end + 1
  65. else:
  66. self.match_position = end
  67. self.matched_lineno = self.lineno
  68. lines = re.findall(r"\n", self.text[mp:self.match_position])
  69. cp = mp - 1
  70. while (cp >= 0 and cp<self.textlength and self.text[cp] != '\n'):
  71. cp -=1
  72. self.matched_charpos = mp - cp
  73. self.lineno += len(lines)
  74. #print "MATCHED:", match.group(0), "LINE START:",
  75. # self.matched_lineno, "LINE END:", self.lineno
  76. #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], \
  77. # (match and "TRUE" or "FALSE")
  78. return match
  79. def parse_until_text(self, *text):
  80. startpos = self.match_position
  81. text_re = r'|'.join(text)
  82. brace_level = 0
  83. while True:
  84. match = self.match(r'#.*\n')
  85. if match:
  86. continue
  87. match = self.match(r'(\"\"\"|\'\'\'|\"|\')((?<!\\)\\\1|.)*?\1',
  88. re.S)
  89. if match:
  90. continue
  91. match = self.match(r'(%s)' % text_re)
  92. if match:
  93. if match.group(1) == '}' and brace_level > 0:
  94. brace_level -= 1
  95. continue
  96. return \
  97. self.text[startpos:\
  98. self.match_position-len(match.group(1))],\
  99. match.group(1)
  100. match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S)
  101. if match:
  102. brace_level += match.group(1).count('{')
  103. brace_level -= match.group(1).count('}')
  104. continue
  105. raise exceptions.SyntaxException(
  106. "Expected: %s" %
  107. ','.join(text),
  108. **self.exception_kwargs)
  109. def append_node(self, nodecls, *args, **kwargs):
  110. kwargs.setdefault('source', self.text)
  111. kwargs.setdefault('lineno', self.matched_lineno)
  112. kwargs.setdefault('pos', self.matched_charpos)
  113. kwargs['filename'] = self.filename
  114. node = nodecls(*args, **kwargs)
  115. if len(self.tag):
  116. self.tag[-1].nodes.append(node)
  117. else:
  118. self.template.nodes.append(node)
  119. # build a set of child nodes for the control line
  120. # (used for loop variable detection)
  121. # also build a set of child nodes on ternary control lines
  122. # (used for determining if a pass needs to be auto-inserted
  123. if self.control_line:
  124. control_frame = self.control_line[-1]
  125. control_frame.nodes.append(node)
  126. if not (isinstance(node, parsetree.ControlLine) and
  127. control_frame.is_ternary(node.keyword)):
  128. if self.ternary_stack and self.ternary_stack[-1]:
  129. self.ternary_stack[-1][-1].nodes.append(node)
  130. if isinstance(node, parsetree.Tag):
  131. if len(self.tag):
  132. node.parent = self.tag[-1]
  133. self.tag.append(node)
  134. elif isinstance(node, parsetree.ControlLine):
  135. if node.isend:
  136. self.control_line.pop()
  137. self.ternary_stack.pop()
  138. elif node.is_primary:
  139. self.control_line.append(node)
  140. self.ternary_stack.append([])
  141. elif self.control_line and \
  142. self.control_line[-1].is_ternary(node.keyword):
  143. self.ternary_stack[-1].append(node)
  144. elif self.control_line and \
  145. not self.control_line[-1].is_ternary(node.keyword):
  146. raise exceptions.SyntaxException(
  147. "Keyword '%s' not a legal ternary for keyword '%s'" %
  148. (node.keyword, self.control_line[-1].keyword),
  149. **self.exception_kwargs)
  150. _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n')
  151. def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
  152. """given string/unicode or bytes/string, determine encoding
  153. from magic encoding comment, return body as unicode
  154. or raw if decode_raw=False
  155. """
  156. if isinstance(text, unicode):
  157. m = self._coding_re.match(text)
  158. encoding = m and m.group(1) or known_encoding or 'ascii'
  159. return encoding, text
  160. if text.startswith(codecs.BOM_UTF8):
  161. text = text[len(codecs.BOM_UTF8):]
  162. parsed_encoding = 'utf-8'
  163. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  164. if m is not None and m.group(1) != 'utf-8':
  165. raise exceptions.CompileException(
  166. "Found utf-8 BOM in file, with conflicting "
  167. "magic encoding comment of '%s'" % m.group(1),
  168. text.decode('utf-8', 'ignore'),
  169. 0, 0, filename)
  170. else:
  171. m = self._coding_re.match(text.decode('utf-8', 'ignore'))
  172. if m:
  173. parsed_encoding = m.group(1)
  174. else:
  175. parsed_encoding = known_encoding or 'ascii'
  176. if decode_raw:
  177. try:
  178. text = text.decode(parsed_encoding)
  179. except UnicodeDecodeError, e:
  180. raise exceptions.CompileException(
  181. "Unicode decode operation of encoding '%s' failed" %
  182. parsed_encoding,
  183. text.decode('utf-8', 'ignore'),
  184. 0, 0, filename)
  185. return parsed_encoding, text
  186. def parse(self):
  187. self.encoding, self.text = self.decode_raw_stream(self.text,
  188. not self.disable_unicode,
  189. self.encoding,
  190. self.filename,)
  191. for preproc in self.preprocessor:
  192. self.text = preproc(self.text)
  193. # push the match marker past the
  194. # encoding comment.
  195. self.match_reg(self._coding_re)
  196. self.textlength = len(self.text)
  197. while (True):
  198. if self.match_position > self.textlength:
  199. break
  200. if self.match_end():
  201. break
  202. if self.match_expression():
  203. continue
  204. if self.match_control_line():
  205. continue
  206. if self.match_comment():
  207. continue
  208. if self.match_tag_start():
  209. continue
  210. if self.match_tag_end():
  211. continue
  212. if self.match_python_block():
  213. continue
  214. if self.match_text():
  215. continue
  216. if self.match_position > self.textlength:
  217. break
  218. raise exceptions.CompileException("assertion failed")
  219. if len(self.tag):
  220. raise exceptions.SyntaxException("Unclosed tag: <%%%s>" %
  221. self.tag[-1].keyword,
  222. **self.exception_kwargs)
  223. if len(self.control_line):
  224. raise exceptions.SyntaxException(
  225. "Unterminated control keyword: '%s'" %
  226. self.control_line[-1].keyword,
  227. self.text,
  228. self.control_line[-1].lineno,
  229. self.control_line[-1].pos, self.filename)
  230. return self.template
  231. def match_tag_start(self):
  232. match = self.match(r'''
  233. \<% # opening tag
  234. ([\w\.\:]+) # keyword
  235. ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = \
  236. # sign, string expression
  237. \s* # more whitespace
  238. (/)?> # closing
  239. ''',
  240. re.I | re.S | re.X)
  241. if match:
  242. keyword, attr, isend = match.groups()
  243. self.keyword = keyword
  244. attributes = {}
  245. if attr:
  246. for att in re.findall(
  247. r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr):
  248. key, val1, val2 = att
  249. text = val1 or val2
  250. text = text.replace('\r\n', '\n')
  251. attributes[key] = text
  252. self.append_node(parsetree.Tag, keyword, attributes)
  253. if isend:
  254. self.tag.pop()
  255. else:
  256. if keyword == 'text':
  257. match = self.match(r'(.*?)(?=\</%text>)', re.S)
  258. if not match:
  259. raise exceptions.SyntaxException(
  260. "Unclosed tag: <%%%s>" %
  261. self.tag[-1].keyword,
  262. **self.exception_kwargs)
  263. self.append_node(parsetree.Text, match.group(1))
  264. return self.match_tag_end()
  265. return True
  266. else:
  267. return False
  268. def match_tag_end(self):
  269. match = self.match(r'\</%[\t ]*(.+?)[\t ]*>')
  270. if match:
  271. if not len(self.tag):
  272. raise exceptions.SyntaxException(
  273. "Closing tag without opening tag: </%%%s>" %
  274. match.group(1),
  275. **self.exception_kwargs)
  276. elif self.tag[-1].keyword != match.group(1):
  277. raise exceptions.SyntaxException(
  278. "Closing tag </%%%s> does not match tag: <%%%s>" %
  279. (match.group(1), self.tag[-1].keyword),
  280. **self.exception_kwargs)
  281. self.tag.pop()
  282. return True
  283. else:
  284. return False
  285. def match_end(self):
  286. match = self.match(r'\Z', re.S)
  287. if match:
  288. string = match.group()
  289. if string:
  290. return string
  291. else:
  292. return True
  293. else:
  294. return False
  295. def match_text(self):
  296. match = self.match(r"""
  297. (.*?) # anything, followed by:
  298. (
  299. (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
  300. # comment preceded by a
  301. # consumed newline and whitespace
  302. |
  303. (?=\${) # an expression
  304. |
  305. (?=\#\*) # multiline comment
  306. |
  307. (?=</?[%&]) # a substitution or block or call start or end
  308. # - don't consume
  309. |
  310. (\\\r?\n) # an escaped newline - throw away
  311. |
  312. \Z # end of string
  313. )""", re.X | re.S)
  314. if match:
  315. text = match.group(1)
  316. if text:
  317. self.append_node(parsetree.Text, text)
  318. return True
  319. else:
  320. return False
  321. def match_python_block(self):
  322. match = self.match(r"<%(!)?")
  323. if match:
  324. line, pos = self.matched_lineno, self.matched_charpos
  325. text, end = self.parse_until_text(r'%>')
  326. # the trailing newline helps
  327. # compiler.parse() not complain about indentation
  328. text = adjust_whitespace(text) + "\n"
  329. self.append_node(
  330. parsetree.Code,
  331. text,
  332. match.group(1)=='!', lineno=line, pos=pos)
  333. return True
  334. else:
  335. return False
  336. def match_expression(self):
  337. match = self.match(r"\${")
  338. if match:
  339. line, pos = self.matched_lineno, self.matched_charpos
  340. text, end = self.parse_until_text(r'\|', r'}')
  341. if end == '|':
  342. escapes, end = self.parse_until_text(r'}')
  343. else:
  344. escapes = ""
  345. text = text.replace('\r\n', '\n')
  346. self.append_node(
  347. parsetree.Expression,
  348. text, escapes.strip(),
  349. lineno=line, pos=pos)
  350. return True
  351. else:
  352. return False
  353. def match_control_line(self):
  354. match = self.match(
  355. r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)"
  356. r"(?:\r?\n|\Z)", re.M)
  357. if match:
  358. operator = match.group(1)
  359. text = match.group(2)
  360. if operator == '%':
  361. m2 = re.match(r'(end)?(\w+)\s*(.*)', text)
  362. if not m2:
  363. raise exceptions.SyntaxException(
  364. "Invalid control line: '%s'" %
  365. text,
  366. **self.exception_kwargs)
  367. isend, keyword = m2.group(1, 2)
  368. isend = (isend is not None)
  369. if isend:
  370. if not len(self.control_line):
  371. raise exceptions.SyntaxException(
  372. "No starting keyword '%s' for '%s'" %
  373. (keyword, text),
  374. **self.exception_kwargs)
  375. elif self.control_line[-1].keyword != keyword:
  376. raise exceptions.SyntaxException(
  377. "Keyword '%s' doesn't match keyword '%s'" %
  378. (text, self.control_line[-1].keyword),
  379. **self.exception_kwargs)
  380. self.append_node(parsetree.ControlLine, keyword, isend, text)
  381. else:
  382. self.append_node(parsetree.Comment, text)
  383. return True
  384. else:
  385. return False
  386. def match_comment(self):
  387. """matches the multiline version of a comment"""
  388. match = self.match(r"<%doc>(.*?)</%doc>", re.S)
  389. if match:
  390. self.append_node(parsetree.Comment, match.group(1))
  391. return True
  392. else:
  393. return False