parsetree.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589
  1. # mako/parsetree.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """defines the parse tree components for Mako templates."""
  7. from mako import exceptions, ast, util, filters
  8. import re
  9. class Node(object):
  10. """base class for a Node in the parse tree."""
  11. def __init__(self, source, lineno, pos, filename):
  12. self.source = source
  13. self.lineno = lineno
  14. self.pos = pos
  15. self.filename = filename
  16. @property
  17. def exception_kwargs(self):
  18. return {'source':self.source, 'lineno':self.lineno,
  19. 'pos':self.pos, 'filename':self.filename}
  20. def get_children(self):
  21. return []
  22. def accept_visitor(self, visitor):
  23. def traverse(node):
  24. for n in node.get_children():
  25. n.accept_visitor(visitor)
  26. method = getattr(visitor, "visit" + self.__class__.__name__, traverse)
  27. method(self)
  28. class TemplateNode(Node):
  29. """a 'container' node that stores the overall collection of nodes."""
  30. def __init__(self, filename):
  31. super(TemplateNode, self).__init__('', 0, 0, filename)
  32. self.nodes = []
  33. self.page_attributes = {}
  34. def get_children(self):
  35. return self.nodes
  36. def __repr__(self):
  37. return "TemplateNode(%s, %r)" % (
  38. util.sorted_dict_repr(self.page_attributes),
  39. self.nodes)
  40. class ControlLine(Node):
  41. """defines a control line, a line-oriented python line or end tag.
  42. e.g.::
  43. % if foo:
  44. (markup)
  45. % endif
  46. """
  47. has_loop_context = False
  48. def __init__(self, keyword, isend, text, **kwargs):
  49. super(ControlLine, self).__init__(**kwargs)
  50. self.text = text
  51. self.keyword = keyword
  52. self.isend = isend
  53. self.is_primary = keyword in ['for', 'if', 'while', 'try', 'with']
  54. self.nodes = []
  55. if self.isend:
  56. self._declared_identifiers = []
  57. self._undeclared_identifiers = []
  58. else:
  59. code = ast.PythonFragment(text, **self.exception_kwargs)
  60. self._declared_identifiers = code.declared_identifiers
  61. self._undeclared_identifiers = code.undeclared_identifiers
  62. def get_children(self):
  63. return self.nodes
  64. def declared_identifiers(self):
  65. return self._declared_identifiers
  66. def undeclared_identifiers(self):
  67. return self._undeclared_identifiers
  68. def is_ternary(self, keyword):
  69. """return true if the given keyword is a ternary keyword
  70. for this ControlLine"""
  71. return keyword in {
  72. 'if':set(['else', 'elif']),
  73. 'try':set(['except', 'finally']),
  74. 'for':set(['else'])
  75. }.get(self.keyword, [])
  76. def __repr__(self):
  77. return "ControlLine(%r, %r, %r, %r)" % (
  78. self.keyword,
  79. self.text,
  80. self.isend,
  81. (self.lineno, self.pos)
  82. )
  83. class Text(Node):
  84. """defines plain text in the template."""
  85. def __init__(self, content, **kwargs):
  86. super(Text, self).__init__(**kwargs)
  87. self.content = content
  88. def __repr__(self):
  89. return "Text(%r, %r)" % (self.content, (self.lineno, self.pos))
  90. class Code(Node):
  91. """defines a Python code block, either inline or module level.
  92. e.g.::
  93. inline:
  94. <%
  95. x = 12
  96. %>
  97. module level:
  98. <%!
  99. import logger
  100. %>
  101. """
  102. def __init__(self, text, ismodule, **kwargs):
  103. super(Code, self).__init__(**kwargs)
  104. self.text = text
  105. self.ismodule = ismodule
  106. self.code = ast.PythonCode(text, **self.exception_kwargs)
  107. def declared_identifiers(self):
  108. return self.code.declared_identifiers
  109. def undeclared_identifiers(self):
  110. return self.code.undeclared_identifiers
  111. def __repr__(self):
  112. return "Code(%r, %r, %r)" % (
  113. self.text,
  114. self.ismodule,
  115. (self.lineno, self.pos)
  116. )
  117. class Comment(Node):
  118. """defines a comment line.
  119. # this is a comment
  120. """
  121. def __init__(self, text, **kwargs):
  122. super(Comment, self).__init__(**kwargs)
  123. self.text = text
  124. def __repr__(self):
  125. return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos))
  126. class Expression(Node):
  127. """defines an inline expression.
  128. ${x+y}
  129. """
  130. def __init__(self, text, escapes, **kwargs):
  131. super(Expression, self).__init__(**kwargs)
  132. self.text = text
  133. self.escapes = escapes
  134. self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs)
  135. self.code = ast.PythonCode(text, **self.exception_kwargs)
  136. def declared_identifiers(self):
  137. return []
  138. def undeclared_identifiers(self):
  139. # TODO: make the "filter" shortcut list configurable at parse/gen time
  140. return self.code.undeclared_identifiers.union(
  141. self.escapes_code.undeclared_identifiers.difference(
  142. set(filters.DEFAULT_ESCAPES.keys())
  143. )
  144. ).difference(self.code.declared_identifiers)
  145. def __repr__(self):
  146. return "Expression(%r, %r, %r)" % (
  147. self.text,
  148. self.escapes_code.args,
  149. (self.lineno, self.pos)
  150. )
  151. class _TagMeta(type):
  152. """metaclass to allow Tag to produce a subclass according to
  153. its keyword"""
  154. _classmap = {}
  155. def __init__(cls, clsname, bases, dict):
  156. if cls.__keyword__ is not None:
  157. cls._classmap[cls.__keyword__] = cls
  158. super(_TagMeta, cls).__init__(clsname, bases, dict)
  159. def __call__(cls, keyword, attributes, **kwargs):
  160. if ":" in keyword:
  161. ns, defname = keyword.split(':')
  162. return type.__call__(CallNamespaceTag, ns, defname,
  163. attributes, **kwargs)
  164. try:
  165. cls = _TagMeta._classmap[keyword]
  166. except KeyError:
  167. raise exceptions.CompileException(
  168. "No such tag: '%s'" % keyword,
  169. source=kwargs['source'],
  170. lineno=kwargs['lineno'],
  171. pos=kwargs['pos'],
  172. filename=kwargs['filename']
  173. )
  174. return type.__call__(cls, keyword, attributes, **kwargs)
  175. class Tag(Node):
  176. """abstract base class for tags.
  177. <%sometag/>
  178. <%someothertag>
  179. stuff
  180. </%someothertag>
  181. """
  182. __metaclass__ = _TagMeta
  183. __keyword__ = None
  184. def __init__(self, keyword, attributes, expressions,
  185. nonexpressions, required, **kwargs):
  186. """construct a new Tag instance.
  187. this constructor not called directly, and is only called
  188. by subclasses.
  189. :param keyword: the tag keyword
  190. :param attributes: raw dictionary of attribute key/value pairs
  191. :param expressions: a set of identifiers that are legal attributes,
  192. which can also contain embedded expressions
  193. :param nonexpressions: a set of identifiers that are legal
  194. attributes, which cannot contain embedded expressions
  195. :param \**kwargs:
  196. other arguments passed to the Node superclass (lineno, pos)
  197. """
  198. super(Tag, self).__init__(**kwargs)
  199. self.keyword = keyword
  200. self.attributes = attributes
  201. self._parse_attributes(expressions, nonexpressions)
  202. missing = [r for r in required if r not in self.parsed_attributes]
  203. if len(missing):
  204. raise exceptions.CompileException(
  205. "Missing attribute(s): %s" %
  206. ",".join([repr(m) for m in missing]),
  207. **self.exception_kwargs)
  208. self.parent = None
  209. self.nodes = []
  210. def is_root(self):
  211. return self.parent is None
  212. def get_children(self):
  213. return self.nodes
  214. def _parse_attributes(self, expressions, nonexpressions):
  215. undeclared_identifiers = set()
  216. self.parsed_attributes = {}
  217. for key in self.attributes:
  218. if key in expressions:
  219. expr = []
  220. for x in re.compile(r'(\${.+?})',
  221. re.S).split(self.attributes[key]):
  222. m = re.compile(r'^\${(.+?)}$', re.S).match(x)
  223. if m:
  224. code = ast.PythonCode(m.group(1).rstrip(),
  225. **self.exception_kwargs)
  226. # we aren't discarding "declared_identifiers" here,
  227. # which we do so that list comprehension-declared
  228. # variables aren't counted. As yet can't find a
  229. # condition that requires it here.
  230. undeclared_identifiers = \
  231. undeclared_identifiers.union(
  232. code.undeclared_identifiers)
  233. expr.append('(%s)' % m.group(1))
  234. else:
  235. if x:
  236. expr.append(repr(x))
  237. self.parsed_attributes[key] = " + ".join(expr) or repr('')
  238. elif key in nonexpressions:
  239. if re.search(r'\${.+?}', self.attributes[key]):
  240. raise exceptions.CompileException(
  241. "Attibute '%s' in tag '%s' does not allow embedded "
  242. "expressions" % (key, self.keyword),
  243. **self.exception_kwargs)
  244. self.parsed_attributes[key] = repr(self.attributes[key])
  245. else:
  246. raise exceptions.CompileException(
  247. "Invalid attribute for tag '%s': '%s'" %
  248. (self.keyword, key),
  249. **self.exception_kwargs)
  250. self.expression_undeclared_identifiers = undeclared_identifiers
  251. def declared_identifiers(self):
  252. return []
  253. def undeclared_identifiers(self):
  254. return self.expression_undeclared_identifiers
  255. def __repr__(self):
  256. return "%s(%r, %s, %r, %r)" % (self.__class__.__name__,
  257. self.keyword,
  258. util.sorted_dict_repr(self.attributes),
  259. (self.lineno, self.pos),
  260. self.nodes
  261. )
  262. class IncludeTag(Tag):
  263. __keyword__ = 'include'
  264. def __init__(self, keyword, attributes, **kwargs):
  265. super(IncludeTag, self).__init__(
  266. keyword,
  267. attributes,
  268. ('file', 'import', 'args'),
  269. (), ('file',), **kwargs)
  270. self.page_args = ast.PythonCode(
  271. "__DUMMY(%s)" % attributes.get('args', ''),
  272. **self.exception_kwargs)
  273. def declared_identifiers(self):
  274. return []
  275. def undeclared_identifiers(self):
  276. identifiers = self.page_args.undeclared_identifiers.\
  277. difference(set(["__DUMMY"])).\
  278. difference(self.page_args.declared_identifiers)
  279. return identifiers.union(super(IncludeTag, self).
  280. undeclared_identifiers())
  281. class NamespaceTag(Tag):
  282. __keyword__ = 'namespace'
  283. def __init__(self, keyword, attributes, **kwargs):
  284. super(NamespaceTag, self).__init__(
  285. keyword, attributes,
  286. ('file',),
  287. ('name','inheritable',
  288. 'import','module'),
  289. (), **kwargs)
  290. self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
  291. if not 'name' in attributes and not 'import' in attributes:
  292. raise exceptions.CompileException(
  293. "'name' and/or 'import' attributes are required "
  294. "for <%namespace>",
  295. **self.exception_kwargs)
  296. if 'file' in attributes and 'module' in attributes:
  297. raise exceptions.CompileException(
  298. "<%namespace> may only have one of 'file' or 'module'",
  299. **self.exception_kwargs
  300. )
  301. def declared_identifiers(self):
  302. return []
  303. class TextTag(Tag):
  304. __keyword__ = 'text'
  305. def __init__(self, keyword, attributes, **kwargs):
  306. super(TextTag, self).__init__(
  307. keyword,
  308. attributes, (),
  309. ('filter'), (), **kwargs)
  310. self.filter_args = ast.ArgumentList(
  311. attributes.get('filter', ''),
  312. **self.exception_kwargs)
  313. class DefTag(Tag):
  314. __keyword__ = 'def'
  315. def __init__(self, keyword, attributes, **kwargs):
  316. expressions = ['buffered', 'cached'] + [
  317. c for c in attributes if c.startswith('cache_')]
  318. super(DefTag, self).__init__(
  319. keyword,
  320. attributes,
  321. expressions,
  322. ('name','filter', 'decorator'),
  323. ('name',),
  324. **kwargs)
  325. name = attributes['name']
  326. if re.match(r'^[\w_]+$',name):
  327. raise exceptions.CompileException(
  328. "Missing parenthesis in %def",
  329. **self.exception_kwargs)
  330. self.function_decl = ast.FunctionDecl("def " + name + ":pass",
  331. **self.exception_kwargs)
  332. self.name = self.function_decl.funcname
  333. self.decorator = attributes.get('decorator', '')
  334. self.filter_args = ast.ArgumentList(
  335. attributes.get('filter', ''),
  336. **self.exception_kwargs)
  337. is_anonymous = False
  338. is_block = False
  339. @property
  340. def funcname(self):
  341. return self.function_decl.funcname
  342. def get_argument_expressions(self, **kw):
  343. return self.function_decl.get_argument_expressions(**kw)
  344. def declared_identifiers(self):
  345. return self.function_decl.argnames
  346. def undeclared_identifiers(self):
  347. res = []
  348. for c in self.function_decl.defaults:
  349. res += list(ast.PythonCode(c, **self.exception_kwargs).
  350. undeclared_identifiers)
  351. return set(res).union(
  352. self.filter_args.\
  353. undeclared_identifiers.\
  354. difference(filters.DEFAULT_ESCAPES.keys())
  355. ).union(
  356. self.expression_undeclared_identifiers
  357. ).difference(
  358. self.function_decl.argnames
  359. )
  360. class BlockTag(Tag):
  361. __keyword__ = 'block'
  362. def __init__(self, keyword, attributes, **kwargs):
  363. expressions = ['buffered', 'cached', 'args'] + [
  364. c for c in attributes if c.startswith('cache_')]
  365. super(BlockTag, self).__init__(
  366. keyword,
  367. attributes,
  368. expressions,
  369. ('name','filter', 'decorator'),
  370. (),
  371. **kwargs)
  372. name = attributes.get('name')
  373. if name and not re.match(r'^[\w_]+$',name):
  374. raise exceptions.CompileException(
  375. "%block may not specify an argument signature",
  376. **self.exception_kwargs)
  377. if not name and attributes.get('args', None):
  378. raise exceptions.CompileException(
  379. "Only named %blocks may specify args",
  380. **self.exception_kwargs
  381. )
  382. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  383. **self.exception_kwargs)
  384. self.name = name
  385. self.decorator = attributes.get('decorator', '')
  386. self.filter_args = ast.ArgumentList(
  387. attributes.get('filter', ''),
  388. **self.exception_kwargs)
  389. is_block = True
  390. @property
  391. def is_anonymous(self):
  392. return self.name is None
  393. @property
  394. def funcname(self):
  395. return self.name or "__M_anon_%d" % (self.lineno, )
  396. def get_argument_expressions(self, **kw):
  397. return self.body_decl.get_argument_expressions(**kw)
  398. def declared_identifiers(self):
  399. return self.body_decl.argnames
  400. def undeclared_identifiers(self):
  401. return (self.filter_args.\
  402. undeclared_identifiers.\
  403. difference(filters.DEFAULT_ESCAPES.keys())
  404. ).union(self.expression_undeclared_identifiers)
  405. class CallTag(Tag):
  406. __keyword__ = 'call'
  407. def __init__(self, keyword, attributes, **kwargs):
  408. super(CallTag, self).__init__(keyword, attributes,
  409. ('args'), ('expr',), ('expr',), **kwargs)
  410. self.expression = attributes['expr']
  411. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  412. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  413. **self.exception_kwargs)
  414. def declared_identifiers(self):
  415. return self.code.declared_identifiers.union(self.body_decl.argnames)
  416. def undeclared_identifiers(self):
  417. return self.code.undeclared_identifiers.\
  418. difference(self.code.declared_identifiers)
  419. class CallNamespaceTag(Tag):
  420. def __init__(self, namespace, defname, attributes, **kwargs):
  421. super(CallNamespaceTag, self).__init__(
  422. namespace + ":" + defname,
  423. attributes,
  424. tuple(attributes.keys()) + ('args', ),
  425. (),
  426. (),
  427. **kwargs)
  428. self.expression = "%s.%s(%s)" % (
  429. namespace,
  430. defname,
  431. ",".join(["%s=%s" % (k, v) for k, v in
  432. self.parsed_attributes.iteritems()
  433. if k != 'args'])
  434. )
  435. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  436. self.body_decl = ast.FunctionArgs(
  437. attributes.get('args', ''),
  438. **self.exception_kwargs)
  439. def declared_identifiers(self):
  440. return self.code.declared_identifiers.union(self.body_decl.argnames)
  441. def undeclared_identifiers(self):
  442. return self.code.undeclared_identifiers.\
  443. difference(self.code.declared_identifiers)
  444. class InheritTag(Tag):
  445. __keyword__ = 'inherit'
  446. def __init__(self, keyword, attributes, **kwargs):
  447. super(InheritTag, self).__init__(
  448. keyword, attributes,
  449. ('file',), (), ('file',), **kwargs)
  450. class PageTag(Tag):
  451. __keyword__ = 'page'
  452. def __init__(self, keyword, attributes, **kwargs):
  453. expressions = ['cached', 'args', 'expression_filter', 'enable_loop'] + [
  454. c for c in attributes if c.startswith('cache_')]
  455. super(PageTag, self).__init__(
  456. keyword,
  457. attributes,
  458. expressions,
  459. (),
  460. (),
  461. **kwargs)
  462. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  463. **self.exception_kwargs)
  464. self.filter_args = ast.ArgumentList(
  465. attributes.get('expression_filter', ''),
  466. **self.exception_kwargs)
  467. def declared_identifiers(self):
  468. return self.body_decl.argnames