pyparser.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551
  1. # mako/pyparser.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """Handles parsing of Python code.
  7. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
  8. module is used.
  9. """
  10. from mako import exceptions, util, compat
  11. from mako.compat import StringIO
  12. import operator
  13. if compat.py3k:
  14. # words that cannot be assigned to (notably
  15. # smaller than the total keys in __builtins__)
  16. reserved = set(['True', 'False', 'None', 'print'])
  17. # the "id" attribute on a function node
  18. arg_id = operator.attrgetter('arg')
  19. else:
  20. # words that cannot be assigned to (notably
  21. # smaller than the total keys in __builtins__)
  22. reserved = set(['True', 'False', 'None'])
  23. # the "id" attribute on a function node
  24. arg_id = operator.attrgetter('id')
  25. try:
  26. import _ast
  27. util.restore__ast(_ast)
  28. from mako import _ast_util
  29. except ImportError:
  30. _ast = None
  31. from compiler import parse as compiler_parse
  32. from compiler import visitor
  33. def parse(code, mode='exec', **exception_kwargs):
  34. """Parse an expression into AST"""
  35. try:
  36. if _ast:
  37. return _ast_util.parse(code, '<unknown>', mode)
  38. else:
  39. if isinstance(code, compat.text_type):
  40. code = code.encode('ascii', 'backslashreplace')
  41. return compiler_parse(code, mode)
  42. except Exception:
  43. raise exceptions.SyntaxException(
  44. "(%s) %s (%r)" % (
  45. compat.exception_as().__class__.__name__,
  46. compat.exception_as(),
  47. code[0:50]
  48. ), **exception_kwargs)
  49. if _ast:
  50. class FindIdentifiers(_ast_util.NodeVisitor):
  51. def __init__(self, listener, **exception_kwargs):
  52. self.in_function = False
  53. self.in_assign_targets = False
  54. self.local_ident_stack = set()
  55. self.listener = listener
  56. self.exception_kwargs = exception_kwargs
  57. def _add_declared(self, name):
  58. if not self.in_function:
  59. self.listener.declared_identifiers.add(name)
  60. else:
  61. self.local_ident_stack.add(name)
  62. def visit_ClassDef(self, node):
  63. self._add_declared(node.name)
  64. def visit_Assign(self, node):
  65. # flip around the visiting of Assign so the expression gets
  66. # evaluated first, in the case of a clause like "x=x+5" (x
  67. # is undeclared)
  68. self.visit(node.value)
  69. in_a = self.in_assign_targets
  70. self.in_assign_targets = True
  71. for n in node.targets:
  72. self.visit(n)
  73. self.in_assign_targets = in_a
  74. if compat.py3k:
  75. # ExceptHandler is in Python 2, but this block only works in
  76. # Python 3 (and is required there)
  77. def visit_ExceptHandler(self, node):
  78. if node.name is not None:
  79. self._add_declared(node.name)
  80. if node.type is not None:
  81. self.listener.undeclared_identifiers.add(node.type.id)
  82. for statement in node.body:
  83. self.visit(statement)
  84. def visit_Lambda(self, node, *args):
  85. self._visit_function(node, True)
  86. def visit_FunctionDef(self, node):
  87. self._add_declared(node.name)
  88. self._visit_function(node, False)
  89. def _expand_tuples(self, args):
  90. for arg in args:
  91. if isinstance(arg, _ast.Tuple):
  92. for n in arg.elts:
  93. yield n
  94. else:
  95. yield arg
  96. def _visit_function(self, node, islambda):
  97. # push function state onto stack. dont log any more
  98. # identifiers as "declared" until outside of the function,
  99. # but keep logging identifiers as "undeclared". track
  100. # argument names in each function header so they arent
  101. # counted as "undeclared"
  102. inf = self.in_function
  103. self.in_function = True
  104. local_ident_stack = self.local_ident_stack
  105. self.local_ident_stack = local_ident_stack.union([
  106. arg_id(arg) for arg in self._expand_tuples(node.args.args)
  107. ])
  108. if islambda:
  109. self.visit(node.body)
  110. else:
  111. for n in node.body:
  112. self.visit(n)
  113. self.in_function = inf
  114. self.local_ident_stack = local_ident_stack
  115. def visit_For(self, node):
  116. # flip around visit
  117. self.visit(node.iter)
  118. self.visit(node.target)
  119. for statement in node.body:
  120. self.visit(statement)
  121. for statement in node.orelse:
  122. self.visit(statement)
  123. def visit_Name(self, node):
  124. if isinstance(node.ctx, _ast.Store):
  125. # this is eqiuvalent to visit_AssName in
  126. # compiler
  127. self._add_declared(node.id)
  128. elif node.id not in reserved and node.id \
  129. not in self.listener.declared_identifiers and node.id \
  130. not in self.local_ident_stack:
  131. self.listener.undeclared_identifiers.add(node.id)
  132. def visit_Import(self, node):
  133. for name in node.names:
  134. if name.asname is not None:
  135. self._add_declared(name.asname)
  136. else:
  137. self._add_declared(name.name.split('.')[0])
  138. def visit_ImportFrom(self, node):
  139. for name in node.names:
  140. if name.asname is not None:
  141. self._add_declared(name.asname)
  142. else:
  143. if name.name == '*':
  144. raise exceptions.CompileException(
  145. "'import *' is not supported, since all identifier "
  146. "names must be explicitly declared. Please use the "
  147. "form 'from <modulename> import <name1>, <name2>, "
  148. "...' instead.", **self.exception_kwargs)
  149. self._add_declared(name.name)
  150. class FindTuple(_ast_util.NodeVisitor):
  151. def __init__(self, listener, code_factory, **exception_kwargs):
  152. self.listener = listener
  153. self.exception_kwargs = exception_kwargs
  154. self.code_factory = code_factory
  155. def visit_Tuple(self, node):
  156. for n in node.elts:
  157. p = self.code_factory(n, **self.exception_kwargs)
  158. self.listener.codeargs.append(p)
  159. self.listener.args.append(ExpressionGenerator(n).value())
  160. self.listener.declared_identifiers = \
  161. self.listener.declared_identifiers.union(
  162. p.declared_identifiers)
  163. self.listener.undeclared_identifiers = \
  164. self.listener.undeclared_identifiers.union(
  165. p.undeclared_identifiers)
  166. class ParseFunc(_ast_util.NodeVisitor):
  167. def __init__(self, listener, **exception_kwargs):
  168. self.listener = listener
  169. self.exception_kwargs = exception_kwargs
  170. def visit_FunctionDef(self, node):
  171. self.listener.funcname = node.name
  172. argnames = [arg_id(arg) for arg in node.args.args]
  173. if node.args.vararg:
  174. argnames.append(node.args.vararg)
  175. if node.args.kwarg:
  176. argnames.append(node.args.kwarg)
  177. self.listener.argnames = argnames
  178. self.listener.defaults = node.args.defaults # ast
  179. self.listener.varargs = node.args.vararg
  180. self.listener.kwargs = node.args.kwarg
  181. class ExpressionGenerator(object):
  182. def __init__(self, astnode):
  183. self.generator = _ast_util.SourceGenerator(' ' * 4)
  184. self.generator.visit(astnode)
  185. def value(self):
  186. return ''.join(self.generator.result)
  187. else:
  188. class FindIdentifiers(object):
  189. def __init__(self, listener, **exception_kwargs):
  190. self.in_function = False
  191. self.local_ident_stack = set()
  192. self.listener = listener
  193. self.exception_kwargs = exception_kwargs
  194. def _add_declared(self, name):
  195. if not self.in_function:
  196. self.listener.declared_identifiers.add(name)
  197. else:
  198. self.local_ident_stack.add(name)
  199. def visitClass(self, node, *args):
  200. self._add_declared(node.name)
  201. def visitAssName(self, node, *args):
  202. self._add_declared(node.name)
  203. def visitAssign(self, node, *args):
  204. # flip around the visiting of Assign so the expression gets
  205. # evaluated first, in the case of a clause like "x=x+5" (x
  206. # is undeclared)
  207. self.visit(node.expr, *args)
  208. for n in node.nodes:
  209. self.visit(n, *args)
  210. def visitLambda(self, node, *args):
  211. self._visit_function(node, args)
  212. def visitFunction(self, node, *args):
  213. self._add_declared(node.name)
  214. self._visit_function(node, args)
  215. def _expand_tuples(self, args):
  216. for arg in args:
  217. if isinstance(arg, tuple):
  218. for n in arg:
  219. yield n
  220. else:
  221. yield arg
  222. def _visit_function(self, node, args):
  223. # push function state onto stack. dont log any more
  224. # identifiers as "declared" until outside of the function,
  225. # but keep logging identifiers as "undeclared". track
  226. # argument names in each function header so they arent
  227. # counted as "undeclared"
  228. inf = self.in_function
  229. self.in_function = True
  230. local_ident_stack = self.local_ident_stack
  231. self.local_ident_stack = local_ident_stack.union([
  232. arg for arg in self._expand_tuples(node.argnames)
  233. ])
  234. for n in node.getChildNodes():
  235. self.visit(n, *args)
  236. self.in_function = inf
  237. self.local_ident_stack = local_ident_stack
  238. def visitFor(self, node, *args):
  239. # flip around visit
  240. self.visit(node.list, *args)
  241. self.visit(node.assign, *args)
  242. self.visit(node.body, *args)
  243. def visitName(self, node, *args):
  244. if node.name not in reserved and node.name \
  245. not in self.listener.declared_identifiers and node.name \
  246. not in self.local_ident_stack:
  247. self.listener.undeclared_identifiers.add(node.name)
  248. def visitImport(self, node, *args):
  249. for mod, alias in node.names:
  250. if alias is not None:
  251. self._add_declared(alias)
  252. else:
  253. self._add_declared(mod.split('.')[0])
  254. def visitFrom(self, node, *args):
  255. for mod, alias in node.names:
  256. if alias is not None:
  257. self._add_declared(alias)
  258. else:
  259. if mod == '*':
  260. raise exceptions.CompileException(
  261. "'import *' is not supported, since all identifier "
  262. "names must be explicitly declared. Please use the "
  263. "form 'from <modulename> import <name1>, <name2>, "
  264. "...' instead.", **self.exception_kwargs)
  265. self._add_declared(mod)
  266. def visit(self, expr):
  267. visitor.walk(expr, self) # , walker=walker())
  268. class FindTuple(object):
  269. def __init__(self, listener, code_factory, **exception_kwargs):
  270. self.listener = listener
  271. self.exception_kwargs = exception_kwargs
  272. self.code_factory = code_factory
  273. def visitTuple(self, node, *args):
  274. for n in node.nodes:
  275. p = self.code_factory(n, **self.exception_kwargs)
  276. self.listener.codeargs.append(p)
  277. self.listener.args.append(ExpressionGenerator(n).value())
  278. self.listener.declared_identifiers = \
  279. self.listener.declared_identifiers.union(
  280. p.declared_identifiers)
  281. self.listener.undeclared_identifiers = \
  282. self.listener.undeclared_identifiers.union(
  283. p.undeclared_identifiers)
  284. def visit(self, expr):
  285. visitor.walk(expr, self) # , walker=walker())
  286. class ParseFunc(object):
  287. def __init__(self, listener, **exception_kwargs):
  288. self.listener = listener
  289. self.exception_kwargs = exception_kwargs
  290. def visitFunction(self, node, *args):
  291. self.listener.funcname = node.name
  292. self.listener.argnames = node.argnames
  293. self.listener.defaults = node.defaults
  294. self.listener.varargs = node.varargs
  295. self.listener.kwargs = node.kwargs
  296. def visit(self, expr):
  297. visitor.walk(expr, self)
  298. class ExpressionGenerator(object):
  299. """given an AST node, generates an equivalent literal Python
  300. expression."""
  301. def __init__(self, astnode):
  302. self.buf = StringIO()
  303. visitor.walk(astnode, self) # , walker=walker())
  304. def value(self):
  305. return self.buf.getvalue()
  306. def operator(self, op, node, *args):
  307. self.buf.write('(')
  308. self.visit(node.left, *args)
  309. self.buf.write(' %s ' % op)
  310. self.visit(node.right, *args)
  311. self.buf.write(')')
  312. def booleanop(self, op, node, *args):
  313. self.visit(node.nodes[0])
  314. for n in node.nodes[1:]:
  315. self.buf.write(' ' + op + ' ')
  316. self.visit(n, *args)
  317. def visitConst(self, node, *args):
  318. self.buf.write(repr(node.value))
  319. def visitAssName(self, node, *args):
  320. # TODO: figure out OP_ASSIGN, other OP_s
  321. self.buf.write(node.name)
  322. def visitName(self, node, *args):
  323. self.buf.write(node.name)
  324. def visitMul(self, node, *args):
  325. self.operator('*', node, *args)
  326. def visitAnd(self, node, *args):
  327. self.booleanop('and', node, *args)
  328. def visitOr(self, node, *args):
  329. self.booleanop('or', node, *args)
  330. def visitBitand(self, node, *args):
  331. self.booleanop('&', node, *args)
  332. def visitBitor(self, node, *args):
  333. self.booleanop('|', node, *args)
  334. def visitBitxor(self, node, *args):
  335. self.booleanop('^', node, *args)
  336. def visitAdd(self, node, *args):
  337. self.operator('+', node, *args)
  338. def visitGetattr(self, node, *args):
  339. self.visit(node.expr, *args)
  340. self.buf.write('.%s' % node.attrname)
  341. def visitSub(self, node, *args):
  342. self.operator('-', node, *args)
  343. def visitNot(self, node, *args):
  344. self.buf.write('not ')
  345. self.visit(node.expr)
  346. def visitDiv(self, node, *args):
  347. self.operator('/', node, *args)
  348. def visitFloorDiv(self, node, *args):
  349. self.operator('//', node, *args)
  350. def visitSubscript(self, node, *args):
  351. self.visit(node.expr)
  352. self.buf.write('[')
  353. [self.visit(x) for x in node.subs]
  354. self.buf.write(']')
  355. def visitUnarySub(self, node, *args):
  356. self.buf.write('-')
  357. self.visit(node.expr)
  358. def visitUnaryAdd(self, node, *args):
  359. self.buf.write('-')
  360. self.visit(node.expr)
  361. def visitSlice(self, node, *args):
  362. self.visit(node.expr)
  363. self.buf.write('[')
  364. if node.lower is not None:
  365. self.visit(node.lower)
  366. self.buf.write(':')
  367. if node.upper is not None:
  368. self.visit(node.upper)
  369. self.buf.write(']')
  370. def visitDict(self, node):
  371. self.buf.write('{')
  372. c = node.getChildren()
  373. for i in range(0, len(c), 2):
  374. self.visit(c[i])
  375. self.buf.write(': ')
  376. self.visit(c[i + 1])
  377. if i < len(c) - 2:
  378. self.buf.write(', ')
  379. self.buf.write('}')
  380. def visitTuple(self, node):
  381. self.buf.write('(')
  382. c = node.getChildren()
  383. for i in range(0, len(c)):
  384. self.visit(c[i])
  385. if i < len(c) - 1:
  386. self.buf.write(', ')
  387. self.buf.write(')')
  388. def visitList(self, node):
  389. self.buf.write('[')
  390. c = node.getChildren()
  391. for i in range(0, len(c)):
  392. self.visit(c[i])
  393. if i < len(c) - 1:
  394. self.buf.write(', ')
  395. self.buf.write(']')
  396. def visitListComp(self, node):
  397. self.buf.write('[')
  398. self.visit(node.expr)
  399. self.buf.write(' ')
  400. for n in node.quals:
  401. self.visit(n)
  402. self.buf.write(']')
  403. def visitListCompFor(self, node):
  404. self.buf.write(' for ')
  405. self.visit(node.assign)
  406. self.buf.write(' in ')
  407. self.visit(node.list)
  408. for n in node.ifs:
  409. self.visit(n)
  410. def visitListCompIf(self, node):
  411. self.buf.write(' if ')
  412. self.visit(node.test)
  413. def visitCompare(self, node):
  414. self.visit(node.expr)
  415. for tup in node.ops:
  416. self.buf.write(tup[0])
  417. self.visit(tup[1])
  418. def visitCallFunc(self, node, *args):
  419. self.visit(node.node)
  420. self.buf.write('(')
  421. if len(node.args):
  422. self.visit(node.args[0])
  423. for a in node.args[1:]:
  424. self.buf.write(', ')
  425. self.visit(a)
  426. self.buf.write(')')
  427. class walker(visitor.ASTVisitor):
  428. def dispatch(self, node, *args):
  429. print('Node:', str(node))
  430. # print "dir:", dir(node)
  431. return visitor.ASTVisitor.dispatch(self, node, *args)