pyparser.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
  1. # mako/pyparser.py
  2. # Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """Handles parsing of Python code.
  7. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
  8. module is used.
  9. """
  10. from StringIO import StringIO
  11. from mako import exceptions, util
  12. import operator
  13. if util.py3k:
  14. # words that cannot be assigned to (notably
  15. # smaller than the total keys in __builtins__)
  16. reserved = set(['True', 'False', 'None', 'print'])
  17. # the "id" attribute on a function node
  18. arg_id = operator.attrgetter('arg')
  19. else:
  20. # words that cannot be assigned to (notably
  21. # smaller than the total keys in __builtins__)
  22. reserved = set(['True', 'False', 'None'])
  23. # the "id" attribute on a function node
  24. arg_id = operator.attrgetter('id')
  25. try:
  26. import _ast
  27. util.restore__ast(_ast)
  28. import _ast_util
  29. except ImportError:
  30. _ast = None
  31. from compiler import parse as compiler_parse
  32. from compiler import visitor
  33. def parse(code, mode='exec', **exception_kwargs):
  34. """Parse an expression into AST"""
  35. try:
  36. if _ast:
  37. return _ast_util.parse(code, '<unknown>', mode)
  38. else:
  39. if isinstance(code, unicode):
  40. code = code.encode('ascii', 'backslashreplace')
  41. return compiler_parse(code, mode)
  42. except Exception, e:
  43. raise exceptions.SyntaxException(
  44. "(%s) %s (%r)" % (
  45. e.__class__.__name__,
  46. e,
  47. code[0:50]
  48. ), **exception_kwargs)
  49. if _ast:
  50. class FindIdentifiers(_ast_util.NodeVisitor):
  51. def __init__(self, listener, **exception_kwargs):
  52. self.in_function = False
  53. self.in_assign_targets = False
  54. self.local_ident_stack = set()
  55. self.listener = listener
  56. self.exception_kwargs = exception_kwargs
  57. def _add_declared(self, name):
  58. if not self.in_function:
  59. self.listener.declared_identifiers.add(name)
  60. else:
  61. self.local_ident_stack.add(name)
  62. def visit_ClassDef(self, node):
  63. self._add_declared(node.name)
  64. def visit_Assign(self, node):
  65. # flip around the visiting of Assign so the expression gets
  66. # evaluated first, in the case of a clause like "x=x+5" (x
  67. # is undeclared)
  68. self.visit(node.value)
  69. in_a = self.in_assign_targets
  70. self.in_assign_targets = True
  71. for n in node.targets:
  72. self.visit(n)
  73. self.in_assign_targets = in_a
  74. if util.py3k:
  75. # ExceptHandler is in Python 2, but this block only works in
  76. # Python 3 (and is required there)
  77. def visit_ExceptHandler(self, node):
  78. if node.name is not None:
  79. self._add_declared(node.name)
  80. if node.type is not None:
  81. self.listener.undeclared_identifiers.add(node.type.id)
  82. for statement in node.body:
  83. self.visit(statement)
  84. def visit_Lambda(self, node, *args):
  85. self._visit_function(node, True)
  86. def visit_FunctionDef(self, node):
  87. self._add_declared(node.name)
  88. self._visit_function(node, False)
  89. def _visit_function(self, node, islambda):
  90. # push function state onto stack. dont log any more
  91. # identifiers as "declared" until outside of the function,
  92. # but keep logging identifiers as "undeclared". track
  93. # argument names in each function header so they arent
  94. # counted as "undeclared"
  95. inf = self.in_function
  96. self.in_function = True
  97. local_ident_stack = self.local_ident_stack
  98. self.local_ident_stack = local_ident_stack.union([
  99. arg_id(arg) for arg in node.args.args
  100. ])
  101. if islambda:
  102. self.visit(node.body)
  103. else:
  104. for n in node.body:
  105. self.visit(n)
  106. self.in_function = inf
  107. self.local_ident_stack = local_ident_stack
  108. def visit_For(self, node):
  109. # flip around visit
  110. self.visit(node.iter)
  111. self.visit(node.target)
  112. for statement in node.body:
  113. self.visit(statement)
  114. for statement in node.orelse:
  115. self.visit(statement)
  116. def visit_Name(self, node):
  117. if isinstance(node.ctx, _ast.Store):
  118. # this is eqiuvalent to visit_AssName in
  119. # compiler
  120. self._add_declared(node.id)
  121. elif node.id not in reserved and node.id \
  122. not in self.listener.declared_identifiers and node.id \
  123. not in self.local_ident_stack:
  124. self.listener.undeclared_identifiers.add(node.id)
  125. def visit_Import(self, node):
  126. for name in node.names:
  127. if name.asname is not None:
  128. self._add_declared(name.asname)
  129. else:
  130. self._add_declared(name.name.split('.')[0])
  131. def visit_ImportFrom(self, node):
  132. for name in node.names:
  133. if name.asname is not None:
  134. self._add_declared(name.asname)
  135. else:
  136. if name.name == '*':
  137. raise exceptions.CompileException(
  138. "'import *' is not supported, since all identifier "
  139. "names must be explicitly declared. Please use the "
  140. "form 'from <modulename> import <name1>, <name2>, "
  141. "...' instead.", **self.exception_kwargs)
  142. self._add_declared(name.name)
  143. class FindTuple(_ast_util.NodeVisitor):
  144. def __init__(self, listener, code_factory, **exception_kwargs):
  145. self.listener = listener
  146. self.exception_kwargs = exception_kwargs
  147. self.code_factory = code_factory
  148. def visit_Tuple(self, node):
  149. for n in node.elts:
  150. p = self.code_factory(n, **self.exception_kwargs)
  151. self.listener.codeargs.append(p)
  152. self.listener.args.append(ExpressionGenerator(n).value())
  153. self.listener.declared_identifiers = \
  154. self.listener.declared_identifiers.union(
  155. p.declared_identifiers)
  156. self.listener.undeclared_identifiers = \
  157. self.listener.undeclared_identifiers.union(
  158. p.undeclared_identifiers)
  159. class ParseFunc(_ast_util.NodeVisitor):
  160. def __init__(self, listener, **exception_kwargs):
  161. self.listener = listener
  162. self.exception_kwargs = exception_kwargs
  163. def visit_FunctionDef(self, node):
  164. self.listener.funcname = node.name
  165. argnames = [arg_id(arg) for arg in node.args.args]
  166. if node.args.vararg:
  167. argnames.append(node.args.vararg)
  168. if node.args.kwarg:
  169. argnames.append(node.args.kwarg)
  170. self.listener.argnames = argnames
  171. self.listener.defaults = node.args.defaults # ast
  172. self.listener.varargs = node.args.vararg
  173. self.listener.kwargs = node.args.kwarg
  174. class ExpressionGenerator(object):
  175. def __init__(self, astnode):
  176. self.generator = _ast_util.SourceGenerator(' ' * 4)
  177. self.generator.visit(astnode)
  178. def value(self):
  179. return ''.join(self.generator.result)
  180. else:
  181. class FindIdentifiers(object):
  182. def __init__(self, listener, **exception_kwargs):
  183. self.in_function = False
  184. self.local_ident_stack = set()
  185. self.listener = listener
  186. self.exception_kwargs = exception_kwargs
  187. def _add_declared(self, name):
  188. if not self.in_function:
  189. self.listener.declared_identifiers.add(name)
  190. else:
  191. self.local_ident_stack.add(name)
  192. def visitClass(self, node, *args):
  193. self._add_declared(node.name)
  194. def visitAssName(self, node, *args):
  195. self._add_declared(node.name)
  196. def visitAssign(self, node, *args):
  197. # flip around the visiting of Assign so the expression gets
  198. # evaluated first, in the case of a clause like "x=x+5" (x
  199. # is undeclared)
  200. self.visit(node.expr, *args)
  201. for n in node.nodes:
  202. self.visit(n, *args)
  203. def visitLambda(self, node, *args):
  204. self._visit_function(node, args)
  205. def visitFunction(self, node, *args):
  206. self._add_declared(node.name)
  207. self._visit_function(node, args)
  208. def _visit_function(self, node, args):
  209. # push function state onto stack. dont log any more
  210. # identifiers as "declared" until outside of the function,
  211. # but keep logging identifiers as "undeclared". track
  212. # argument names in each function header so they arent
  213. # counted as "undeclared"
  214. inf = self.in_function
  215. self.in_function = True
  216. local_ident_stack = self.local_ident_stack
  217. self.local_ident_stack = local_ident_stack.union([
  218. arg for arg in node.argnames
  219. ])
  220. for n in node.getChildNodes():
  221. self.visit(n, *args)
  222. self.in_function = inf
  223. self.local_ident_stack = local_ident_stack
  224. def visitFor(self, node, *args):
  225. # flip around visit
  226. self.visit(node.list, *args)
  227. self.visit(node.assign, *args)
  228. self.visit(node.body, *args)
  229. def visitName(self, node, *args):
  230. if node.name not in reserved and node.name \
  231. not in self.listener.declared_identifiers and node.name \
  232. not in self.local_ident_stack:
  233. self.listener.undeclared_identifiers.add(node.name)
  234. def visitImport(self, node, *args):
  235. for mod, alias in node.names:
  236. if alias is not None:
  237. self._add_declared(alias)
  238. else:
  239. self._add_declared(mod.split('.')[0])
  240. def visitFrom(self, node, *args):
  241. for mod, alias in node.names:
  242. if alias is not None:
  243. self._add_declared(alias)
  244. else:
  245. if mod == '*':
  246. raise exceptions.CompileException(
  247. "'import *' is not supported, since all identifier "
  248. "names must be explicitly declared. Please use the "
  249. "form 'from <modulename> import <name1>, <name2>, "
  250. "...' instead.", **self.exception_kwargs)
  251. self._add_declared(mod)
  252. def visit(self, expr):
  253. visitor.walk(expr, self) # , walker=walker())
  254. class FindTuple(object):
  255. def __init__(self, listener, code_factory, **exception_kwargs):
  256. self.listener = listener
  257. self.exception_kwargs = exception_kwargs
  258. self.code_factory = code_factory
  259. def visitTuple(self, node, *args):
  260. for n in node.nodes:
  261. p = self.code_factory(n, **self.exception_kwargs)
  262. self.listener.codeargs.append(p)
  263. self.listener.args.append(ExpressionGenerator(n).value())
  264. self.listener.declared_identifiers = \
  265. self.listener.declared_identifiers.union(
  266. p.declared_identifiers)
  267. self.listener.undeclared_identifiers = \
  268. self.listener.undeclared_identifiers.union(
  269. p.undeclared_identifiers)
  270. def visit(self, expr):
  271. visitor.walk(expr, self) # , walker=walker())
  272. class ParseFunc(object):
  273. def __init__(self, listener, **exception_kwargs):
  274. self.listener = listener
  275. self.exception_kwargs = exception_kwargs
  276. def visitFunction(self, node, *args):
  277. self.listener.funcname = node.name
  278. self.listener.argnames = node.argnames
  279. self.listener.defaults = node.defaults
  280. self.listener.varargs = node.varargs
  281. self.listener.kwargs = node.kwargs
  282. def visit(self, expr):
  283. visitor.walk(expr, self)
  284. class ExpressionGenerator(object):
  285. """given an AST node, generates an equivalent literal Python
  286. expression."""
  287. def __init__(self, astnode):
  288. self.buf = StringIO()
  289. visitor.walk(astnode, self) # , walker=walker())
  290. def value(self):
  291. return self.buf.getvalue()
  292. def operator(self, op, node, *args):
  293. self.buf.write('(')
  294. self.visit(node.left, *args)
  295. self.buf.write(' %s ' % op)
  296. self.visit(node.right, *args)
  297. self.buf.write(')')
  298. def booleanop(self, op, node, *args):
  299. self.visit(node.nodes[0])
  300. for n in node.nodes[1:]:
  301. self.buf.write(' ' + op + ' ')
  302. self.visit(n, *args)
  303. def visitConst(self, node, *args):
  304. self.buf.write(repr(node.value))
  305. def visitAssName(self, node, *args):
  306. # TODO: figure out OP_ASSIGN, other OP_s
  307. self.buf.write(node.name)
  308. def visitName(self, node, *args):
  309. self.buf.write(node.name)
  310. def visitMul(self, node, *args):
  311. self.operator('*', node, *args)
  312. def visitAnd(self, node, *args):
  313. self.booleanop('and', node, *args)
  314. def visitOr(self, node, *args):
  315. self.booleanop('or', node, *args)
  316. def visitBitand(self, node, *args):
  317. self.booleanop('&', node, *args)
  318. def visitBitor(self, node, *args):
  319. self.booleanop('|', node, *args)
  320. def visitBitxor(self, node, *args):
  321. self.booleanop('^', node, *args)
  322. def visitAdd(self, node, *args):
  323. self.operator('+', node, *args)
  324. def visitGetattr(self, node, *args):
  325. self.visit(node.expr, *args)
  326. self.buf.write('.%s' % node.attrname)
  327. def visitSub(self, node, *args):
  328. self.operator('-', node, *args)
  329. def visitNot(self, node, *args):
  330. self.buf.write('not ')
  331. self.visit(node.expr)
  332. def visitDiv(self, node, *args):
  333. self.operator('/', node, *args)
  334. def visitFloorDiv(self, node, *args):
  335. self.operator('//', node, *args)
  336. def visitSubscript(self, node, *args):
  337. self.visit(node.expr)
  338. self.buf.write('[')
  339. [self.visit(x) for x in node.subs]
  340. self.buf.write(']')
  341. def visitUnarySub(self, node, *args):
  342. self.buf.write('-')
  343. self.visit(node.expr)
  344. def visitUnaryAdd(self, node, *args):
  345. self.buf.write('-')
  346. self.visit(node.expr)
  347. def visitSlice(self, node, *args):
  348. self.visit(node.expr)
  349. self.buf.write('[')
  350. if node.lower is not None:
  351. self.visit(node.lower)
  352. self.buf.write(':')
  353. if node.upper is not None:
  354. self.visit(node.upper)
  355. self.buf.write(']')
  356. def visitDict(self, node):
  357. self.buf.write('{')
  358. c = node.getChildren()
  359. for i in range(0, len(c), 2):
  360. self.visit(c[i])
  361. self.buf.write(': ')
  362. self.visit(c[i + 1])
  363. if i < len(c) - 2:
  364. self.buf.write(', ')
  365. self.buf.write('}')
  366. def visitTuple(self, node):
  367. self.buf.write('(')
  368. c = node.getChildren()
  369. for i in range(0, len(c)):
  370. self.visit(c[i])
  371. if i < len(c) - 1:
  372. self.buf.write(', ')
  373. self.buf.write(')')
  374. def visitList(self, node):
  375. self.buf.write('[')
  376. c = node.getChildren()
  377. for i in range(0, len(c)):
  378. self.visit(c[i])
  379. if i < len(c) - 1:
  380. self.buf.write(', ')
  381. self.buf.write(']')
  382. def visitListComp(self, node):
  383. self.buf.write('[')
  384. self.visit(node.expr)
  385. self.buf.write(' ')
  386. for n in node.quals:
  387. self.visit(n)
  388. self.buf.write(']')
  389. def visitListCompFor(self, node):
  390. self.buf.write(' for ')
  391. self.visit(node.assign)
  392. self.buf.write(' in ')
  393. self.visit(node.list)
  394. for n in node.ifs:
  395. self.visit(n)
  396. def visitListCompIf(self, node):
  397. self.buf.write(' if ')
  398. self.visit(node.test)
  399. def visitCompare(self, node):
  400. self.visit(node.expr)
  401. for tup in node.ops:
  402. self.buf.write(tup[0])
  403. self.visit(tup[1])
  404. def visitCallFunc(self, node, *args):
  405. self.visit(node.node)
  406. self.buf.write('(')
  407. if len(node.args):
  408. self.visit(node.args[0])
  409. for a in node.args[1:]:
  410. self.buf.write(', ')
  411. self.visit(a)
  412. self.buf.write(')')
  413. class walker(visitor.ASTVisitor):
  414. def dispatch(self, node, *args):
  415. print 'Node:', str(node)
  416. # print "dir:", dir(node)
  417. return visitor.ASTVisitor.dispatch(self, node, *args)