pyparser.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. # ast.py
  2. # Copyright (C) Mako developers
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """Handles parsing of Python code.
  7. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
  8. module is used.
  9. """
  10. from StringIO import StringIO
  11. from mako import exceptions, util
  12. import operator
  13. if util.py3k:
  14. # words that cannot be assigned to (notably
  15. # smaller than the total keys in __builtins__)
  16. reserved = set(['True', 'False', 'None', 'print'])
  17. # the "id" attribute on a function node
  18. arg_id = operator.attrgetter('arg')
  19. else:
  20. # words that cannot be assigned to (notably
  21. # smaller than the total keys in __builtins__)
  22. reserved = set(['True', 'False', 'None'])
  23. # the "id" attribute on a function node
  24. arg_id = operator.attrgetter('id')
  25. try:
  26. import _ast
  27. util.restore__ast(_ast)
  28. import _ast_util
  29. except ImportError:
  30. _ast = None
  31. from compiler import parse as compiler_parse
  32. from compiler import visitor
  33. def parse(code, mode='exec', **exception_kwargs):
  34. """Parse an expression into AST"""
  35. try:
  36. if _ast:
  37. return _ast_util.parse(code, '<unknown>', mode)
  38. else:
  39. if isinstance(code, unicode):
  40. code = code.encode('ascii', 'backslashreplace')
  41. return compiler_parse(code, mode)
  42. except Exception, e:
  43. raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
  44. if _ast:
  45. class FindIdentifiers(_ast_util.NodeVisitor):
  46. def __init__(self, listener, **exception_kwargs):
  47. self.in_function = False
  48. self.in_assign_targets = False
  49. self.local_ident_stack = {}
  50. self.listener = listener
  51. self.exception_kwargs = exception_kwargs
  52. def _add_declared(self, name):
  53. if not self.in_function:
  54. self.listener.declared_identifiers.add(name)
  55. def visit_ClassDef(self, node):
  56. self._add_declared(node.name)
  57. def visit_Assign(self, node):
  58. # flip around the visiting of Assign so the expression gets evaluated first,
  59. # in the case of a clause like "x=x+5" (x is undeclared)
  60. self.visit(node.value)
  61. in_a = self.in_assign_targets
  62. self.in_assign_targets = True
  63. for n in node.targets:
  64. self.visit(n)
  65. self.in_assign_targets = in_a
  66. if util.py3k:
  67. # ExceptHandler is in Python 2, but this
  68. # block only works in Python 3 (and is required there)
  69. def visit_ExceptHandler(self, node):
  70. if node.name is not None:
  71. self._add_declared(node.name)
  72. if node.type is not None:
  73. self.listener.undeclared_identifiers.add(node.type.id)
  74. for statement in node.body:
  75. self.visit(statement)
  76. def visit_FunctionDef(self, node):
  77. self._add_declared(node.name)
  78. # push function state onto stack. dont log any
  79. # more identifiers as "declared" until outside of the function,
  80. # but keep logging identifiers as "undeclared".
  81. # track argument names in each function header so they arent counted as "undeclared"
  82. saved = {}
  83. inf = self.in_function
  84. self.in_function = True
  85. for arg in node.args.args:
  86. if arg_id(arg) in self.local_ident_stack:
  87. saved[arg_id(arg)] = True
  88. else:
  89. self.local_ident_stack[arg_id(arg)] = True
  90. for n in node.body:
  91. self.visit(n)
  92. self.in_function = inf
  93. for arg in node.args.args:
  94. if arg_id(arg) not in saved:
  95. del self.local_ident_stack[arg_id(arg)]
  96. def visit_For(self, node):
  97. # flip around visit
  98. self.visit(node.iter)
  99. self.visit(node.target)
  100. for statement in node.body:
  101. self.visit(statement)
  102. for statement in node.orelse:
  103. self.visit(statement)
  104. def visit_Name(self, node):
  105. if isinstance(node.ctx, _ast.Store):
  106. self._add_declared(node.id)
  107. if node.id not in reserved and \
  108. node.id not in self.listener.declared_identifiers and \
  109. node.id not in self.local_ident_stack:
  110. self.listener.undeclared_identifiers.add(node.id)
  111. def visit_Import(self, node):
  112. for name in node.names:
  113. if name.asname is not None:
  114. self._add_declared(name.asname)
  115. else:
  116. self._add_declared(name.name.split('.')[0])
  117. def visit_ImportFrom(self, node):
  118. for name in node.names:
  119. if name.asname is not None:
  120. self._add_declared(name.asname)
  121. else:
  122. if name.name == '*':
  123. raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
  124. self._add_declared(name.name)
  125. class FindTuple(_ast_util.NodeVisitor):
  126. def __init__(self, listener, code_factory, **exception_kwargs):
  127. self.listener = listener
  128. self.exception_kwargs = exception_kwargs
  129. self.code_factory = code_factory
  130. def visit_Tuple(self, node):
  131. for n in node.elts:
  132. p = self.code_factory(n, **self.exception_kwargs)
  133. self.listener.codeargs.append(p)
  134. self.listener.args.append(ExpressionGenerator(n).value())
  135. self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
  136. self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)
  137. class ParseFunc(_ast_util.NodeVisitor):
  138. def __init__(self, listener, **exception_kwargs):
  139. self.listener = listener
  140. self.exception_kwargs = exception_kwargs
  141. def visit_FunctionDef(self, node):
  142. self.listener.funcname = node.name
  143. argnames = [arg_id(arg) for arg in node.args.args]
  144. if node.args.vararg:
  145. argnames.append(node.args.vararg)
  146. if node.args.kwarg:
  147. argnames.append(node.args.kwarg)
  148. self.listener.argnames = argnames
  149. self.listener.defaults = node.args.defaults # ast
  150. self.listener.varargs = node.args.vararg
  151. self.listener.kwargs = node.args.kwarg
  152. class ExpressionGenerator(object):
  153. def __init__(self, astnode):
  154. self.generator = _ast_util.SourceGenerator(' ' * 4)
  155. self.generator.visit(astnode)
  156. def value(self):
  157. return ''.join(self.generator.result)
  158. else:
  159. class FindIdentifiers(object):
  160. def __init__(self, listener, **exception_kwargs):
  161. self.in_function = False
  162. self.local_ident_stack = {}
  163. self.listener = listener
  164. self.exception_kwargs = exception_kwargs
  165. def _add_declared(self, name):
  166. if not self.in_function:
  167. self.listener.declared_identifiers.add(name)
  168. def visitClass(self, node, *args):
  169. self._add_declared(node.name)
  170. def visitAssName(self, node, *args):
  171. self._add_declared(node.name)
  172. def visitAssign(self, node, *args):
  173. # flip around the visiting of Assign so the expression gets evaluated first,
  174. # in the case of a clause like "x=x+5" (x is undeclared)
  175. self.visit(node.expr, *args)
  176. for n in node.nodes:
  177. self.visit(n, *args)
  178. def visitFunction(self,node, *args):
  179. self._add_declared(node.name)
  180. # push function state onto stack. dont log any
  181. # more identifiers as "declared" until outside of the function,
  182. # but keep logging identifiers as "undeclared".
  183. # track argument names in each function header so they arent counted as "undeclared"
  184. saved = {}
  185. inf = self.in_function
  186. self.in_function = True
  187. for arg in node.argnames:
  188. if arg in self.local_ident_stack:
  189. saved[arg] = True
  190. else:
  191. self.local_ident_stack[arg] = True
  192. for n in node.getChildNodes():
  193. self.visit(n, *args)
  194. self.in_function = inf
  195. for arg in node.argnames:
  196. if arg not in saved:
  197. del self.local_ident_stack[arg]
  198. def visitFor(self, node, *args):
  199. # flip around visit
  200. self.visit(node.list, *args)
  201. self.visit(node.assign, *args)
  202. self.visit(node.body, *args)
  203. def visitName(self, node, *args):
  204. if node.name not in reserved and node.name not in self.listener.declared_identifiers and node.name not in self.local_ident_stack:
  205. self.listener.undeclared_identifiers.add(node.name)
  206. def visitImport(self, node, *args):
  207. for (mod, alias) in node.names:
  208. if alias is not None:
  209. self._add_declared(alias)
  210. else:
  211. self._add_declared(mod.split('.')[0])
  212. def visitFrom(self, node, *args):
  213. for (mod, alias) in node.names:
  214. if alias is not None:
  215. self._add_declared(alias)
  216. else:
  217. if mod == '*':
  218. raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
  219. self._add_declared(mod)
  220. def visit(self, expr):
  221. visitor.walk(expr, self) #, walker=walker())
  222. class FindTuple(object):
  223. def __init__(self, listener, code_factory, **exception_kwargs):
  224. self.listener = listener
  225. self.exception_kwargs = exception_kwargs
  226. self.code_factory = code_factory
  227. def visitTuple(self, node, *args):
  228. for n in node.nodes:
  229. p = self.code_factory(n, **self.exception_kwargs)
  230. self.listener.codeargs.append(p)
  231. self.listener.args.append(ExpressionGenerator(n).value())
  232. self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
  233. self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)
  234. def visit(self, expr):
  235. visitor.walk(expr, self) #, walker=walker())
  236. class ParseFunc(object):
  237. def __init__(self, listener, **exception_kwargs):
  238. self.listener = listener
  239. self.exception_kwargs = exception_kwargs
  240. def visitFunction(self, node, *args):
  241. self.listener.funcname = node.name
  242. self.listener.argnames = node.argnames
  243. self.listener.defaults = node.defaults
  244. self.listener.varargs = node.varargs
  245. self.listener.kwargs = node.kwargs
  246. def visit(self, expr):
  247. visitor.walk(expr, self)
  248. class ExpressionGenerator(object):
  249. """given an AST node, generates an equivalent literal Python expression."""
  250. def __init__(self, astnode):
  251. self.buf = StringIO()
  252. visitor.walk(astnode, self) #, walker=walker())
  253. def value(self):
  254. return self.buf.getvalue()
  255. def operator(self, op, node, *args):
  256. self.buf.write("(")
  257. self.visit(node.left, *args)
  258. self.buf.write(" %s " % op)
  259. self.visit(node.right, *args)
  260. self.buf.write(")")
  261. def booleanop(self, op, node, *args):
  262. self.visit(node.nodes[0])
  263. for n in node.nodes[1:]:
  264. self.buf.write(" " + op + " ")
  265. self.visit(n, *args)
  266. def visitConst(self, node, *args):
  267. self.buf.write(repr(node.value))
  268. def visitAssName(self, node, *args):
  269. # TODO: figure out OP_ASSIGN, other OP_s
  270. self.buf.write(node.name)
  271. def visitName(self, node, *args):
  272. self.buf.write(node.name)
  273. def visitMul(self, node, *args):
  274. self.operator("*", node, *args)
  275. def visitAnd(self, node, *args):
  276. self.booleanop("and", node, *args)
  277. def visitOr(self, node, *args):
  278. self.booleanop("or", node, *args)
  279. def visitBitand(self, node, *args):
  280. self.booleanop("&", node, *args)
  281. def visitBitor(self, node, *args):
  282. self.booleanop("|", node, *args)
  283. def visitBitxor(self, node, *args):
  284. self.booleanop("^", node, *args)
  285. def visitAdd(self, node, *args):
  286. self.operator("+", node, *args)
  287. def visitGetattr(self, node, *args):
  288. self.visit(node.expr, *args)
  289. self.buf.write(".%s" % node.attrname)
  290. def visitSub(self, node, *args):
  291. self.operator("-", node, *args)
  292. def visitNot(self, node, *args):
  293. self.buf.write("not ")
  294. self.visit(node.expr)
  295. def visitDiv(self, node, *args):
  296. self.operator("/", node, *args)
  297. def visitFloorDiv(self, node, *args):
  298. self.operator("//", node, *args)
  299. def visitSubscript(self, node, *args):
  300. self.visit(node.expr)
  301. self.buf.write("[")
  302. [self.visit(x) for x in node.subs]
  303. self.buf.write("]")
  304. def visitUnarySub(self, node, *args):
  305. self.buf.write("-")
  306. self.visit(node.expr)
  307. def visitUnaryAdd(self, node, *args):
  308. self.buf.write("-")
  309. self.visit(node.expr)
  310. def visitSlice(self, node, *args):
  311. self.visit(node.expr)
  312. self.buf.write("[")
  313. if node.lower is not None:
  314. self.visit(node.lower)
  315. self.buf.write(":")
  316. if node.upper is not None:
  317. self.visit(node.upper)
  318. self.buf.write("]")
  319. def visitDict(self, node):
  320. self.buf.write("{")
  321. c = node.getChildren()
  322. for i in range(0, len(c), 2):
  323. self.visit(c[i])
  324. self.buf.write(": ")
  325. self.visit(c[i+1])
  326. if i<len(c) -2:
  327. self.buf.write(", ")
  328. self.buf.write("}")
  329. def visitTuple(self, node):
  330. self.buf.write("(")
  331. c = node.getChildren()
  332. for i in range(0, len(c)):
  333. self.visit(c[i])
  334. if i<len(c) - 1:
  335. self.buf.write(", ")
  336. self.buf.write(")")
  337. def visitList(self, node):
  338. self.buf.write("[")
  339. c = node.getChildren()
  340. for i in range(0, len(c)):
  341. self.visit(c[i])
  342. if i<len(c) - 1:
  343. self.buf.write(", ")
  344. self.buf.write("]")
  345. def visitListComp(self, node):
  346. self.buf.write("[")
  347. self.visit(node.expr)
  348. self.buf.write(" ")
  349. for n in node.quals:
  350. self.visit(n)
  351. self.buf.write("]")
  352. def visitListCompFor(self, node):
  353. self.buf.write(" for ")
  354. self.visit(node.assign)
  355. self.buf.write(" in ")
  356. self.visit(node.list)
  357. for n in node.ifs:
  358. self.visit(n)
  359. def visitListCompIf(self, node):
  360. self.buf.write(" if ")
  361. self.visit(node.test)
  362. def visitCompare(self, node):
  363. self.visit(node.expr)
  364. for tup in node.ops:
  365. self.buf.write(tup[0])
  366. self.visit(tup[1])
  367. def visitCallFunc(self, node, *args):
  368. self.visit(node.node)
  369. self.buf.write("(")
  370. if len(node.args):
  371. self.visit(node.args[0])
  372. for a in node.args[1:]:
  373. self.buf.write(", ")
  374. self.visit(a)
  375. self.buf.write(")")
  376. class walker(visitor.ASTVisitor):
  377. def dispatch(self, node, *args):
  378. print "Node:", str(node)
  379. #print "dir:", dir(node)
  380. return visitor.ASTVisitor.dispatch(self, node, *args)