testyacc.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. # testyacc.py
  2. import unittest
  3. try:
  4. import StringIO
  5. except ImportError:
  6. import io as StringIO
  7. import sys
  8. import os
  9. import warnings
  10. import re
  11. import platform
  12. sys.path.insert(0,"..")
  13. sys.tracebacklimit = 0
  14. import ply.yacc
  15. def make_pymodule_path(filename):
  16. path = os.path.dirname(filename)
  17. file = os.path.basename(filename)
  18. mod, ext = os.path.splitext(file)
  19. if sys.hexversion >= 0x3040000:
  20. import importlib.util
  21. fullpath = importlib.util.cache_from_source(filename, ext=='.pyc')
  22. elif sys.hexversion >= 0x3020000:
  23. import imp
  24. modname = mod+"."+imp.get_tag()+ext
  25. fullpath = os.path.join(path,'__pycache__',modname)
  26. else:
  27. fullpath = filename
  28. return fullpath
  29. def pymodule_out_exists(filename):
  30. return os.path.exists(make_pymodule_path(filename))
  31. def pymodule_out_remove(filename):
  32. os.remove(make_pymodule_path(filename))
  33. def implementation():
  34. if platform.system().startswith("Java"):
  35. return "Jython"
  36. elif hasattr(sys, "pypy_version_info"):
  37. return "PyPy"
  38. else:
  39. return "CPython"
  40. # Check the output to see if it contains all of a set of expected output lines.
  41. # This alternate implementation looks weird, but is needed to properly handle
  42. # some variations in error message order that occurs due to dict hash table
  43. # randomization that was introduced in Python 3.3
  44. def check_expected(result, expected):
  45. # Normalize 'state n' text to account for randomization effects in Python 3.3
  46. expected = re.sub(r' state \d+', 'state <n>', expected)
  47. result = re.sub(r' state \d+', 'state <n>', result)
  48. resultlines = set()
  49. for line in result.splitlines():
  50. if line.startswith("WARNING: "):
  51. line = line[9:]
  52. elif line.startswith("ERROR: "):
  53. line = line[7:]
  54. resultlines.add(line)
  55. # Selectively remove expected lines from the output
  56. for eline in expected.splitlines():
  57. resultlines = set(line for line in resultlines if not line.endswith(eline))
  58. # Return True if no result lines remain
  59. return not bool(resultlines)
  60. def run_import(module):
  61. code = "import "+module
  62. exec(code)
  63. del sys.modules[module]
  64. # Tests related to errors and warnings when building parsers
  65. class YaccErrorWarningTests(unittest.TestCase):
  66. def setUp(self):
  67. sys.stderr = StringIO.StringIO()
  68. sys.stdout = StringIO.StringIO()
  69. try:
  70. os.remove("parsetab.py")
  71. pymodule_out_remove("parsetab.pyc")
  72. except OSError:
  73. pass
  74. if sys.hexversion >= 0x3020000:
  75. warnings.filterwarnings('ignore', category=ResourceWarning)
  76. warnings.filterwarnings('ignore', category=DeprecationWarning)
  77. def tearDown(self):
  78. sys.stderr = sys.__stderr__
  79. sys.stdout = sys.__stdout__
  80. def test_yacc_badargs(self):
  81. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badargs")
  82. result = sys.stderr.getvalue()
  83. self.assert_(check_expected(result,
  84. "yacc_badargs.py:23: Rule 'p_statement_assign' has too many arguments\n"
  85. "yacc_badargs.py:27: Rule 'p_statement_expr' requires an argument\n"
  86. ))
  87. def test_yacc_badid(self):
  88. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badid")
  89. result = sys.stderr.getvalue()
  90. self.assert_(check_expected(result,
  91. "yacc_badid.py:32: Illegal name 'bad&rule' in rule 'statement'\n"
  92. "yacc_badid.py:36: Illegal rule name 'bad&rule'\n"
  93. ))
  94. def test_yacc_badprec(self):
  95. try:
  96. run_import("yacc_badprec")
  97. except ply.yacc.YaccError:
  98. result = sys.stderr.getvalue()
  99. self.assert_(check_expected(result,
  100. "precedence must be a list or tuple\n"
  101. ))
  102. def test_yacc_badprec2(self):
  103. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badprec2")
  104. result = sys.stderr.getvalue()
  105. self.assert_(check_expected(result,
  106. "Bad precedence table\n"
  107. ))
  108. def test_yacc_badprec3(self):
  109. run_import("yacc_badprec3")
  110. result = sys.stderr.getvalue()
  111. self.assert_(check_expected(result,
  112. "Precedence already specified for terminal 'MINUS'\n"
  113. "Generating LALR tables\n"
  114. ))
  115. def test_yacc_badrule(self):
  116. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badrule")
  117. result = sys.stderr.getvalue()
  118. self.assert_(check_expected(result,
  119. "yacc_badrule.py:24: Syntax error. Expected ':'\n"
  120. "yacc_badrule.py:28: Syntax error in rule 'statement'\n"
  121. "yacc_badrule.py:33: Syntax error. Expected ':'\n"
  122. "yacc_badrule.py:42: Syntax error. Expected ':'\n"
  123. ))
  124. def test_yacc_badtok(self):
  125. try:
  126. run_import("yacc_badtok")
  127. except ply.yacc.YaccError:
  128. result = sys.stderr.getvalue()
  129. self.assert_(check_expected(result,
  130. "tokens must be a list or tuple\n"))
  131. def test_yacc_dup(self):
  132. run_import("yacc_dup")
  133. result = sys.stderr.getvalue()
  134. self.assert_(check_expected(result,
  135. "yacc_dup.py:27: Function p_statement redefined. Previously defined on line 23\n"
  136. "Token 'EQUALS' defined, but not used\n"
  137. "There is 1 unused token\n"
  138. "Generating LALR tables\n"
  139. ))
  140. def test_yacc_error1(self):
  141. try:
  142. run_import("yacc_error1")
  143. except ply.yacc.YaccError:
  144. result = sys.stderr.getvalue()
  145. self.assert_(check_expected(result,
  146. "yacc_error1.py:61: p_error() requires 1 argument\n"))
  147. def test_yacc_error2(self):
  148. try:
  149. run_import("yacc_error2")
  150. except ply.yacc.YaccError:
  151. result = sys.stderr.getvalue()
  152. self.assert_(check_expected(result,
  153. "yacc_error2.py:61: p_error() requires 1 argument\n"))
  154. def test_yacc_error3(self):
  155. try:
  156. run_import("yacc_error3")
  157. except ply.yacc.YaccError:
  158. e = sys.exc_info()[1]
  159. result = sys.stderr.getvalue()
  160. self.assert_(check_expected(result,
  161. "'p_error' defined, but is not a function or method\n"))
  162. def test_yacc_error4(self):
  163. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_error4")
  164. result = sys.stderr.getvalue()
  165. self.assert_(check_expected(result,
  166. "yacc_error4.py:62: Illegal rule name 'error'. Already defined as a token\n"
  167. ))
  168. def test_yacc_error5(self):
  169. run_import("yacc_error5")
  170. result = sys.stdout.getvalue()
  171. self.assert_(check_expected(result,
  172. "Group at 3:10 to 3:12\n"
  173. "Undefined name 'a'\n"
  174. "Syntax error at 'b'\n"
  175. "Syntax error at 4:18 to 4:22\n"
  176. "Assignment Error at 2:5 to 5:27\n"
  177. "13\n"
  178. ))
  179. def test_yacc_error6(self):
  180. run_import("yacc_error6")
  181. result = sys.stdout.getvalue()
  182. self.assert_(check_expected(result,
  183. "a=7\n"
  184. "Line 3: Syntax error at '*'\n"
  185. "c=21\n"
  186. ))
  187. def test_yacc_error7(self):
  188. run_import("yacc_error7")
  189. result = sys.stdout.getvalue()
  190. self.assert_(check_expected(result,
  191. "a=7\n"
  192. "Line 3: Syntax error at '*'\n"
  193. "c=21\n"
  194. ))
  195. def test_yacc_inf(self):
  196. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_inf")
  197. result = sys.stderr.getvalue()
  198. self.assert_(check_expected(result,
  199. "Token 'NUMBER' defined, but not used\n"
  200. "There is 1 unused token\n"
  201. "Infinite recursion detected for symbol 'statement'\n"
  202. "Infinite recursion detected for symbol 'expression'\n"
  203. ))
  204. def test_yacc_literal(self):
  205. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_literal")
  206. result = sys.stderr.getvalue()
  207. self.assert_(check_expected(result,
  208. "yacc_literal.py:36: Literal token '**' in rule 'expression' may only be a single character\n"
  209. ))
  210. def test_yacc_misplaced(self):
  211. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_misplaced")
  212. result = sys.stderr.getvalue()
  213. self.assert_(check_expected(result,
  214. "yacc_misplaced.py:32: Misplaced '|'\n"
  215. ))
  216. def test_yacc_missing1(self):
  217. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_missing1")
  218. result = sys.stderr.getvalue()
  219. self.assert_(check_expected(result,
  220. "yacc_missing1.py:24: Symbol 'location' used, but not defined as a token or a rule\n"
  221. ))
  222. def test_yacc_nested(self):
  223. run_import("yacc_nested")
  224. result = sys.stdout.getvalue()
  225. self.assert_(check_expected(result,
  226. "A\n"
  227. "A\n"
  228. "A\n",
  229. ))
  230. def test_yacc_nodoc(self):
  231. run_import("yacc_nodoc")
  232. result = sys.stderr.getvalue()
  233. self.assert_(check_expected(result,
  234. "yacc_nodoc.py:27: No documentation string specified in function 'p_statement_expr' (ignored)\n"
  235. "Generating LALR tables\n"
  236. ))
  237. def test_yacc_noerror(self):
  238. run_import("yacc_noerror")
  239. result = sys.stderr.getvalue()
  240. self.assert_(check_expected(result,
  241. "no p_error() function is defined\n"
  242. "Generating LALR tables\n"
  243. ))
  244. def test_yacc_nop(self):
  245. run_import("yacc_nop")
  246. result = sys.stderr.getvalue()
  247. self.assert_(check_expected(result,
  248. "yacc_nop.py:27: Possible grammar rule 'statement_expr' defined without p_ prefix\n"
  249. "Generating LALR tables\n"
  250. ))
  251. def test_yacc_notfunc(self):
  252. run_import("yacc_notfunc")
  253. result = sys.stderr.getvalue()
  254. self.assert_(check_expected(result,
  255. "'p_statement_assign' not defined as a function\n"
  256. "Token 'EQUALS' defined, but not used\n"
  257. "There is 1 unused token\n"
  258. "Generating LALR tables\n"
  259. ))
  260. def test_yacc_notok(self):
  261. try:
  262. run_import("yacc_notok")
  263. except ply.yacc.YaccError:
  264. result = sys.stderr.getvalue()
  265. self.assert_(check_expected(result,
  266. "No token list is defined\n"))
  267. def test_yacc_rr(self):
  268. run_import("yacc_rr")
  269. result = sys.stderr.getvalue()
  270. self.assert_(check_expected(result,
  271. "Generating LALR tables\n"
  272. "1 reduce/reduce conflict\n"
  273. "reduce/reduce conflict in state 15 resolved using rule (statement -> NAME EQUALS NUMBER)\n"
  274. "rejected rule (expression -> NUMBER) in state 15\n"
  275. ))
  276. def test_yacc_rr_unused(self):
  277. run_import("yacc_rr_unused")
  278. result = sys.stderr.getvalue()
  279. self.assert_(check_expected(result,
  280. "no p_error() function is defined\n"
  281. "Generating LALR tables\n"
  282. "3 reduce/reduce conflicts\n"
  283. "reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\n"
  284. "rejected rule (rule4 -> A) in state 1\n"
  285. "reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\n"
  286. "rejected rule (rule5 -> A) in state 1\n"
  287. "reduce/reduce conflict in state 1 resolved using rule (rule4 -> A)\n"
  288. "rejected rule (rule5 -> A) in state 1\n"
  289. "Rule (rule5 -> A) is never reduced\n"
  290. ))
  291. def test_yacc_simple(self):
  292. run_import("yacc_simple")
  293. result = sys.stderr.getvalue()
  294. self.assert_(check_expected(result,
  295. "Generating LALR tables\n"
  296. ))
  297. def test_yacc_sr(self):
  298. run_import("yacc_sr")
  299. result = sys.stderr.getvalue()
  300. self.assert_(check_expected(result,
  301. "Generating LALR tables\n"
  302. "20 shift/reduce conflicts\n"
  303. ))
  304. def test_yacc_term1(self):
  305. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_term1")
  306. result = sys.stderr.getvalue()
  307. self.assert_(check_expected(result,
  308. "yacc_term1.py:24: Illegal rule name 'NUMBER'. Already defined as a token\n"
  309. ))
  310. def test_yacc_unicode_literals(self):
  311. run_import("yacc_unicode_literals")
  312. result = sys.stderr.getvalue()
  313. self.assert_(check_expected(result,
  314. "Generating LALR tables\n"
  315. ))
  316. def test_yacc_unused(self):
  317. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_unused")
  318. result = sys.stderr.getvalue()
  319. self.assert_(check_expected(result,
  320. "yacc_unused.py:62: Symbol 'COMMA' used, but not defined as a token or a rule\n"
  321. "Symbol 'COMMA' is unreachable\n"
  322. "Symbol 'exprlist' is unreachable\n"
  323. ))
  324. def test_yacc_unused_rule(self):
  325. run_import("yacc_unused_rule")
  326. result = sys.stderr.getvalue()
  327. self.assert_(check_expected(result,
  328. "yacc_unused_rule.py:62: Rule 'integer' defined, but not used\n"
  329. "There is 1 unused rule\n"
  330. "Symbol 'integer' is unreachable\n"
  331. "Generating LALR tables\n"
  332. ))
  333. def test_yacc_uprec(self):
  334. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_uprec")
  335. result = sys.stderr.getvalue()
  336. self.assert_(check_expected(result,
  337. "yacc_uprec.py:37: Nothing known about the precedence of 'UMINUS'\n"
  338. ))
  339. def test_yacc_uprec2(self):
  340. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_uprec2")
  341. result = sys.stderr.getvalue()
  342. self.assert_(check_expected(result,
  343. "yacc_uprec2.py:37: Syntax error. Nothing follows %prec\n"
  344. ))
  345. def test_yacc_prec1(self):
  346. self.assertRaises(ply.yacc.YaccError,run_import,"yacc_prec1")
  347. result = sys.stderr.getvalue()
  348. self.assert_(check_expected(result,
  349. "Precedence rule 'left' defined for unknown symbol '+'\n"
  350. "Precedence rule 'left' defined for unknown symbol '*'\n"
  351. "Precedence rule 'left' defined for unknown symbol '-'\n"
  352. "Precedence rule 'left' defined for unknown symbol '/'\n"
  353. ))
  354. def test_pkg_test1(self):
  355. from pkg_test1 import parser
  356. self.assertTrue(os.path.exists('pkg_test1/parsing/parsetab.py'))
  357. self.assertTrue(os.path.exists('pkg_test1/parsing/lextab.py'))
  358. self.assertTrue(os.path.exists('pkg_test1/parsing/parser.out'))
  359. r = parser.parse('3+4+5')
  360. self.assertEqual(r, 12)
  361. def test_pkg_test2(self):
  362. from pkg_test2 import parser
  363. self.assertTrue(os.path.exists('pkg_test2/parsing/calcparsetab.py'))
  364. self.assertTrue(os.path.exists('pkg_test2/parsing/calclextab.py'))
  365. self.assertTrue(os.path.exists('pkg_test2/parsing/parser.out'))
  366. r = parser.parse('3+4+5')
  367. self.assertEqual(r, 12)
  368. def test_pkg_test3(self):
  369. from pkg_test3 import parser
  370. self.assertTrue(os.path.exists('pkg_test3/generated/parsetab.py'))
  371. self.assertTrue(os.path.exists('pkg_test3/generated/lextab.py'))
  372. self.assertTrue(os.path.exists('pkg_test3/generated/parser.out'))
  373. r = parser.parse('3+4+5')
  374. self.assertEqual(r, 12)
  375. def test_pkg_test4(self):
  376. from pkg_test4 import parser
  377. self.assertFalse(os.path.exists('pkg_test4/parsing/parsetab.py'))
  378. self.assertFalse(os.path.exists('pkg_test4/parsing/lextab.py'))
  379. self.assertFalse(os.path.exists('pkg_test4/parsing/parser.out'))
  380. r = parser.parse('3+4+5')
  381. self.assertEqual(r, 12)
  382. def test_pkg_test5(self):
  383. from pkg_test5 import parser
  384. self.assertTrue(os.path.exists('pkg_test5/parsing/parsetab.py'))
  385. self.assertTrue(os.path.exists('pkg_test5/parsing/lextab.py'))
  386. self.assertTrue(os.path.exists('pkg_test5/parsing/parser.out'))
  387. r = parser.parse('3+4+5')
  388. self.assertEqual(r, 12)
  389. def test_pkg_test6(self):
  390. from pkg_test6 import parser
  391. self.assertTrue(os.path.exists('pkg_test6/parsing/parsetab.py'))
  392. self.assertTrue(os.path.exists('pkg_test6/parsing/lextab.py'))
  393. self.assertTrue(os.path.exists('pkg_test6/parsing/parser.out'))
  394. r = parser.parse('3+4+5')
  395. self.assertEqual(r, 12)
  396. unittest.main()