testlex.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680
  1. # testlex.py
  2. import unittest
  3. try:
  4. import StringIO
  5. except ImportError:
  6. import io as StringIO
  7. import sys
  8. import os
  9. import warnings
  10. import platform
  11. sys.path.insert(0,"..")
  12. sys.tracebacklimit = 0
  13. import ply.lex
  14. try:
  15. from importlib.util import cache_from_source
  16. except ImportError:
  17. # Python 2.7, but we don't care.
  18. cache_from_source = None
  19. def make_pymodule_path(filename, optimization=None):
  20. path = os.path.dirname(filename)
  21. file = os.path.basename(filename)
  22. mod, ext = os.path.splitext(file)
  23. if sys.hexversion >= 0x3050000:
  24. fullpath = cache_from_source(filename, optimization=optimization)
  25. elif sys.hexversion >= 0x3040000:
  26. fullpath = cache_from_source(filename, ext=='.pyc')
  27. elif sys.hexversion >= 0x3020000:
  28. import imp
  29. modname = mod+"."+imp.get_tag()+ext
  30. fullpath = os.path.join(path,'__pycache__',modname)
  31. else:
  32. fullpath = filename
  33. return fullpath
  34. def pymodule_out_exists(filename, optimization=None):
  35. return os.path.exists(make_pymodule_path(filename,
  36. optimization=optimization))
  37. def pymodule_out_remove(filename, optimization=None):
  38. os.remove(make_pymodule_path(filename, optimization=optimization))
  39. def implementation():
  40. if platform.system().startswith("Java"):
  41. return "Jython"
  42. elif hasattr(sys, "pypy_version_info"):
  43. return "PyPy"
  44. else:
  45. return "CPython"
  46. test_pyo = (implementation() == 'CPython')
  47. def check_expected(result, expected, contains=False):
  48. if sys.version_info[0] >= 3:
  49. if isinstance(result,str):
  50. result = result.encode('ascii')
  51. if isinstance(expected,str):
  52. expected = expected.encode('ascii')
  53. resultlines = result.splitlines()
  54. expectedlines = expected.splitlines()
  55. if len(resultlines) != len(expectedlines):
  56. return False
  57. for rline,eline in zip(resultlines,expectedlines):
  58. if contains:
  59. if eline not in rline:
  60. return False
  61. else:
  62. if not rline.endswith(eline):
  63. return False
  64. return True
  65. def run_import(module):
  66. code = "import "+module
  67. exec(code)
  68. del sys.modules[module]
  69. # Tests related to errors and warnings when building lexers
  70. class LexErrorWarningTests(unittest.TestCase):
  71. def setUp(self):
  72. sys.stderr = StringIO.StringIO()
  73. sys.stdout = StringIO.StringIO()
  74. if sys.hexversion >= 0x3020000:
  75. warnings.filterwarnings('ignore',category=ResourceWarning)
  76. def tearDown(self):
  77. sys.stderr = sys.__stderr__
  78. sys.stdout = sys.__stdout__
  79. def test_lex_doc1(self):
  80. self.assertRaises(SyntaxError,run_import,"lex_doc1")
  81. result = sys.stderr.getvalue()
  82. self.assert_(check_expected(result,
  83. "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
  84. def test_lex_dup1(self):
  85. self.assertRaises(SyntaxError,run_import,"lex_dup1")
  86. result = sys.stderr.getvalue()
  87. self.assert_(check_expected(result,
  88. "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
  89. def test_lex_dup2(self):
  90. self.assertRaises(SyntaxError,run_import,"lex_dup2")
  91. result = sys.stderr.getvalue()
  92. self.assert_(check_expected(result,
  93. "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
  94. def test_lex_dup3(self):
  95. self.assertRaises(SyntaxError,run_import,"lex_dup3")
  96. result = sys.stderr.getvalue()
  97. self.assert_(check_expected(result,
  98. "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
  99. def test_lex_empty(self):
  100. self.assertRaises(SyntaxError,run_import,"lex_empty")
  101. result = sys.stderr.getvalue()
  102. self.assert_(check_expected(result,
  103. "No rules of the form t_rulename are defined\n"
  104. "No rules defined for state 'INITIAL'\n"))
  105. def test_lex_error1(self):
  106. run_import("lex_error1")
  107. result = sys.stderr.getvalue()
  108. self.assert_(check_expected(result,
  109. "No t_error rule is defined\n"))
  110. def test_lex_error2(self):
  111. self.assertRaises(SyntaxError,run_import,"lex_error2")
  112. result = sys.stderr.getvalue()
  113. self.assert_(check_expected(result,
  114. "Rule 't_error' must be defined as a function\n")
  115. )
  116. def test_lex_error3(self):
  117. self.assertRaises(SyntaxError,run_import,"lex_error3")
  118. result = sys.stderr.getvalue()
  119. self.assert_(check_expected(result,
  120. "lex_error3.py:20: Rule 't_error' requires an argument\n"))
  121. def test_lex_error4(self):
  122. self.assertRaises(SyntaxError,run_import,"lex_error4")
  123. result = sys.stderr.getvalue()
  124. self.assert_(check_expected(result,
  125. "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
  126. def test_lex_ignore(self):
  127. self.assertRaises(SyntaxError,run_import,"lex_ignore")
  128. result = sys.stderr.getvalue()
  129. self.assert_(check_expected(result,
  130. "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
  131. def test_lex_ignore2(self):
  132. run_import("lex_ignore2")
  133. result = sys.stderr.getvalue()
  134. self.assert_(check_expected(result,
  135. "t_ignore contains a literal backslash '\\'\n"))
  136. def test_lex_re1(self):
  137. self.assertRaises(SyntaxError,run_import,"lex_re1")
  138. result = sys.stderr.getvalue()
  139. if sys.hexversion < 0x3050000:
  140. msg = "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"
  141. else:
  142. msg = "Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0"
  143. self.assert_(check_expected(result,
  144. msg,
  145. contains=True))
  146. def test_lex_re2(self):
  147. self.assertRaises(SyntaxError,run_import,"lex_re2")
  148. result = sys.stderr.getvalue()
  149. self.assert_(check_expected(result,
  150. "Regular expression for rule 't_PLUS' matches empty string\n"))
  151. def test_lex_re3(self):
  152. self.assertRaises(SyntaxError,run_import,"lex_re3")
  153. result = sys.stderr.getvalue()
  154. # self.assert_(check_expected(result,
  155. # "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
  156. # "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
  157. if sys.hexversion < 0x3050000:
  158. msg = ("Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
  159. "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")
  160. else:
  161. msg = ("Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\n"
  162. "ERROR: Make sure '#' in rule 't_POUND' is escaped with '\#'")
  163. self.assert_(check_expected(result,
  164. msg,
  165. contains=True), result)
  166. def test_lex_rule1(self):
  167. self.assertRaises(SyntaxError,run_import,"lex_rule1")
  168. result = sys.stderr.getvalue()
  169. self.assert_(check_expected(result,
  170. "t_NUMBER not defined as a function or string\n"))
  171. def test_lex_rule2(self):
  172. self.assertRaises(SyntaxError,run_import,"lex_rule2")
  173. result = sys.stderr.getvalue()
  174. self.assert_(check_expected(result,
  175. "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
  176. def test_lex_rule3(self):
  177. self.assertRaises(SyntaxError,run_import,"lex_rule3")
  178. result = sys.stderr.getvalue()
  179. self.assert_(check_expected(result,
  180. "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
  181. def test_lex_state1(self):
  182. self.assertRaises(SyntaxError,run_import,"lex_state1")
  183. result = sys.stderr.getvalue()
  184. self.assert_(check_expected(result,
  185. "states must be defined as a tuple or list\n"))
  186. def test_lex_state2(self):
  187. self.assertRaises(SyntaxError,run_import,"lex_state2")
  188. result = sys.stderr.getvalue()
  189. self.assert_(check_expected(result,
  190. "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
  191. "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
  192. def test_lex_state3(self):
  193. self.assertRaises(SyntaxError,run_import,"lex_state3")
  194. result = sys.stderr.getvalue()
  195. self.assert_(check_expected(result,
  196. "State name 1 must be a string\n"
  197. "No rules defined for state 'example'\n"))
  198. def test_lex_state4(self):
  199. self.assertRaises(SyntaxError,run_import,"lex_state4")
  200. result = sys.stderr.getvalue()
  201. self.assert_(check_expected(result,
  202. "State type for state comment must be 'inclusive' or 'exclusive'\n"))
  203. def test_lex_state5(self):
  204. self.assertRaises(SyntaxError,run_import,"lex_state5")
  205. result = sys.stderr.getvalue()
  206. self.assert_(check_expected(result,
  207. "State 'comment' already defined\n"))
  208. def test_lex_state_noerror(self):
  209. run_import("lex_state_noerror")
  210. result = sys.stderr.getvalue()
  211. self.assert_(check_expected(result,
  212. "No error rule is defined for exclusive state 'comment'\n"))
  213. def test_lex_state_norule(self):
  214. self.assertRaises(SyntaxError,run_import,"lex_state_norule")
  215. result = sys.stderr.getvalue()
  216. self.assert_(check_expected(result,
  217. "No rules defined for state 'example'\n"))
  218. def test_lex_token1(self):
  219. self.assertRaises(SyntaxError,run_import,"lex_token1")
  220. result = sys.stderr.getvalue()
  221. self.assert_(check_expected(result,
  222. "No token list is defined\n"
  223. "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
  224. "Rule 't_PLUS' defined for an unspecified token PLUS\n"
  225. "Rule 't_MINUS' defined for an unspecified token MINUS\n"
  226. ))
  227. def test_lex_token2(self):
  228. self.assertRaises(SyntaxError,run_import,"lex_token2")
  229. result = sys.stderr.getvalue()
  230. self.assert_(check_expected(result,
  231. "tokens must be a list or tuple\n"
  232. "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
  233. "Rule 't_PLUS' defined for an unspecified token PLUS\n"
  234. "Rule 't_MINUS' defined for an unspecified token MINUS\n"
  235. ))
  236. def test_lex_token3(self):
  237. self.assertRaises(SyntaxError,run_import,"lex_token3")
  238. result = sys.stderr.getvalue()
  239. self.assert_(check_expected(result,
  240. "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
  241. def test_lex_token4(self):
  242. self.assertRaises(SyntaxError,run_import,"lex_token4")
  243. result = sys.stderr.getvalue()
  244. self.assert_(check_expected(result,
  245. "Bad token name '-'\n"))
  246. def test_lex_token5(self):
  247. try:
  248. run_import("lex_token5")
  249. except ply.lex.LexError:
  250. e = sys.exc_info()[1]
  251. self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
  252. def test_lex_token_dup(self):
  253. run_import("lex_token_dup")
  254. result = sys.stderr.getvalue()
  255. self.assert_(check_expected(result,
  256. "Token 'MINUS' multiply defined\n"))
  257. def test_lex_literal1(self):
  258. self.assertRaises(SyntaxError,run_import,"lex_literal1")
  259. result = sys.stderr.getvalue()
  260. self.assert_(check_expected(result,
  261. "Invalid literal '**'. Must be a single character\n"))
  262. def test_lex_literal2(self):
  263. self.assertRaises(SyntaxError,run_import,"lex_literal2")
  264. result = sys.stderr.getvalue()
  265. self.assert_(check_expected(result,
  266. "Invalid literals specification. literals must be a sequence of characters\n"))
  267. import os
  268. import subprocess
  269. import shutil
  270. # Tests related to various build options associated with lexers
  271. class LexBuildOptionTests(unittest.TestCase):
  272. def setUp(self):
  273. sys.stderr = StringIO.StringIO()
  274. sys.stdout = StringIO.StringIO()
  275. def tearDown(self):
  276. sys.stderr = sys.__stderr__
  277. sys.stdout = sys.__stdout__
  278. try:
  279. shutil.rmtree("lexdir")
  280. except OSError:
  281. pass
  282. def test_lex_module(self):
  283. run_import("lex_module")
  284. result = sys.stdout.getvalue()
  285. self.assert_(check_expected(result,
  286. "(NUMBER,3,1,0)\n"
  287. "(PLUS,'+',1,1)\n"
  288. "(NUMBER,4,1,2)\n"))
  289. def test_lex_object(self):
  290. run_import("lex_object")
  291. result = sys.stdout.getvalue()
  292. self.assert_(check_expected(result,
  293. "(NUMBER,3,1,0)\n"
  294. "(PLUS,'+',1,1)\n"
  295. "(NUMBER,4,1,2)\n"))
  296. def test_lex_closure(self):
  297. run_import("lex_closure")
  298. result = sys.stdout.getvalue()
  299. self.assert_(check_expected(result,
  300. "(NUMBER,3,1,0)\n"
  301. "(PLUS,'+',1,1)\n"
  302. "(NUMBER,4,1,2)\n"))
  303. def test_lex_optimize(self):
  304. try:
  305. os.remove("lextab.py")
  306. except OSError:
  307. pass
  308. try:
  309. os.remove("lextab.pyc")
  310. except OSError:
  311. pass
  312. try:
  313. os.remove("lextab.pyo")
  314. except OSError:
  315. pass
  316. run_import("lex_optimize")
  317. result = sys.stdout.getvalue()
  318. self.assert_(check_expected(result,
  319. "(NUMBER,3,1,0)\n"
  320. "(PLUS,'+',1,1)\n"
  321. "(NUMBER,4,1,2)\n"))
  322. self.assert_(os.path.exists("lextab.py"))
  323. p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
  324. stdout=subprocess.PIPE)
  325. result = p.stdout.read()
  326. self.assert_(check_expected(result,
  327. "(NUMBER,3,1,0)\n"
  328. "(PLUS,'+',1,1)\n"
  329. "(NUMBER,4,1,2)\n"))
  330. if test_pyo:
  331. self.assert_(pymodule_out_exists("lextab.pyo", 1))
  332. pymodule_out_remove("lextab.pyo", 1)
  333. p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
  334. stdout=subprocess.PIPE)
  335. result = p.stdout.read()
  336. self.assert_(check_expected(result,
  337. "(NUMBER,3,1,0)\n"
  338. "(PLUS,'+',1,1)\n"
  339. "(NUMBER,4,1,2)\n"))
  340. if test_pyo:
  341. self.assert_(pymodule_out_exists("lextab.pyo", 2))
  342. try:
  343. os.remove("lextab.py")
  344. except OSError:
  345. pass
  346. try:
  347. pymodule_out_remove("lextab.pyc")
  348. except OSError:
  349. pass
  350. try:
  351. pymodule_out_remove("lextab.pyo", 2)
  352. except OSError:
  353. pass
  354. def test_lex_optimize2(self):
  355. try:
  356. os.remove("opt2tab.py")
  357. except OSError:
  358. pass
  359. try:
  360. os.remove("opt2tab.pyc")
  361. except OSError:
  362. pass
  363. try:
  364. os.remove("opt2tab.pyo")
  365. except OSError:
  366. pass
  367. run_import("lex_optimize2")
  368. result = sys.stdout.getvalue()
  369. self.assert_(check_expected(result,
  370. "(NUMBER,3,1,0)\n"
  371. "(PLUS,'+',1,1)\n"
  372. "(NUMBER,4,1,2)\n"))
  373. self.assert_(os.path.exists("opt2tab.py"))
  374. p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
  375. stdout=subprocess.PIPE)
  376. result = p.stdout.read()
  377. self.assert_(check_expected(result,
  378. "(NUMBER,3,1,0)\n"
  379. "(PLUS,'+',1,1)\n"
  380. "(NUMBER,4,1,2)\n"))
  381. if test_pyo:
  382. self.assert_(pymodule_out_exists("opt2tab.pyo", 1))
  383. pymodule_out_remove("opt2tab.pyo", 1)
  384. p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
  385. stdout=subprocess.PIPE)
  386. result = p.stdout.read()
  387. self.assert_(check_expected(result,
  388. "(NUMBER,3,1,0)\n"
  389. "(PLUS,'+',1,1)\n"
  390. "(NUMBER,4,1,2)\n"))
  391. if test_pyo:
  392. self.assert_(pymodule_out_exists("opt2tab.pyo", 2))
  393. try:
  394. os.remove("opt2tab.py")
  395. except OSError:
  396. pass
  397. try:
  398. pymodule_out_remove("opt2tab.pyc")
  399. except OSError:
  400. pass
  401. try:
  402. pymodule_out_remove("opt2tab.pyo", 2)
  403. except OSError:
  404. pass
  405. def test_lex_optimize3(self):
  406. try:
  407. shutil.rmtree("lexdir")
  408. except OSError:
  409. pass
  410. os.mkdir("lexdir")
  411. os.mkdir("lexdir/sub")
  412. open("lexdir/__init__.py","w").write("")
  413. open("lexdir/sub/__init__.py","w").write("")
  414. run_import("lex_optimize3")
  415. result = sys.stdout.getvalue()
  416. self.assert_(check_expected(result,
  417. "(NUMBER,3,1,0)\n"
  418. "(PLUS,'+',1,1)\n"
  419. "(NUMBER,4,1,2)\n"))
  420. self.assert_(os.path.exists("lexdir/sub/calctab.py"))
  421. p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
  422. stdout=subprocess.PIPE)
  423. result = p.stdout.read()
  424. self.assert_(check_expected(result,
  425. "(NUMBER,3,1,0)\n"
  426. "(PLUS,'+',1,1)\n"
  427. "(NUMBER,4,1,2)\n"))
  428. if test_pyo:
  429. self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 1))
  430. pymodule_out_remove("lexdir/sub/calctab.pyo", 1)
  431. p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
  432. stdout=subprocess.PIPE)
  433. result = p.stdout.read()
  434. self.assert_(check_expected(result,
  435. "(NUMBER,3,1,0)\n"
  436. "(PLUS,'+',1,1)\n"
  437. "(NUMBER,4,1,2)\n"))
  438. if test_pyo:
  439. self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 2))
  440. try:
  441. shutil.rmtree("lexdir")
  442. except OSError:
  443. pass
  444. def test_lex_optimize4(self):
  445. # Regression test to make sure that reflags works correctly
  446. # on Python 3.
  447. for extension in ['py', 'pyc']:
  448. try:
  449. os.remove("opt4tab.{0}".format(extension))
  450. except OSError:
  451. pass
  452. run_import("lex_optimize4")
  453. run_import("lex_optimize4")
  454. for extension in ['py', 'pyc']:
  455. try:
  456. os.remove("opt4tab.{0}".format(extension))
  457. except OSError:
  458. pass
  459. def test_lex_opt_alias(self):
  460. try:
  461. os.remove("aliastab.py")
  462. except OSError:
  463. pass
  464. try:
  465. os.remove("aliastab.pyc")
  466. except OSError:
  467. pass
  468. try:
  469. os.remove("aliastab.pyo")
  470. except OSError:
  471. pass
  472. run_import("lex_opt_alias")
  473. result = sys.stdout.getvalue()
  474. self.assert_(check_expected(result,
  475. "(NUMBER,3,1,0)\n"
  476. "(+,'+',1,1)\n"
  477. "(NUMBER,4,1,2)\n"))
  478. self.assert_(os.path.exists("aliastab.py"))
  479. p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
  480. stdout=subprocess.PIPE)
  481. result = p.stdout.read()
  482. self.assert_(check_expected(result,
  483. "(NUMBER,3,1,0)\n"
  484. "(+,'+',1,1)\n"
  485. "(NUMBER,4,1,2)\n"))
  486. if test_pyo:
  487. self.assert_(pymodule_out_exists("aliastab.pyo", 1))
  488. pymodule_out_remove("aliastab.pyo", 1)
  489. p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
  490. stdout=subprocess.PIPE)
  491. result = p.stdout.read()
  492. self.assert_(check_expected(result,
  493. "(NUMBER,3,1,0)\n"
  494. "(+,'+',1,1)\n"
  495. "(NUMBER,4,1,2)\n"))
  496. if test_pyo:
  497. self.assert_(pymodule_out_exists("aliastab.pyo", 2))
  498. try:
  499. os.remove("aliastab.py")
  500. except OSError:
  501. pass
  502. try:
  503. pymodule_out_remove("aliastab.pyc")
  504. except OSError:
  505. pass
  506. try:
  507. pymodule_out_remove("aliastab.pyo", 2)
  508. except OSError:
  509. pass
  510. def test_lex_many_tokens(self):
  511. try:
  512. os.remove("manytab.py")
  513. except OSError:
  514. pass
  515. try:
  516. os.remove("manytab.pyc")
  517. except OSError:
  518. pass
  519. try:
  520. os.remove("manytab.pyo")
  521. except OSError:
  522. pass
  523. run_import("lex_many_tokens")
  524. result = sys.stdout.getvalue()
  525. self.assert_(check_expected(result,
  526. "(TOK34,'TOK34:',1,0)\n"
  527. "(TOK143,'TOK143:',1,7)\n"
  528. "(TOK269,'TOK269:',1,15)\n"
  529. "(TOK372,'TOK372:',1,23)\n"
  530. "(TOK452,'TOK452:',1,31)\n"
  531. "(TOK561,'TOK561:',1,39)\n"
  532. "(TOK999,'TOK999:',1,47)\n"
  533. ))
  534. self.assert_(os.path.exists("manytab.py"))
  535. if implementation() == 'CPython':
  536. p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
  537. stdout=subprocess.PIPE)
  538. result = p.stdout.read()
  539. self.assert_(check_expected(result,
  540. "(TOK34,'TOK34:',1,0)\n"
  541. "(TOK143,'TOK143:',1,7)\n"
  542. "(TOK269,'TOK269:',1,15)\n"
  543. "(TOK372,'TOK372:',1,23)\n"
  544. "(TOK452,'TOK452:',1,31)\n"
  545. "(TOK561,'TOK561:',1,39)\n"
  546. "(TOK999,'TOK999:',1,47)\n"
  547. ))
  548. self.assert_(pymodule_out_exists("manytab.pyo", 1))
  549. pymodule_out_remove("manytab.pyo", 1)
  550. try:
  551. os.remove("manytab.py")
  552. except OSError:
  553. pass
  554. try:
  555. os.remove("manytab.pyc")
  556. except OSError:
  557. pass
  558. try:
  559. os.remove("manytab.pyo")
  560. except OSError:
  561. pass
  562. # Tests related to run-time behavior of lexers
  563. class LexRunTests(unittest.TestCase):
  564. def setUp(self):
  565. sys.stderr = StringIO.StringIO()
  566. sys.stdout = StringIO.StringIO()
  567. def tearDown(self):
  568. sys.stderr = sys.__stderr__
  569. sys.stdout = sys.__stdout__
  570. def test_lex_hedit(self):
  571. run_import("lex_hedit")
  572. result = sys.stdout.getvalue()
  573. self.assert_(check_expected(result,
  574. "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
  575. "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
  576. "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
  577. def test_lex_state_try(self):
  578. run_import("lex_state_try")
  579. result = sys.stdout.getvalue()
  580. self.assert_(check_expected(result,
  581. "(NUMBER,'3',1,0)\n"
  582. "(PLUS,'+',1,2)\n"
  583. "(NUMBER,'4',1,4)\n"
  584. "Entering comment state\n"
  585. "comment body LexToken(body_part,'This is a comment */',1,9)\n"
  586. "(PLUS,'+',1,30)\n"
  587. "(NUMBER,'10',1,32)\n"
  588. ))
  589. unittest.main()