| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660 |
- # testlex.py
- import unittest
- try:
- import StringIO
- except ImportError:
- import io as StringIO
- import sys
- import os
- import warnings
- import platform
- sys.path.insert(0,"..")
- sys.tracebacklimit = 0
- import ply.lex
- try:
- from importlib.util import cache_from_source
- except ImportError:
- # Python 2.7, but we don't care.
- cache_from_source = None
- def make_pymodule_path(filename, optimization=None):
- path = os.path.dirname(filename)
- file = os.path.basename(filename)
- mod, ext = os.path.splitext(file)
- if sys.hexversion >= 0x3050000:
- fullpath = cache_from_source(filename, optimization=optimization)
- elif sys.hexversion >= 0x3040000:
- fullpath = cache_from_source(filename, ext=='.pyc')
- elif sys.hexversion >= 0x3020000:
- import imp
- modname = mod+"."+imp.get_tag()+ext
- fullpath = os.path.join(path,'__pycache__',modname)
- else:
- fullpath = filename
- return fullpath
- def pymodule_out_exists(filename, optimization=None):
- return os.path.exists(make_pymodule_path(filename,
- optimization=optimization))
- def pymodule_out_remove(filename, optimization=None):
- os.remove(make_pymodule_path(filename, optimization=optimization))
- def implementation():
- if platform.system().startswith("Java"):
- return "Jython"
- elif hasattr(sys, "pypy_version_info"):
- return "PyPy"
- else:
- return "CPython"
- test_pyo = (implementation() == 'CPython')
- def check_expected(result, expected, contains=False):
- if sys.version_info[0] >= 3:
- if isinstance(result,str):
- result = result.encode('ascii')
- if isinstance(expected,str):
- expected = expected.encode('ascii')
- resultlines = result.splitlines()
- expectedlines = expected.splitlines()
- if len(resultlines) != len(expectedlines):
- return False
- for rline,eline in zip(resultlines,expectedlines):
- if contains:
- if eline not in rline:
- return False
- else:
- if not rline.endswith(eline):
- return False
- return True
- def run_import(module):
- code = "import "+module
- exec(code)
- del sys.modules[module]
-
- # Tests related to errors and warnings when building lexers
- class LexErrorWarningTests(unittest.TestCase):
- def setUp(self):
- sys.stderr = StringIO.StringIO()
- sys.stdout = StringIO.StringIO()
- if sys.hexversion >= 0x3020000:
- warnings.filterwarnings('ignore',category=ResourceWarning)
- def tearDown(self):
- sys.stderr = sys.__stderr__
- sys.stdout = sys.__stdout__
- def test_lex_doc1(self):
- self.assertRaises(SyntaxError,run_import,"lex_doc1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
- def test_lex_dup1(self):
- self.assertRaises(SyntaxError,run_import,"lex_dup1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
-
- def test_lex_dup2(self):
- self.assertRaises(SyntaxError,run_import,"lex_dup2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
-
- def test_lex_dup3(self):
- self.assertRaises(SyntaxError,run_import,"lex_dup3")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
- def test_lex_empty(self):
- self.assertRaises(SyntaxError,run_import,"lex_empty")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "No rules of the form t_rulename are defined\n"
- "No rules defined for state 'INITIAL'\n"))
- def test_lex_error1(self):
- run_import("lex_error1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "No t_error rule is defined\n"))
- def test_lex_error2(self):
- self.assertRaises(SyntaxError,run_import,"lex_error2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Rule 't_error' must be defined as a function\n")
- )
- def test_lex_error3(self):
- self.assertRaises(SyntaxError,run_import,"lex_error3")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_error3.py:20: Rule 't_error' requires an argument\n"))
- def test_lex_error4(self):
- self.assertRaises(SyntaxError,run_import,"lex_error4")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
- def test_lex_ignore(self):
- self.assertRaises(SyntaxError,run_import,"lex_ignore")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
- def test_lex_ignore2(self):
- run_import("lex_ignore2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "t_ignore contains a literal backslash '\\'\n"))
- def test_lex_re1(self):
- self.assertRaises(SyntaxError,run_import,"lex_re1")
- result = sys.stderr.getvalue()
- if sys.hexversion < 0x3050000:
- msg = "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"
- else:
- msg = "Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0"
- self.assert_(check_expected(result,
- msg,
- contains=True))
- def test_lex_re2(self):
- self.assertRaises(SyntaxError,run_import,"lex_re2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Regular expression for rule 't_PLUS' matches empty string\n"))
- def test_lex_re3(self):
- self.assertRaises(SyntaxError,run_import,"lex_re3")
- result = sys.stderr.getvalue()
- # self.assert_(check_expected(result,
- # "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
- # "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
- if sys.hexversion < 0x3050000:
- msg = ("Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
- "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")
- else:
- msg = ("Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\n"
- "ERROR: Make sure '#' in rule 't_POUND' is escaped with '\#'")
- self.assert_(check_expected(result,
- msg,
- contains=True), result)
- def test_lex_rule1(self):
- self.assertRaises(SyntaxError,run_import,"lex_rule1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "t_NUMBER not defined as a function or string\n"))
- def test_lex_rule2(self):
- self.assertRaises(SyntaxError,run_import,"lex_rule2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
- def test_lex_rule3(self):
- self.assertRaises(SyntaxError,run_import,"lex_rule3")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
- def test_lex_state1(self):
- self.assertRaises(SyntaxError,run_import,"lex_state1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "states must be defined as a tuple or list\n"))
- def test_lex_state2(self):
- self.assertRaises(SyntaxError,run_import,"lex_state2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
- "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
- def test_lex_state3(self):
- self.assertRaises(SyntaxError,run_import,"lex_state3")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "State name 1 must be a string\n"
- "No rules defined for state 'example'\n"))
- def test_lex_state4(self):
- self.assertRaises(SyntaxError,run_import,"lex_state4")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "State type for state comment must be 'inclusive' or 'exclusive'\n"))
- def test_lex_state5(self):
- self.assertRaises(SyntaxError,run_import,"lex_state5")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "State 'comment' already defined\n"))
- def test_lex_state_noerror(self):
- run_import("lex_state_noerror")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "No error rule is defined for exclusive state 'comment'\n"))
- def test_lex_state_norule(self):
- self.assertRaises(SyntaxError,run_import,"lex_state_norule")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "No rules defined for state 'example'\n"))
- def test_lex_token1(self):
- self.assertRaises(SyntaxError,run_import,"lex_token1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "No token list is defined\n"
- "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
- "Rule 't_PLUS' defined for an unspecified token PLUS\n"
- "Rule 't_MINUS' defined for an unspecified token MINUS\n"
- ))
- def test_lex_token2(self):
- self.assertRaises(SyntaxError,run_import,"lex_token2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "tokens must be a list or tuple\n"
- "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
- "Rule 't_PLUS' defined for an unspecified token PLUS\n"
- "Rule 't_MINUS' defined for an unspecified token MINUS\n"
- ))
-
- def test_lex_token3(self):
- self.assertRaises(SyntaxError,run_import,"lex_token3")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
- def test_lex_token4(self):
- self.assertRaises(SyntaxError,run_import,"lex_token4")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Bad token name '-'\n"))
- def test_lex_token5(self):
- try:
- run_import("lex_token5")
- except ply.lex.LexError:
- e = sys.exc_info()[1]
- self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
- def test_lex_token_dup(self):
- run_import("lex_token_dup")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Token 'MINUS' multiply defined\n"))
-
- def test_lex_literal1(self):
- self.assertRaises(SyntaxError,run_import,"lex_literal1")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Invalid literal '**'. Must be a single character\n"))
- def test_lex_literal2(self):
- self.assertRaises(SyntaxError,run_import,"lex_literal2")
- result = sys.stderr.getvalue()
- self.assert_(check_expected(result,
- "Invalid literals specification. literals must be a sequence of characters\n"))
- import os
- import subprocess
- import shutil
- # Tests related to various build options associated with lexers
- class LexBuildOptionTests(unittest.TestCase):
- def setUp(self):
- sys.stderr = StringIO.StringIO()
- sys.stdout = StringIO.StringIO()
- def tearDown(self):
- sys.stderr = sys.__stderr__
- sys.stdout = sys.__stdout__
- try:
- shutil.rmtree("lexdir")
- except OSError:
- pass
- def test_lex_module(self):
- run_import("lex_module")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
-
- def test_lex_object(self):
- run_import("lex_object")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- def test_lex_closure(self):
- run_import("lex_closure")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- def test_lex_optimize(self):
- try:
- os.remove("lextab.py")
- except OSError:
- pass
- try:
- os.remove("lextab.pyc")
- except OSError:
- pass
- try:
- os.remove("lextab.pyo")
- except OSError:
- pass
- run_import("lex_optimize")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- self.assert_(os.path.exists("lextab.py"))
- p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("lextab.pyo", 1))
- pymodule_out_remove("lextab.pyo", 1)
- p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("lextab.pyo", 2))
- try:
- os.remove("lextab.py")
- except OSError:
- pass
- try:
- pymodule_out_remove("lextab.pyc")
- except OSError:
- pass
- try:
- pymodule_out_remove("lextab.pyo", 2)
- except OSError:
- pass
- def test_lex_optimize2(self):
- try:
- os.remove("opt2tab.py")
- except OSError:
- pass
- try:
- os.remove("opt2tab.pyc")
- except OSError:
- pass
- try:
- os.remove("opt2tab.pyo")
- except OSError:
- pass
- run_import("lex_optimize2")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- self.assert_(os.path.exists("opt2tab.py"))
- p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("opt2tab.pyo", 1))
- pymodule_out_remove("opt2tab.pyo", 1)
- p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("opt2tab.pyo", 2))
- try:
- os.remove("opt2tab.py")
- except OSError:
- pass
- try:
- pymodule_out_remove("opt2tab.pyc")
- except OSError:
- pass
- try:
- pymodule_out_remove("opt2tab.pyo", 2)
- except OSError:
- pass
- def test_lex_optimize3(self):
- try:
- shutil.rmtree("lexdir")
- except OSError:
- pass
-
- os.mkdir("lexdir")
- os.mkdir("lexdir/sub")
- open("lexdir/__init__.py","w").write("")
- open("lexdir/sub/__init__.py","w").write("")
- run_import("lex_optimize3")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- self.assert_(os.path.exists("lexdir/sub/calctab.py"))
- p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 1))
- pymodule_out_remove("lexdir/sub/calctab.pyo", 1)
- p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(PLUS,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 2))
- try:
- shutil.rmtree("lexdir")
- except OSError:
- pass
- def test_lex_opt_alias(self):
- try:
- os.remove("aliastab.py")
- except OSError:
- pass
- try:
- os.remove("aliastab.pyc")
- except OSError:
- pass
- try:
- os.remove("aliastab.pyo")
- except OSError:
- pass
- run_import("lex_opt_alias")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(+,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- self.assert_(os.path.exists("aliastab.py"))
- p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(+,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("aliastab.pyo", 1))
- pymodule_out_remove("aliastab.pyo", 1)
- p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(NUMBER,3,1,0)\n"
- "(+,'+',1,1)\n"
- "(NUMBER,4,1,2)\n"))
- if test_pyo:
- self.assert_(pymodule_out_exists("aliastab.pyo", 2))
- try:
- os.remove("aliastab.py")
- except OSError:
- pass
- try:
- pymodule_out_remove("aliastab.pyc")
- except OSError:
- pass
- try:
- pymodule_out_remove("aliastab.pyo", 2)
- except OSError:
- pass
- def test_lex_many_tokens(self):
- try:
- os.remove("manytab.py")
- except OSError:
- pass
- try:
- os.remove("manytab.pyc")
- except OSError:
- pass
- try:
- os.remove("manytab.pyo")
- except OSError:
- pass
- run_import("lex_many_tokens")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(TOK34,'TOK34:',1,0)\n"
- "(TOK143,'TOK143:',1,7)\n"
- "(TOK269,'TOK269:',1,15)\n"
- "(TOK372,'TOK372:',1,23)\n"
- "(TOK452,'TOK452:',1,31)\n"
- "(TOK561,'TOK561:',1,39)\n"
- "(TOK999,'TOK999:',1,47)\n"
- ))
- self.assert_(os.path.exists("manytab.py"))
- if implementation() == 'CPython':
- p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
- stdout=subprocess.PIPE)
- result = p.stdout.read()
- self.assert_(check_expected(result,
- "(TOK34,'TOK34:',1,0)\n"
- "(TOK143,'TOK143:',1,7)\n"
- "(TOK269,'TOK269:',1,15)\n"
- "(TOK372,'TOK372:',1,23)\n"
- "(TOK452,'TOK452:',1,31)\n"
- "(TOK561,'TOK561:',1,39)\n"
- "(TOK999,'TOK999:',1,47)\n"
- ))
- self.assert_(pymodule_out_exists("manytab.pyo", 1))
- pymodule_out_remove("manytab.pyo", 1)
- try:
- os.remove("manytab.py")
- except OSError:
- pass
- try:
- os.remove("manytab.pyc")
- except OSError:
- pass
- try:
- os.remove("manytab.pyo")
- except OSError:
- pass
-
- # Tests related to run-time behavior of lexers
- class LexRunTests(unittest.TestCase):
- def setUp(self):
- sys.stderr = StringIO.StringIO()
- sys.stdout = StringIO.StringIO()
- def tearDown(self):
- sys.stderr = sys.__stderr__
- sys.stdout = sys.__stdout__
- def test_lex_hedit(self):
- run_import("lex_hedit")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
- "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
- "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
-
- def test_lex_state_try(self):
- run_import("lex_state_try")
- result = sys.stdout.getvalue()
- self.assert_(check_expected(result,
- "(NUMBER,'3',1,0)\n"
- "(PLUS,'+',1,2)\n"
- "(NUMBER,'4',1,4)\n"
- "Entering comment state\n"
- "comment body LexToken(body_part,'This is a comment */',1,9)\n"
- "(PLUS,'+',1,30)\n"
- "(NUMBER,'10',1,32)\n"
- ))
- unittest.main()
|