1# testlex.py
2
3import unittest
4try:
5    import StringIO
6except ImportError:
7    import io as StringIO
8
9import sys
10import os
11import warnings
12import platform
13
14sys.path.insert(0,"..")
15sys.tracebacklimit = 0
16
17import ply.lex
18
19try:
20    from importlib.util import cache_from_source
21except ImportError:
22    # Python 2.7, but we don't care.
23    cache_from_source = None
24
25
26def make_pymodule_path(filename, optimization=None):
27    path = os.path.dirname(filename)
28    file = os.path.basename(filename)
29    mod, ext = os.path.splitext(file)
30
31    if sys.hexversion >= 0x3050000:
32        fullpath = cache_from_source(filename, optimization=optimization)
33    elif sys.hexversion >= 0x3040000:
34        fullpath = cache_from_source(filename, ext=='.pyc')
35    elif sys.hexversion >= 0x3020000:
36        import imp
37        modname = mod+"."+imp.get_tag()+ext
38        fullpath = os.path.join(path,'__pycache__',modname)
39    else:
40        fullpath = filename
41    return fullpath
42
43def pymodule_out_exists(filename, optimization=None):
44    return os.path.exists(make_pymodule_path(filename,
45                                             optimization=optimization))
46
47def pymodule_out_remove(filename, optimization=None):
48    os.remove(make_pymodule_path(filename, optimization=optimization))
49
50def implementation():
51    if platform.system().startswith("Java"):
52        return "Jython"
53    elif hasattr(sys, "pypy_version_info"):
54        return "PyPy"
55    else:
56        return "CPython"
57
58test_pyo = (implementation() == 'CPython')
59
60def check_expected(result, expected, contains=False):
61    if sys.version_info[0] >= 3:
62        if isinstance(result,str):
63            result = result.encode('ascii')
64        if isinstance(expected,str):
65            expected = expected.encode('ascii')
66    resultlines = result.splitlines()
67    expectedlines = expected.splitlines()
68
69    if len(resultlines) != len(expectedlines):
70        return False
71
72    for rline,eline in zip(resultlines,expectedlines):
73        if contains:
74            if eline not in rline:
75                return False
76        else:
77            if not rline.endswith(eline):
78                return False
79    return True
80
81def run_import(module):
82    code = "import "+module
83    exec(code)
84    del sys.modules[module]
85
86# Tests related to errors and warnings when building lexers
87class LexErrorWarningTests(unittest.TestCase):
88    def setUp(self):
89        sys.stderr = StringIO.StringIO()
90        sys.stdout = StringIO.StringIO()
91        if sys.hexversion >= 0x3020000:
92            warnings.filterwarnings('ignore',category=ResourceWarning)
93
94    def tearDown(self):
95        sys.stderr = sys.__stderr__
96        sys.stdout = sys.__stdout__
97    def test_lex_doc1(self):
98        self.assertRaises(SyntaxError,run_import,"lex_doc1")
99        result = sys.stderr.getvalue()
100        self.assert_(check_expected(result,
101                              "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
102    def test_lex_dup1(self):
103        self.assertRaises(SyntaxError,run_import,"lex_dup1")
104        result = sys.stderr.getvalue()
105        self.assert_(check_expected(result,
106                                    "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
107
108    def test_lex_dup2(self):
109        self.assertRaises(SyntaxError,run_import,"lex_dup2")
110        result = sys.stderr.getvalue()
111        self.assert_(check_expected(result,
112                                    "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
113
114    def test_lex_dup3(self):
115        self.assertRaises(SyntaxError,run_import,"lex_dup3")
116        result = sys.stderr.getvalue()
117        self.assert_(check_expected(result,
118                                    "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
119
120    def test_lex_empty(self):
121        self.assertRaises(SyntaxError,run_import,"lex_empty")
122        result = sys.stderr.getvalue()
123        self.assert_(check_expected(result,
124                                    "No rules of the form t_rulename are defined\n"
125                                    "No rules defined for state 'INITIAL'\n"))
126
127    def test_lex_error1(self):
128        run_import("lex_error1")
129        result = sys.stderr.getvalue()
130        self.assert_(check_expected(result,
131                                    "No t_error rule is defined\n"))
132
133    def test_lex_error2(self):
134        self.assertRaises(SyntaxError,run_import,"lex_error2")
135        result = sys.stderr.getvalue()
136        self.assert_(check_expected(result,
137                                    "Rule 't_error' must be defined as a function\n")
138                     )
139
140    def test_lex_error3(self):
141        self.assertRaises(SyntaxError,run_import,"lex_error3")
142        result = sys.stderr.getvalue()
143        self.assert_(check_expected(result,
144                                    "lex_error3.py:20: Rule 't_error' requires an argument\n"))
145
146    def test_lex_error4(self):
147        self.assertRaises(SyntaxError,run_import,"lex_error4")
148        result = sys.stderr.getvalue()
149        self.assert_(check_expected(result,
150                                    "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
151
152    def test_lex_ignore(self):
153        self.assertRaises(SyntaxError,run_import,"lex_ignore")
154        result = sys.stderr.getvalue()
155        self.assert_(check_expected(result,
156                                    "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
157
158    def test_lex_ignore2(self):
159        run_import("lex_ignore2")
160        result = sys.stderr.getvalue()
161        self.assert_(check_expected(result,
162                                    "t_ignore contains a literal backslash '\\'\n"))
163
164
165    def test_lex_re1(self):
166        self.assertRaises(SyntaxError,run_import,"lex_re1")
167        result = sys.stderr.getvalue()
168        if sys.hexversion < 0x3050000:
169            msg = "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"
170        else:
171            msg = "Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0"
172        self.assert_(check_expected(result,
173                                    msg,
174                                    contains=True))
175
176    def test_lex_re2(self):
177        self.assertRaises(SyntaxError,run_import,"lex_re2")
178        result = sys.stderr.getvalue()
179        self.assert_(check_expected(result,
180                                    "Regular expression for rule 't_PLUS' matches empty string\n"))
181
182    def test_lex_re3(self):
183        self.assertRaises(SyntaxError,run_import,"lex_re3")
184        result = sys.stderr.getvalue()
185#        self.assert_(check_expected(result,
186#                                    "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
187#                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
188
189        if sys.hexversion < 0x3050000:
190            msg = ("Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
191                   "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")
192        else:
193            msg = ("Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\n"
194                   "ERROR: Make sure '#' in rule 't_POUND' is escaped with '\#'")
195        self.assert_(check_expected(result,
196                                    msg,
197                                    contains=True), result)
198
199    def test_lex_rule1(self):
200        self.assertRaises(SyntaxError,run_import,"lex_rule1")
201        result = sys.stderr.getvalue()
202        self.assert_(check_expected(result,
203                                    "t_NUMBER not defined as a function or string\n"))
204
205    def test_lex_rule2(self):
206        self.assertRaises(SyntaxError,run_import,"lex_rule2")
207        result = sys.stderr.getvalue()
208        self.assert_(check_expected(result,
209                                    "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
210
211    def test_lex_rule3(self):
212        self.assertRaises(SyntaxError,run_import,"lex_rule3")
213        result = sys.stderr.getvalue()
214        self.assert_(check_expected(result,
215                                    "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
216
217
218    def test_lex_state1(self):
219        self.assertRaises(SyntaxError,run_import,"lex_state1")
220        result = sys.stderr.getvalue()
221        self.assert_(check_expected(result,
222                                   "states must be defined as a tuple or list\n"))
223
224    def test_lex_state2(self):
225        self.assertRaises(SyntaxError,run_import,"lex_state2")
226        result = sys.stderr.getvalue()
227        self.assert_(check_expected(result,
228                                    "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
229                                    "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
230
231    def test_lex_state3(self):
232        self.assertRaises(SyntaxError,run_import,"lex_state3")
233        result = sys.stderr.getvalue()
234        self.assert_(check_expected(result,
235                                    "State name 1 must be a string\n"
236                                    "No rules defined for state 'example'\n"))
237
238    def test_lex_state4(self):
239        self.assertRaises(SyntaxError,run_import,"lex_state4")
240        result = sys.stderr.getvalue()
241        self.assert_(check_expected(result,
242                                    "State type for state comment must be 'inclusive' or 'exclusive'\n"))
243
244
245    def test_lex_state5(self):
246        self.assertRaises(SyntaxError,run_import,"lex_state5")
247        result = sys.stderr.getvalue()
248        self.assert_(check_expected(result,
249                                    "State 'comment' already defined\n"))
250
251    def test_lex_state_noerror(self):
252        run_import("lex_state_noerror")
253        result = sys.stderr.getvalue()
254        self.assert_(check_expected(result,
255                                    "No error rule is defined for exclusive state 'comment'\n"))
256
257    def test_lex_state_norule(self):
258        self.assertRaises(SyntaxError,run_import,"lex_state_norule")
259        result = sys.stderr.getvalue()
260        self.assert_(check_expected(result,
261                                    "No rules defined for state 'example'\n"))
262
263    def test_lex_token1(self):
264        self.assertRaises(SyntaxError,run_import,"lex_token1")
265        result = sys.stderr.getvalue()
266        self.assert_(check_expected(result,
267                                    "No token list is defined\n"
268                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
269                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
270                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
271))
272
273    def test_lex_token2(self):
274        self.assertRaises(SyntaxError,run_import,"lex_token2")
275        result = sys.stderr.getvalue()
276        self.assert_(check_expected(result,
277                                    "tokens must be a list or tuple\n"
278                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
279                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
280                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
281))
282
283    def test_lex_token3(self):
284        self.assertRaises(SyntaxError,run_import,"lex_token3")
285        result = sys.stderr.getvalue()
286        self.assert_(check_expected(result,
287                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
288
289
290    def test_lex_token4(self):
291        self.assertRaises(SyntaxError,run_import,"lex_token4")
292        result = sys.stderr.getvalue()
293        self.assert_(check_expected(result,
294                                    "Bad token name '-'\n"))
295
296
297    def test_lex_token5(self):
298        try:
299            run_import("lex_token5")
300        except ply.lex.LexError:
301            e = sys.exc_info()[1]
302        self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
303
304    def test_lex_token_dup(self):
305        run_import("lex_token_dup")
306        result = sys.stderr.getvalue()
307        self.assert_(check_expected(result,
308                                    "Token 'MINUS' multiply defined\n"))
309
310
311    def test_lex_literal1(self):
312        self.assertRaises(SyntaxError,run_import,"lex_literal1")
313        result = sys.stderr.getvalue()
314        self.assert_(check_expected(result,
315                                    "Invalid literal '**'. Must be a single character\n"))
316
317    def test_lex_literal2(self):
318        self.assertRaises(SyntaxError,run_import,"lex_literal2")
319        result = sys.stderr.getvalue()
320        self.assert_(check_expected(result,
321                                    "Invalid literals specification. literals must be a sequence of characters\n"))
322
323import os
324import subprocess
325import shutil
326
327# Tests related to various build options associated with lexers
328class LexBuildOptionTests(unittest.TestCase):
329    def setUp(self):
330        sys.stderr = StringIO.StringIO()
331        sys.stdout = StringIO.StringIO()
332    def tearDown(self):
333        sys.stderr = sys.__stderr__
334        sys.stdout = sys.__stdout__
335        try:
336            shutil.rmtree("lexdir")
337        except OSError:
338            pass
339
340    def test_lex_module(self):
341        run_import("lex_module")
342        result = sys.stdout.getvalue()
343        self.assert_(check_expected(result,
344                                    "(NUMBER,3,1,0)\n"
345                                    "(PLUS,'+',1,1)\n"
346                                    "(NUMBER,4,1,2)\n"))
347
348    def test_lex_object(self):
349        run_import("lex_object")
350        result = sys.stdout.getvalue()
351        self.assert_(check_expected(result,
352                                    "(NUMBER,3,1,0)\n"
353                                    "(PLUS,'+',1,1)\n"
354                                    "(NUMBER,4,1,2)\n"))
355
356    def test_lex_closure(self):
357        run_import("lex_closure")
358        result = sys.stdout.getvalue()
359        self.assert_(check_expected(result,
360                                    "(NUMBER,3,1,0)\n"
361                                    "(PLUS,'+',1,1)\n"
362                                    "(NUMBER,4,1,2)\n"))
363
364    def test_lex_optimize(self):
365        try:
366            os.remove("lextab.py")
367        except OSError:
368            pass
369        try:
370            os.remove("lextab.pyc")
371        except OSError:
372            pass
373        try:
374            os.remove("lextab.pyo")
375        except OSError:
376            pass
377        run_import("lex_optimize")
378
379        result = sys.stdout.getvalue()
380        self.assert_(check_expected(result,
381                                    "(NUMBER,3,1,0)\n"
382                                    "(PLUS,'+',1,1)\n"
383                                    "(NUMBER,4,1,2)\n"))
384        self.assert_(os.path.exists("lextab.py"))
385
386        p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
387                             stdout=subprocess.PIPE)
388        result = p.stdout.read()
389
390        self.assert_(check_expected(result,
391                                    "(NUMBER,3,1,0)\n"
392                                    "(PLUS,'+',1,1)\n"
393                                    "(NUMBER,4,1,2)\n"))
394        if test_pyo:
395            self.assert_(pymodule_out_exists("lextab.pyo", 1))
396            pymodule_out_remove("lextab.pyo", 1)
397
398        p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
399                             stdout=subprocess.PIPE)
400        result = p.stdout.read()
401        self.assert_(check_expected(result,
402                                    "(NUMBER,3,1,0)\n"
403                                    "(PLUS,'+',1,1)\n"
404                                    "(NUMBER,4,1,2)\n"))
405
406        if test_pyo:
407            self.assert_(pymodule_out_exists("lextab.pyo", 2))
408        try:
409            os.remove("lextab.py")
410        except OSError:
411            pass
412        try:
413            pymodule_out_remove("lextab.pyc")
414        except OSError:
415            pass
416        try:
417            pymodule_out_remove("lextab.pyo", 2)
418        except OSError:
419            pass
420
421    def test_lex_optimize2(self):
422        try:
423            os.remove("opt2tab.py")
424        except OSError:
425            pass
426        try:
427            os.remove("opt2tab.pyc")
428        except OSError:
429            pass
430        try:
431            os.remove("opt2tab.pyo")
432        except OSError:
433            pass
434        run_import("lex_optimize2")
435        result = sys.stdout.getvalue()
436        self.assert_(check_expected(result,
437                                    "(NUMBER,3,1,0)\n"
438                                    "(PLUS,'+',1,1)\n"
439                                    "(NUMBER,4,1,2)\n"))
440        self.assert_(os.path.exists("opt2tab.py"))
441
442        p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
443                             stdout=subprocess.PIPE)
444        result = p.stdout.read()
445        self.assert_(check_expected(result,
446                                    "(NUMBER,3,1,0)\n"
447                                    "(PLUS,'+',1,1)\n"
448                                    "(NUMBER,4,1,2)\n"))
449        if test_pyo:
450            self.assert_(pymodule_out_exists("opt2tab.pyo", 1))
451            pymodule_out_remove("opt2tab.pyo", 1)
452        p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
453                             stdout=subprocess.PIPE)
454        result = p.stdout.read()
455        self.assert_(check_expected(result,
456                                    "(NUMBER,3,1,0)\n"
457                                    "(PLUS,'+',1,1)\n"
458                                    "(NUMBER,4,1,2)\n"))
459        if test_pyo:
460            self.assert_(pymodule_out_exists("opt2tab.pyo", 2))
461        try:
462            os.remove("opt2tab.py")
463        except OSError:
464            pass
465        try:
466            pymodule_out_remove("opt2tab.pyc")
467        except OSError:
468            pass
469        try:
470            pymodule_out_remove("opt2tab.pyo", 2)
471        except OSError:
472            pass
473
474    def test_lex_optimize3(self):
475        try:
476            shutil.rmtree("lexdir")
477        except OSError:
478            pass
479
480        os.mkdir("lexdir")
481        os.mkdir("lexdir/sub")
482        open("lexdir/__init__.py","w").write("")
483        open("lexdir/sub/__init__.py","w").write("")
484        run_import("lex_optimize3")
485        result = sys.stdout.getvalue()
486        self.assert_(check_expected(result,
487                                    "(NUMBER,3,1,0)\n"
488                                    "(PLUS,'+',1,1)\n"
489                                    "(NUMBER,4,1,2)\n"))
490        self.assert_(os.path.exists("lexdir/sub/calctab.py"))
491
492        p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
493                             stdout=subprocess.PIPE)
494        result = p.stdout.read()
495        self.assert_(check_expected(result,
496                                    "(NUMBER,3,1,0)\n"
497                                    "(PLUS,'+',1,1)\n"
498                                    "(NUMBER,4,1,2)\n"))
499        if test_pyo:
500            self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 1))
501            pymodule_out_remove("lexdir/sub/calctab.pyo", 1)
502
503        p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
504                             stdout=subprocess.PIPE)
505        result = p.stdout.read()
506        self.assert_(check_expected(result,
507                                    "(NUMBER,3,1,0)\n"
508                                    "(PLUS,'+',1,1)\n"
509                                    "(NUMBER,4,1,2)\n"))
510        if test_pyo:
511            self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 2))
512        try:
513            shutil.rmtree("lexdir")
514        except OSError:
515            pass
516
517    def test_lex_optimize4(self):
518
519        # Regression test to make sure that reflags works correctly
520        # on Python 3.
521
522        for extension in ['py', 'pyc']:
523            try:
524                os.remove("opt4tab.{0}".format(extension))
525            except OSError:
526                pass
527
528        run_import("lex_optimize4")
529        run_import("lex_optimize4")
530
531        for extension in ['py', 'pyc']:
532            try:
533                os.remove("opt4tab.{0}".format(extension))
534            except OSError:
535                pass
536
537    def test_lex_opt_alias(self):
538        try:
539            os.remove("aliastab.py")
540        except OSError:
541            pass
542        try:
543            os.remove("aliastab.pyc")
544        except OSError:
545            pass
546        try:
547            os.remove("aliastab.pyo")
548        except OSError:
549            pass
550        run_import("lex_opt_alias")
551        result = sys.stdout.getvalue()
552        self.assert_(check_expected(result,
553                                    "(NUMBER,3,1,0)\n"
554                                    "(+,'+',1,1)\n"
555                                    "(NUMBER,4,1,2)\n"))
556        self.assert_(os.path.exists("aliastab.py"))
557
558        p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
559                             stdout=subprocess.PIPE)
560        result = p.stdout.read()
561        self.assert_(check_expected(result,
562                                    "(NUMBER,3,1,0)\n"
563                                    "(+,'+',1,1)\n"
564                                    "(NUMBER,4,1,2)\n"))
565        if test_pyo:
566            self.assert_(pymodule_out_exists("aliastab.pyo", 1))
567            pymodule_out_remove("aliastab.pyo", 1)
568
569        p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
570                             stdout=subprocess.PIPE)
571        result = p.stdout.read()
572        self.assert_(check_expected(result,
573                                    "(NUMBER,3,1,0)\n"
574                                    "(+,'+',1,1)\n"
575                                    "(NUMBER,4,1,2)\n"))
576
577        if test_pyo:
578            self.assert_(pymodule_out_exists("aliastab.pyo", 2))
579        try:
580            os.remove("aliastab.py")
581        except OSError:
582            pass
583        try:
584            pymodule_out_remove("aliastab.pyc")
585        except OSError:
586            pass
587        try:
588            pymodule_out_remove("aliastab.pyo", 2)
589        except OSError:
590            pass
591
592    def test_lex_many_tokens(self):
593        try:
594            os.remove("manytab.py")
595        except OSError:
596            pass
597        try:
598            os.remove("manytab.pyc")
599        except OSError:
600            pass
601        try:
602            os.remove("manytab.pyo")
603        except OSError:
604            pass
605        run_import("lex_many_tokens")
606        result = sys.stdout.getvalue()
607        self.assert_(check_expected(result,
608                                    "(TOK34,'TOK34:',1,0)\n"
609                                    "(TOK143,'TOK143:',1,7)\n"
610                                    "(TOK269,'TOK269:',1,15)\n"
611                                    "(TOK372,'TOK372:',1,23)\n"
612                                    "(TOK452,'TOK452:',1,31)\n"
613                                    "(TOK561,'TOK561:',1,39)\n"
614                                    "(TOK999,'TOK999:',1,47)\n"
615                                    ))
616
617        self.assert_(os.path.exists("manytab.py"))
618
619        if implementation() == 'CPython':
620            p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
621                                 stdout=subprocess.PIPE)
622            result = p.stdout.read()
623            self.assert_(check_expected(result,
624                                        "(TOK34,'TOK34:',1,0)\n"
625                                        "(TOK143,'TOK143:',1,7)\n"
626                                        "(TOK269,'TOK269:',1,15)\n"
627                                        "(TOK372,'TOK372:',1,23)\n"
628                                        "(TOK452,'TOK452:',1,31)\n"
629                                        "(TOK561,'TOK561:',1,39)\n"
630                                        "(TOK999,'TOK999:',1,47)\n"
631                                        ))
632
633            self.assert_(pymodule_out_exists("manytab.pyo", 1))
634            pymodule_out_remove("manytab.pyo", 1)
635        try:
636            os.remove("manytab.py")
637        except OSError:
638            pass
639        try:
640            os.remove("manytab.pyc")
641        except OSError:
642            pass
643        try:
644            os.remove("manytab.pyo")
645        except OSError:
646            pass
647
648# Tests related to run-time behavior of lexers
649class LexRunTests(unittest.TestCase):
650    def setUp(self):
651        sys.stderr = StringIO.StringIO()
652        sys.stdout = StringIO.StringIO()
653    def tearDown(self):
654        sys.stderr = sys.__stderr__
655        sys.stdout = sys.__stdout__
656
657    def test_lex_hedit(self):
658        run_import("lex_hedit")
659        result = sys.stdout.getvalue()
660        self.assert_(check_expected(result,
661                                    "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
662                                    "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
663                                    "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
664
665    def test_lex_state_try(self):
666        run_import("lex_state_try")
667        result = sys.stdout.getvalue()
668        self.assert_(check_expected(result,
669                                    "(NUMBER,'3',1,0)\n"
670                                    "(PLUS,'+',1,2)\n"
671                                    "(NUMBER,'4',1,4)\n"
672                                    "Entering comment state\n"
673                                    "comment body LexToken(body_part,'This is a comment */',1,9)\n"
674                                    "(PLUS,'+',1,30)\n"
675                                    "(NUMBER,'10',1,32)\n"
676                                    ))
677
678
679
680unittest.main()
681