1import copy 2import parser 3import pickle 4import unittest 5import sys 6import struct 7from test import test_support as support 8from test.script_helper import assert_python_failure 9 10# 11# First, we test that we can generate trees from valid source fragments, 12# and that these valid trees are indeed allowed by the tree-loading side 13# of the parser module. 14# 15 16class RoundtripLegalSyntaxTestCase(unittest.TestCase): 17 18 def roundtrip(self, f, s): 19 st1 = f(s) 20 t = st1.totuple() 21 try: 22 st2 = parser.sequence2st(t) 23 except parser.ParserError, why: 24 self.fail("could not roundtrip %r: %s" % (s, why)) 25 26 self.assertEqual(t, st2.totuple(), 27 "could not re-generate syntax tree") 28 29 def check_expr(self, s): 30 self.roundtrip(parser.expr, s) 31 32 def test_flags_passed(self): 33 # The unicode literals flags has to be passed from the paser to AST 34 # generation. 35 suite = parser.suite("from __future__ import unicode_literals; x = ''") 36 code = suite.compile() 37 scope = {} 38 exec code in scope 39 self.assertIsInstance(scope["x"], unicode) 40 41 def check_suite(self, s): 42 self.roundtrip(parser.suite, s) 43 44 def test_yield_statement(self): 45 self.check_suite("def f(): yield 1") 46 self.check_suite("def f(): yield") 47 self.check_suite("def f(): x += yield") 48 self.check_suite("def f(): x = yield 1") 49 self.check_suite("def f(): x = y = yield 1") 50 self.check_suite("def f(): x = yield") 51 self.check_suite("def f(): x = y = yield") 52 self.check_suite("def f(): 1 + (yield)*2") 53 self.check_suite("def f(): (yield 1)*2") 54 self.check_suite("def f(): return; yield 1") 55 self.check_suite("def f(): yield 1; return") 56 self.check_suite("def f():\n" 57 " for x in range(30):\n" 58 " yield x\n") 59 self.check_suite("def f():\n" 60 " if (yield):\n" 61 " yield x\n") 62 63 def test_expressions(self): 64 self.check_expr("foo(1)") 65 self.check_expr("{1:1}") 66 self.check_expr("{1:1, 2:2, 3:3}") 67 self.check_expr("{1:1, 2:2, 3:3,}") 68 self.check_expr("{1}") 69 self.check_expr("{1, 2, 3}") 70 self.check_expr("{1, 2, 3,}") 71 self.check_expr("[]") 72 self.check_expr("[1]") 73 self.check_expr("[1, 2, 3]") 74 self.check_expr("[1, 2, 3,]") 75 self.check_expr("()") 76 self.check_expr("(1,)") 77 self.check_expr("(1, 2, 3)") 78 self.check_expr("(1, 2, 3,)") 79 self.check_expr("[x**3 for x in range(20)]") 80 self.check_expr("[x**3 for x in range(20) if x % 3]") 81 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") 82 self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]") 83 #self.check_expr("[x for x in lambda: True, lambda: False if x()]") 84 self.check_expr("list(x**3 for x in range(20))") 85 self.check_expr("list(x**3 for x in range(20) if x % 3)") 86 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") 87 self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)") 88 self.check_expr("{x**3 for x in range(30)}") 89 self.check_expr("{x**3 for x in range(30) if x % 3}") 90 self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}") 91 self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}") 92 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}") 93 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}") 94 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}") 95 self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}") 96 self.check_expr("foo(*args)") 97 self.check_expr("foo(*args, **kw)") 98 self.check_expr("foo(**kw)") 99 self.check_expr("foo(key=value)") 100 self.check_expr("foo(key=value, *args)") 101 self.check_expr("foo(key=value, *args, **kw)") 102 self.check_expr("foo(key=value, **kw)") 103 self.check_expr("foo(a, b, c, *args)") 104 self.check_expr("foo(a, b, c, *args, **kw)") 105 self.check_expr("foo(a, b, c, **kw)") 106 self.check_expr("foo(a, *args, keyword=23)") 107 self.check_expr("foo + bar") 108 self.check_expr("foo - bar") 109 self.check_expr("foo * bar") 110 self.check_expr("foo / bar") 111 self.check_expr("foo // bar") 112 self.check_expr("lambda: 0") 113 self.check_expr("lambda x: 0") 114 self.check_expr("lambda *y: 0") 115 self.check_expr("lambda *y, **z: 0") 116 self.check_expr("lambda **z: 0") 117 self.check_expr("lambda x, y: 0") 118 self.check_expr("lambda foo=bar: 0") 119 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") 120 self.check_expr("lambda foo=bar, **z: 0") 121 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") 122 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") 123 self.check_expr("lambda x, *y, **z: 0") 124 self.check_expr("lambda x: 5 if x else 2") 125 self.check_expr("(x for x in range(10))") 126 self.check_expr("foo(x for x in range(10))") 127 128 def test_print(self): 129 self.check_suite("print") 130 self.check_suite("print 1") 131 self.check_suite("print 1,") 132 self.check_suite("print >>fp") 133 self.check_suite("print >>fp, 1") 134 self.check_suite("print >>fp, 1,") 135 136 def test_simple_expression(self): 137 # expr_stmt 138 self.check_suite("a") 139 140 def test_simple_assignments(self): 141 self.check_suite("a = b") 142 self.check_suite("a = b = c = d = e") 143 144 def test_simple_augmented_assignments(self): 145 self.check_suite("a += b") 146 self.check_suite("a -= b") 147 self.check_suite("a *= b") 148 self.check_suite("a /= b") 149 self.check_suite("a //= b") 150 self.check_suite("a %= b") 151 self.check_suite("a &= b") 152 self.check_suite("a |= b") 153 self.check_suite("a ^= b") 154 self.check_suite("a <<= b") 155 self.check_suite("a >>= b") 156 self.check_suite("a **= b") 157 158 def test_function_defs(self): 159 self.check_suite("def f(): pass") 160 self.check_suite("def f(*args): pass") 161 self.check_suite("def f(*args, **kw): pass") 162 self.check_suite("def f(**kw): pass") 163 self.check_suite("def f(foo=bar): pass") 164 self.check_suite("def f(foo=bar, *args): pass") 165 self.check_suite("def f(foo=bar, *args, **kw): pass") 166 self.check_suite("def f(foo=bar, **kw): pass") 167 168 self.check_suite("def f(a, b): pass") 169 self.check_suite("def f(a, b, *args): pass") 170 self.check_suite("def f(a, b, *args, **kw): pass") 171 self.check_suite("def f(a, b, **kw): pass") 172 self.check_suite("def f(a, b, foo=bar): pass") 173 self.check_suite("def f(a, b, foo=bar, *args): pass") 174 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") 175 self.check_suite("def f(a, b, foo=bar, **kw): pass") 176 177 self.check_suite("@staticmethod\n" 178 "def f(): pass") 179 self.check_suite("@staticmethod\n" 180 "@funcattrs(x, y)\n" 181 "def f(): pass") 182 self.check_suite("@funcattrs()\n" 183 "def f(): pass") 184 185 def test_class_defs(self): 186 self.check_suite("class foo():pass") 187 self.check_suite("@class_decorator\n" 188 "class foo():pass") 189 self.check_suite("@class_decorator(arg)\n" 190 "class foo():pass") 191 self.check_suite("@decorator1\n" 192 "@decorator2\n" 193 "class foo():pass") 194 195 196 def test_import_from_statement(self): 197 self.check_suite("from sys.path import *") 198 self.check_suite("from sys.path import dirname") 199 self.check_suite("from sys.path import (dirname)") 200 self.check_suite("from sys.path import (dirname,)") 201 self.check_suite("from sys.path import dirname as my_dirname") 202 self.check_suite("from sys.path import (dirname as my_dirname)") 203 self.check_suite("from sys.path import (dirname as my_dirname,)") 204 self.check_suite("from sys.path import dirname, basename") 205 self.check_suite("from sys.path import (dirname, basename)") 206 self.check_suite("from sys.path import (dirname, basename,)") 207 self.check_suite( 208 "from sys.path import dirname as my_dirname, basename") 209 self.check_suite( 210 "from sys.path import (dirname as my_dirname, basename)") 211 self.check_suite( 212 "from sys.path import (dirname as my_dirname, basename,)") 213 self.check_suite( 214 "from sys.path import dirname, basename as my_basename") 215 self.check_suite( 216 "from sys.path import (dirname, basename as my_basename)") 217 self.check_suite( 218 "from sys.path import (dirname, basename as my_basename,)") 219 self.check_suite("from .bogus import x") 220 221 def test_basic_import_statement(self): 222 self.check_suite("import sys") 223 self.check_suite("import sys as system") 224 self.check_suite("import sys, math") 225 self.check_suite("import sys as system, math") 226 self.check_suite("import sys, math as my_math") 227 228 def test_relative_imports(self): 229 self.check_suite("from . import name") 230 self.check_suite("from .. import name") 231 self.check_suite("from .pkg import name") 232 self.check_suite("from ..pkg import name") 233 234 def test_pep263(self): 235 self.check_suite("# -*- coding: iso-8859-1 -*-\n" 236 "pass\n") 237 238 def test_assert(self): 239 self.check_suite("assert alo < ahi and blo < bhi\n") 240 241 def test_with(self): 242 self.check_suite("with open('x'): pass\n") 243 self.check_suite("with open('x') as f: pass\n") 244 self.check_suite("with open('x') as f, open('y') as g: pass\n") 245 246 def test_try_stmt(self): 247 self.check_suite("try: pass\nexcept: pass\n") 248 self.check_suite("try: pass\nfinally: pass\n") 249 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n") 250 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n" 251 "finally: pass\n") 252 self.check_suite("try: pass\nexcept: pass\nelse: pass\n") 253 self.check_suite("try: pass\nexcept: pass\nelse: pass\n" 254 "finally: pass\n") 255 256 def test_except_clause(self): 257 self.check_suite("try: pass\nexcept: pass\n") 258 self.check_suite("try: pass\nexcept A: pass\n") 259 self.check_suite("try: pass\nexcept A, e: pass\n") 260 self.check_suite("try: pass\nexcept A as e: pass\n") 261 262 def test_position(self): 263 # An absolutely minimal test of position information. Better 264 # tests would be a big project. 265 code = "def f(x):\n return x + 1" 266 st = parser.suite(code) 267 268 def walk(tree): 269 node_type = tree[0] 270 next = tree[1] 271 if isinstance(next, (tuple, list)): 272 for elt in tree[1:]: 273 for x in walk(elt): 274 yield x 275 else: 276 yield tree 277 278 expected = [ 279 (1, 'def', 1, 0), 280 (1, 'f', 1, 4), 281 (7, '(', 1, 5), 282 (1, 'x', 1, 6), 283 (8, ')', 1, 7), 284 (11, ':', 1, 8), 285 (4, '', 1, 9), 286 (5, '', 2, -1), 287 (1, 'return', 2, 4), 288 (1, 'x', 2, 11), 289 (14, '+', 2, 13), 290 (2, '1', 2, 15), 291 (4, '', 2, 16), 292 (6, '', 2, -1), 293 (4, '', 2, -1), 294 (0, '', 2, -1), 295 ] 296 297 self.assertEqual(list(walk(st.totuple(line_info=True, col_info=True))), 298 expected) 299 self.assertEqual(list(walk(st.totuple())), 300 [(t, n) for t, n, l, c in expected]) 301 self.assertEqual(list(walk(st.totuple(line_info=True))), 302 [(t, n, l) for t, n, l, c in expected]) 303 self.assertEqual(list(walk(st.totuple(col_info=True))), 304 [(t, n, c) for t, n, l, c in expected]) 305 self.assertEqual(list(walk(st.tolist(line_info=True, col_info=True))), 306 [list(x) for x in expected]) 307 self.assertEqual(list(walk(parser.st2tuple(st, line_info=True, 308 col_info=True))), 309 expected) 310 self.assertEqual(list(walk(parser.st2list(st, line_info=True, 311 col_info=True))), 312 [list(x) for x in expected]) 313 314 315# 316# Second, we take *invalid* trees and make sure we get ParserError 317# rejections for them. 318# 319 320class IllegalSyntaxTestCase(unittest.TestCase): 321 322 def check_bad_tree(self, tree, label): 323 try: 324 parser.sequence2st(tree) 325 except parser.ParserError: 326 pass 327 else: 328 self.fail("did not detect invalid tree for %r" % label) 329 330 def test_junk(self): 331 # not even remotely valid: 332 self.check_bad_tree((1, 2, 3), "<junk>") 333 334 def test_illegal_terminal(self): 335 tree = \ 336 (257, 337 (267, 338 (268, 339 (269, 340 (274, 341 (1,))), 342 (4, ''))), 343 (4, ''), 344 (0, '')) 345 self.check_bad_tree(tree, "too small items in terminal node") 346 tree = \ 347 (257, 348 (267, 349 (268, 350 (269, 351 (274, 352 (1, u'pass'))), 353 (4, ''))), 354 (4, ''), 355 (0, '')) 356 self.check_bad_tree(tree, "non-string second item in terminal node") 357 tree = \ 358 (257, 359 (267, 360 (268, 361 (269, 362 (274, 363 (1, 'pass', '0', 0))), 364 (4, ''))), 365 (4, ''), 366 (0, '')) 367 self.check_bad_tree(tree, "non-integer third item in terminal node") 368 tree = \ 369 (257, 370 (267, 371 (268, 372 (269, 373 (274, 374 (1, 'pass', 0, 0))), 375 (4, ''))), 376 (4, ''), 377 (0, '')) 378 self.check_bad_tree(tree, "too many items in terminal node") 379 380 def test_illegal_yield_1(self): 381 # Illegal yield statement: def f(): return 1; yield 1 382 tree = \ 383 (257, 384 (264, 385 (285, 386 (259, 387 (1, 'def'), 388 (1, 'f'), 389 (260, (7, '('), (8, ')')), 390 (11, ':'), 391 (291, 392 (4, ''), 393 (5, ''), 394 (264, 395 (265, 396 (266, 397 (272, 398 (275, 399 (1, 'return'), 400 (313, 401 (292, 402 (293, 403 (294, 404 (295, 405 (297, 406 (298, 407 (299, 408 (300, 409 (301, 410 (302, (303, (304, (305, (2, '1')))))))))))))))))), 411 (264, 412 (265, 413 (266, 414 (272, 415 (276, 416 (1, 'yield'), 417 (313, 418 (292, 419 (293, 420 (294, 421 (295, 422 (297, 423 (298, 424 (299, 425 (300, 426 (301, 427 (302, 428 (303, (304, (305, (2, '1')))))))))))))))))), 429 (4, ''))), 430 (6, ''))))), 431 (4, ''), 432 (0, '')))) 433 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 434 435 def test_illegal_yield_2(self): 436 # Illegal return in generator: def f(): return 1; yield 1 437 tree = \ 438 (257, 439 (264, 440 (265, 441 (266, 442 (278, 443 (1, 'from'), 444 (281, (1, '__future__')), 445 (1, 'import'), 446 (279, (1, 'generators')))), 447 (4, ''))), 448 (264, 449 (285, 450 (259, 451 (1, 'def'), 452 (1, 'f'), 453 (260, (7, '('), (8, ')')), 454 (11, ':'), 455 (291, 456 (4, ''), 457 (5, ''), 458 (264, 459 (265, 460 (266, 461 (272, 462 (275, 463 (1, 'return'), 464 (313, 465 (292, 466 (293, 467 (294, 468 (295, 469 (297, 470 (298, 471 (299, 472 (300, 473 (301, 474 (302, (303, (304, (305, (2, '1')))))))))))))))))), 475 (264, 476 (265, 477 (266, 478 (272, 479 (276, 480 (1, 'yield'), 481 (313, 482 (292, 483 (293, 484 (294, 485 (295, 486 (297, 487 (298, 488 (299, 489 (300, 490 (301, 491 (302, 492 (303, (304, (305, (2, '1')))))))))))))))))), 493 (4, ''))), 494 (6, ''))))), 495 (4, ''), 496 (0, '')))) 497 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 498 499 def test_print_chevron_comma(self): 500 # Illegal input: print >>fp, 501 tree = \ 502 (257, 503 (264, 504 (265, 505 (266, 506 (268, 507 (1, 'print'), 508 (35, '>>'), 509 (290, 510 (291, 511 (292, 512 (293, 513 (295, 514 (296, 515 (297, 516 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))), 517 (12, ','))), 518 (4, ''))), 519 (0, '')) 520 self.check_bad_tree(tree, "print >>fp,") 521 522 def test_a_comma_comma_c(self): 523 # Illegal input: a,,c 524 tree = \ 525 (258, 526 (311, 527 (290, 528 (291, 529 (292, 530 (293, 531 (295, 532 (296, 533 (297, 534 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), 535 (12, ','), 536 (12, ','), 537 (290, 538 (291, 539 (292, 540 (293, 541 (295, 542 (296, 543 (297, 544 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), 545 (4, ''), 546 (0, '')) 547 self.check_bad_tree(tree, "a,,c") 548 549 def test_illegal_operator(self): 550 # Illegal input: a $= b 551 tree = \ 552 (257, 553 (264, 554 (265, 555 (266, 556 (267, 557 (312, 558 (291, 559 (292, 560 (293, 561 (294, 562 (296, 563 (297, 564 (298, 565 (299, 566 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), 567 (268, (37, '$=')), 568 (312, 569 (291, 570 (292, 571 (293, 572 (294, 573 (296, 574 (297, 575 (298, 576 (299, 577 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), 578 (4, ''))), 579 (0, '')) 580 self.check_bad_tree(tree, "a $= b") 581 582 def test_malformed_global(self): 583 #doesn't have global keyword in ast 584 tree = (257, 585 (264, 586 (265, 587 (266, 588 (282, (1, 'foo'))), (4, ''))), 589 (4, ''), 590 (0, '')) 591 self.check_bad_tree(tree, "malformed global ast") 592 593 def test_missing_import_source(self): 594 # from import a 595 tree = \ 596 (257, 597 (267, 598 (268, 599 (269, 600 (281, 601 (283, (1, 'from'), (1, 'import'), 602 (286, (284, (1, 'fred')))))), 603 (4, ''))), 604 (4, ''), (0, '')) 605 self.check_bad_tree(tree, "from import a") 606 607 def test_illegal_encoding(self): 608 # Illegal encoding declaration 609 tree = \ 610 (339, 611 (257, (0, ''))) 612 self.check_bad_tree(tree, "missed encoding") 613 tree = \ 614 (339, 615 (257, (0, '')), 616 u'iso-8859-1') 617 self.check_bad_tree(tree, "non-string encoding") 618 619 620class CompileTestCase(unittest.TestCase): 621 622 # These tests are very minimal. :-( 623 624 def test_compile_expr(self): 625 st = parser.expr('2 + 3') 626 code = parser.compilest(st) 627 self.assertEqual(eval(code), 5) 628 629 def test_compile_suite(self): 630 st = parser.suite('x = 2; y = x + 3') 631 code = parser.compilest(st) 632 globs = {} 633 exec code in globs 634 self.assertEqual(globs['y'], 5) 635 636 def test_compile_error(self): 637 st = parser.suite('1 = 3 + 4') 638 self.assertRaises(SyntaxError, parser.compilest, st) 639 640 def test_compile_badunicode(self): 641 st = parser.suite('a = u"\U12345678"') 642 self.assertRaises(SyntaxError, parser.compilest, st) 643 st = parser.suite('a = u"\u1"') 644 self.assertRaises(SyntaxError, parser.compilest, st) 645 646 def test_issue_9011(self): 647 # Issue 9011: compilation of an unary minus expression changed 648 # the meaning of the ST, so that a second compilation produced 649 # incorrect results. 650 st = parser.expr('-3') 651 code1 = parser.compilest(st) 652 self.assertEqual(eval(code1), -3) 653 code2 = parser.compilest(st) 654 self.assertEqual(eval(code2), -3) 655 656 657class ParserStackLimitTestCase(unittest.TestCase): 658 """try to push the parser to/over its limits. 659 see http://bugs.python.org/issue1881 for a discussion 660 """ 661 def _nested_expression(self, level): 662 return "["*level+"]"*level 663 664 def test_deeply_nested_list(self): 665 e = self._nested_expression(99) 666 st = parser.expr(e) 667 st.compile() 668 669 def test_trigger_memory_error(self): 670 e = self._nested_expression(100) 671 rc, out, err = assert_python_failure('-c', e) 672 # parsing the expression will result in an error message 673 # followed by a MemoryError (see #11963) 674 self.assertIn(b's_push: parser stack overflow', err) 675 self.assertIn(b'MemoryError', err) 676 677class STObjectTestCase(unittest.TestCase): 678 """Test operations on ST objects themselves""" 679 680 def test_copy_pickle(self): 681 sts = [ 682 parser.expr('2 + 3'), 683 parser.suite('x = 2; y = x + 3'), 684 parser.expr('list(x**3 for x in range(20))') 685 ] 686 for st in sts: 687 st_copy = copy.copy(st) 688 self.assertEqual(st_copy.totuple(), st.totuple()) 689 st_copy = copy.deepcopy(st) 690 self.assertEqual(st_copy.totuple(), st.totuple()) 691 for proto in range(pickle.HIGHEST_PROTOCOL+1): 692 st_copy = pickle.loads(pickle.dumps(st, proto)) 693 self.assertEqual(st_copy.totuple(), st.totuple()) 694 695 check_sizeof = support.check_sizeof 696 697 @support.cpython_only 698 def test_sizeof(self): 699 def XXXROUNDUP(n): 700 if n <= 1: 701 return n 702 if n <= 128: 703 return (n + 3) & ~3 704 return 1 << (n - 1).bit_length() 705 706 basesize = support.calcobjsize('Pii') 707 nodesize = struct.calcsize('hP3iP0h') 708 def sizeofchildren(node): 709 if node is None: 710 return 0 711 res = 0 712 hasstr = len(node) > 1 and isinstance(node[-1], str) 713 if hasstr: 714 res += len(node[-1]) + 1 715 children = node[1:-1] if hasstr else node[1:] 716 if children: 717 res += XXXROUNDUP(len(children)) * nodesize 718 for child in children: 719 res += sizeofchildren(child) 720 return res 721 722 def check_st_sizeof(st): 723 self.check_sizeof(st, basesize + nodesize + 724 sizeofchildren(st.totuple())) 725 726 check_st_sizeof(parser.expr('2 + 3')) 727 check_st_sizeof(parser.expr('2 + 3 + 4')) 728 check_st_sizeof(parser.suite('x = 2 + 3')) 729 check_st_sizeof(parser.suite('')) 730 check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-')) 731 check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']')) 732 733 734 # XXX tests for pickling and unpickling of ST objects should go here 735 736def test_main(): 737 support.run_unittest( 738 RoundtripLegalSyntaxTestCase, 739 IllegalSyntaxTestCase, 740 CompileTestCase, 741 ParserStackLimitTestCase, 742 STObjectTestCase, 743 ) 744 745 746if __name__ == "__main__": 747 test_main() 748