1# cython: auto_cpdef=True, infer_types=True, language_level=3, py2_import=True 2# 3# Parser 4# 5 6from __future__ import absolute_import 7 8# This should be done automatically 9import cython 10cython.declare(Nodes=object, ExprNodes=object, EncodedString=object, 11 bytes_literal=object, StringEncoding=object, 12 FileSourceDescriptor=object, lookup_unicodechar=object, unicode_category=object, 13 Future=object, Options=object, error=object, warning=object, 14 Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object, 15 re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object, 16 partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint, 17 _CDEF_MODIFIERS=tuple) 18 19from io import StringIO 20import re 21import sys 22from unicodedata import lookup as lookup_unicodechar, category as unicode_category 23from functools import partial, reduce 24 25from .Scanning import PyrexScanner, FileSourceDescriptor, StringSourceDescriptor 26from . import Nodes 27from . import ExprNodes 28from . import Builtin 29from . import StringEncoding 30from .StringEncoding import EncodedString, bytes_literal, _unicode, _bytes 31from .ModuleNode import ModuleNode 32from .Errors import error, warning 33from .. import Utils 34from . import Future 35from . import Options 36 37_IS_PY3 = sys.version_info[0] >= 3 38_IS_2BYTE_UNICODE = sys.maxunicode == 0xffff 39_CDEF_MODIFIERS = ('inline', 'nogil', 'api') 40 41 42class Ctx(object): 43 # Parsing context 44 level = 'other' 45 visibility = 'private' 46 cdef_flag = 0 47 typedef_flag = 0 48 api = 0 49 overridable = 0 50 nogil = 0 51 namespace = None 52 templates = None 53 allow_struct_enum_decorator = False 54 55 def __init__(self, **kwds): 56 self.__dict__.update(kwds) 57 58 def __call__(self, **kwds): 59 ctx = Ctx() 60 d = ctx.__dict__ 61 d.update(self.__dict__) 62 d.update(kwds) 63 return ctx 64 65 66def p_ident(s, message="Expected an identifier"): 67 if s.sy == 'IDENT': 68 name = s.systring 69 s.next() 70 return name 71 else: 72 s.error(message) 73 74def p_ident_list(s): 75 names = [] 76 while s.sy == 'IDENT': 77 names.append(s.systring) 78 s.next() 79 if s.sy != ',': 80 break 81 s.next() 82 return names 83 84#------------------------------------------ 85# 86# Expressions 87# 88#------------------------------------------ 89 90def p_binop_operator(s): 91 pos = s.position() 92 op = s.sy 93 s.next() 94 return op, pos 95 96def p_binop_expr(s, ops, p_sub_expr): 97 n1 = p_sub_expr(s) 98 while s.sy in ops: 99 op, pos = p_binop_operator(s) 100 n2 = p_sub_expr(s) 101 n1 = ExprNodes.binop_node(pos, op, n1, n2) 102 if op == '/': 103 if Future.division in s.context.future_directives: 104 n1.truedivision = True 105 else: 106 n1.truedivision = None # unknown 107 return n1 108 109#lambdef: 'lambda' [varargslist] ':' test 110 111def p_lambdef(s, allow_conditional=True): 112 # s.sy == 'lambda' 113 pos = s.position() 114 s.next() 115 if s.sy == ':': 116 args = [] 117 star_arg = starstar_arg = None 118 else: 119 args, star_arg, starstar_arg = p_varargslist( 120 s, terminator=':', annotated=False) 121 s.expect(':') 122 if allow_conditional: 123 expr = p_test(s) 124 else: 125 expr = p_test_nocond(s) 126 return ExprNodes.LambdaNode( 127 pos, args = args, 128 star_arg = star_arg, starstar_arg = starstar_arg, 129 result_expr = expr) 130 131#lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 132 133def p_lambdef_nocond(s): 134 return p_lambdef(s, allow_conditional=False) 135 136#test: or_test ['if' or_test 'else' test] | lambdef 137 138def p_test(s): 139 if s.sy == 'lambda': 140 return p_lambdef(s) 141 pos = s.position() 142 expr = p_or_test(s) 143 if s.sy == 'if': 144 s.next() 145 test = p_or_test(s) 146 s.expect('else') 147 other = p_test(s) 148 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other) 149 else: 150 return expr 151 152#test_nocond: or_test | lambdef_nocond 153 154def p_test_nocond(s): 155 if s.sy == 'lambda': 156 return p_lambdef_nocond(s) 157 else: 158 return p_or_test(s) 159 160#or_test: and_test ('or' and_test)* 161 162def p_or_test(s): 163 return p_rassoc_binop_expr(s, ('or',), p_and_test) 164 165def p_rassoc_binop_expr(s, ops, p_subexpr): 166 n1 = p_subexpr(s) 167 if s.sy in ops: 168 pos = s.position() 169 op = s.sy 170 s.next() 171 n2 = p_rassoc_binop_expr(s, ops, p_subexpr) 172 n1 = ExprNodes.binop_node(pos, op, n1, n2) 173 return n1 174 175#and_test: not_test ('and' not_test)* 176 177def p_and_test(s): 178 #return p_binop_expr(s, ('and',), p_not_test) 179 return p_rassoc_binop_expr(s, ('and',), p_not_test) 180 181#not_test: 'not' not_test | comparison 182 183def p_not_test(s): 184 if s.sy == 'not': 185 pos = s.position() 186 s.next() 187 return ExprNodes.NotNode(pos, operand = p_not_test(s)) 188 else: 189 return p_comparison(s) 190 191#comparison: expr (comp_op expr)* 192#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 193 194def p_comparison(s): 195 n1 = p_starred_expr(s) 196 if s.sy in comparison_ops: 197 pos = s.position() 198 op = p_cmp_op(s) 199 n2 = p_starred_expr(s) 200 n1 = ExprNodes.PrimaryCmpNode(pos, 201 operator = op, operand1 = n1, operand2 = n2) 202 if s.sy in comparison_ops: 203 n1.cascade = p_cascaded_cmp(s) 204 return n1 205 206def p_test_or_starred_expr(s): 207 if s.sy == '*': 208 return p_starred_expr(s) 209 else: 210 return p_test(s) 211 212def p_starred_expr(s): 213 pos = s.position() 214 if s.sy == '*': 215 starred = True 216 s.next() 217 else: 218 starred = False 219 expr = p_bit_expr(s) 220 if starred: 221 expr = ExprNodes.StarredUnpackingNode(pos, expr) 222 return expr 223 224def p_cascaded_cmp(s): 225 pos = s.position() 226 op = p_cmp_op(s) 227 n2 = p_starred_expr(s) 228 result = ExprNodes.CascadedCmpNode(pos, 229 operator = op, operand2 = n2) 230 if s.sy in comparison_ops: 231 result.cascade = p_cascaded_cmp(s) 232 return result 233 234def p_cmp_op(s): 235 if s.sy == 'not': 236 s.next() 237 s.expect('in') 238 op = 'not_in' 239 elif s.sy == 'is': 240 s.next() 241 if s.sy == 'not': 242 s.next() 243 op = 'is_not' 244 else: 245 op = 'is' 246 else: 247 op = s.sy 248 s.next() 249 if op == '<>': 250 op = '!=' 251 return op 252 253comparison_ops = cython.declare(set, set([ 254 '<', '>', '==', '>=', '<=', '<>', '!=', 255 'in', 'is', 'not' 256])) 257 258#expr: xor_expr ('|' xor_expr)* 259 260def p_bit_expr(s): 261 return p_binop_expr(s, ('|',), p_xor_expr) 262 263#xor_expr: and_expr ('^' and_expr)* 264 265def p_xor_expr(s): 266 return p_binop_expr(s, ('^',), p_and_expr) 267 268#and_expr: shift_expr ('&' shift_expr)* 269 270def p_and_expr(s): 271 return p_binop_expr(s, ('&',), p_shift_expr) 272 273#shift_expr: arith_expr (('<<'|'>>') arith_expr)* 274 275def p_shift_expr(s): 276 return p_binop_expr(s, ('<<', '>>'), p_arith_expr) 277 278#arith_expr: term (('+'|'-') term)* 279 280def p_arith_expr(s): 281 return p_binop_expr(s, ('+', '-'), p_term) 282 283#term: factor (('*'|'@'|'/'|'%'|'//') factor)* 284 285def p_term(s): 286 return p_binop_expr(s, ('*', '@', '/', '%', '//'), p_factor) 287 288#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power 289 290def p_factor(s): 291 # little indirection for C-ification purposes 292 return _p_factor(s) 293 294def _p_factor(s): 295 sy = s.sy 296 if sy in ('+', '-', '~'): 297 op = s.sy 298 pos = s.position() 299 s.next() 300 return ExprNodes.unop_node(pos, op, p_factor(s)) 301 elif not s.in_python_file: 302 if sy == '&': 303 pos = s.position() 304 s.next() 305 arg = p_factor(s) 306 return ExprNodes.AmpersandNode(pos, operand = arg) 307 elif sy == "<": 308 return p_typecast(s) 309 elif sy == 'IDENT' and s.systring == "sizeof": 310 return p_sizeof(s) 311 return p_power(s) 312 313def p_typecast(s): 314 # s.sy == "<" 315 pos = s.position() 316 s.next() 317 base_type = p_c_base_type(s) 318 is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode) 319 is_template = isinstance(base_type, Nodes.TemplatedTypeNode) 320 is_const = isinstance(base_type, Nodes.CConstTypeNode) 321 if (not is_memslice and not is_template and not is_const 322 and base_type.name is None): 323 s.error("Unknown type") 324 declarator = p_c_declarator(s, empty = 1) 325 if s.sy == '?': 326 s.next() 327 typecheck = 1 328 else: 329 typecheck = 0 330 s.expect(">") 331 operand = p_factor(s) 332 if is_memslice: 333 return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, 334 operand=operand) 335 336 return ExprNodes.TypecastNode(pos, 337 base_type = base_type, 338 declarator = declarator, 339 operand = operand, 340 typecheck = typecheck) 341 342def p_sizeof(s): 343 # s.sy == ident "sizeof" 344 pos = s.position() 345 s.next() 346 s.expect('(') 347 # Here we decide if we are looking at an expression or type 348 # If it is actually a type, but parsable as an expression, 349 # we treat it as an expression here. 350 if looking_at_expr(s): 351 operand = p_test(s) 352 node = ExprNodes.SizeofVarNode(pos, operand = operand) 353 else: 354 base_type = p_c_base_type(s) 355 declarator = p_c_declarator(s, empty = 1) 356 node = ExprNodes.SizeofTypeNode(pos, 357 base_type = base_type, declarator = declarator) 358 s.expect(')') 359 return node 360 361 362def p_yield_expression(s): 363 # s.sy == "yield" 364 pos = s.position() 365 s.next() 366 is_yield_from = False 367 if s.sy == 'from': 368 is_yield_from = True 369 s.next() 370 if s.sy != ')' and s.sy not in statement_terminators: 371 # "yield from" does not support implicit tuples, but "yield" does ("yield 1,2") 372 arg = p_test(s) if is_yield_from else p_testlist(s) 373 else: 374 if is_yield_from: 375 s.error("'yield from' requires a source argument", 376 pos=pos, fatal=False) 377 arg = None 378 if is_yield_from: 379 return ExprNodes.YieldFromExprNode(pos, arg=arg) 380 else: 381 return ExprNodes.YieldExprNode(pos, arg=arg) 382 383 384def p_yield_statement(s): 385 # s.sy == "yield" 386 yield_expr = p_yield_expression(s) 387 return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr) 388 389 390def p_async_statement(s, ctx, decorators): 391 # s.sy >> 'async' ... 392 if s.sy == 'def': 393 # 'async def' statements aren't allowed in pxd files 394 if 'pxd' in ctx.level: 395 s.error('def statement not allowed here') 396 s.level = ctx.level 397 return p_def_statement(s, decorators, is_async_def=True) 398 elif decorators: 399 s.error("Decorators can only be followed by functions or classes") 400 elif s.sy == 'for': 401 return p_for_statement(s, is_async=True) 402 elif s.sy == 'with': 403 s.next() 404 return p_with_items(s, is_async=True) 405 else: 406 s.error("expected one of 'def', 'for', 'with' after 'async'") 407 408 409#power: atom_expr ('**' factor)* 410#atom_expr: ['await'] atom trailer* 411 412def p_power(s): 413 if s.systring == 'new' and s.peek()[0] == 'IDENT': 414 return p_new_expr(s) 415 await_pos = None 416 if s.sy == 'await': 417 await_pos = s.position() 418 s.next() 419 n1 = p_atom(s) 420 while s.sy in ('(', '[', '.'): 421 n1 = p_trailer(s, n1) 422 if await_pos: 423 n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1) 424 if s.sy == '**': 425 pos = s.position() 426 s.next() 427 n2 = p_factor(s) 428 n1 = ExprNodes.binop_node(pos, '**', n1, n2) 429 return n1 430 431 432def p_new_expr(s): 433 # s.systring == 'new'. 434 pos = s.position() 435 s.next() 436 cppclass = p_c_base_type(s) 437 return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass)) 438 439#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 440 441def p_trailer(s, node1): 442 pos = s.position() 443 if s.sy == '(': 444 return p_call(s, node1) 445 elif s.sy == '[': 446 return p_index(s, node1) 447 else: # s.sy == '.' 448 s.next() 449 name = p_ident(s) 450 return ExprNodes.AttributeNode(pos, 451 obj=node1, attribute=name) 452 453 454# arglist: argument (',' argument)* [','] 455# argument: [test '='] test # Really [keyword '='] test 456 457# since PEP 448: 458# argument: ( test [comp_for] | 459# test '=' test | 460# '**' expr | 461# star_expr ) 462 463def p_call_parse_args(s, allow_genexp=True): 464 # s.sy == '(' 465 pos = s.position() 466 s.next() 467 positional_args = [] 468 keyword_args = [] 469 starstar_seen = False 470 last_was_tuple_unpack = False 471 while s.sy != ')': 472 if s.sy == '*': 473 if starstar_seen: 474 s.error("Non-keyword arg following keyword arg", pos=s.position()) 475 s.next() 476 positional_args.append(p_test(s)) 477 last_was_tuple_unpack = True 478 elif s.sy == '**': 479 s.next() 480 keyword_args.append(p_test(s)) 481 starstar_seen = True 482 else: 483 arg = p_test(s) 484 if s.sy == '=': 485 s.next() 486 if not arg.is_name: 487 s.error("Expected an identifier before '='", 488 pos=arg.pos) 489 encoded_name = s.context.intern_ustring(arg.name) 490 keyword = ExprNodes.IdentifierStringNode( 491 arg.pos, value=encoded_name) 492 arg = p_test(s) 493 keyword_args.append((keyword, arg)) 494 else: 495 if keyword_args: 496 s.error("Non-keyword arg following keyword arg", pos=arg.pos) 497 if positional_args and not last_was_tuple_unpack: 498 positional_args[-1].append(arg) 499 else: 500 positional_args.append([arg]) 501 last_was_tuple_unpack = False 502 if s.sy != ',': 503 break 504 s.next() 505 506 if s.sy in ('for', 'async'): 507 if not keyword_args and not last_was_tuple_unpack: 508 if len(positional_args) == 1 and len(positional_args[0]) == 1: 509 positional_args = [[p_genexp(s, positional_args[0][0])]] 510 s.expect(')') 511 return positional_args or [[]], keyword_args 512 513 514def p_call_build_packed_args(pos, positional_args, keyword_args): 515 keyword_dict = None 516 517 subtuples = [ 518 ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg) 519 for arg in positional_args 520 ] 521 # TODO: implement a faster way to join tuples than creating each one and adding them 522 arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples) 523 524 if keyword_args: 525 kwargs = [] 526 dict_items = [] 527 for item in keyword_args: 528 if isinstance(item, tuple): 529 key, value = item 530 dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)) 531 elif item.is_dict_literal: 532 # unpack "**{a:b}" directly 533 dict_items.extend(item.key_value_pairs) 534 else: 535 if dict_items: 536 kwargs.append(ExprNodes.DictNode( 537 dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True)) 538 dict_items = [] 539 kwargs.append(item) 540 541 if dict_items: 542 kwargs.append(ExprNodes.DictNode( 543 dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True)) 544 545 if kwargs: 546 if len(kwargs) == 1 and kwargs[0].is_dict_literal: 547 # only simple keyword arguments found -> one dict 548 keyword_dict = kwargs[0] 549 else: 550 # at least one **kwargs 551 keyword_dict = ExprNodes.MergedDictNode(pos, keyword_args=kwargs) 552 553 return arg_tuple, keyword_dict 554 555 556def p_call(s, function): 557 # s.sy == '(' 558 pos = s.position() 559 positional_args, keyword_args = p_call_parse_args(s) 560 561 if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list): 562 return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0]) 563 else: 564 arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args) 565 return ExprNodes.GeneralCallNode( 566 pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict) 567 568 569#lambdef: 'lambda' [varargslist] ':' test 570 571#subscriptlist: subscript (',' subscript)* [','] 572 573def p_index(s, base): 574 # s.sy == '[' 575 pos = s.position() 576 s.next() 577 subscripts, is_single_value = p_subscript_list(s) 578 if is_single_value and len(subscripts[0]) == 2: 579 start, stop = subscripts[0] 580 result = ExprNodes.SliceIndexNode(pos, 581 base = base, start = start, stop = stop) 582 else: 583 indexes = make_slice_nodes(pos, subscripts) 584 if is_single_value: 585 index = indexes[0] 586 else: 587 index = ExprNodes.TupleNode(pos, args = indexes) 588 result = ExprNodes.IndexNode(pos, 589 base = base, index = index) 590 s.expect(']') 591 return result 592 593def p_subscript_list(s): 594 is_single_value = True 595 items = [p_subscript(s)] 596 while s.sy == ',': 597 is_single_value = False 598 s.next() 599 if s.sy == ']': 600 break 601 items.append(p_subscript(s)) 602 return items, is_single_value 603 604#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]] 605 606def p_subscript(s): 607 # Parse a subscript and return a list of 608 # 1, 2 or 3 ExprNodes, depending on how 609 # many slice elements were encountered. 610 pos = s.position() 611 start = p_slice_element(s, (':',)) 612 if s.sy != ':': 613 return [start] 614 s.next() 615 stop = p_slice_element(s, (':', ',', ']')) 616 if s.sy != ':': 617 return [start, stop] 618 s.next() 619 step = p_slice_element(s, (':', ',', ']')) 620 return [start, stop, step] 621 622def p_slice_element(s, follow_set): 623 # Simple expression which may be missing iff 624 # it is followed by something in follow_set. 625 if s.sy not in follow_set: 626 return p_test(s) 627 else: 628 return None 629 630def expect_ellipsis(s): 631 s.expect('.') 632 s.expect('.') 633 s.expect('.') 634 635def make_slice_nodes(pos, subscripts): 636 # Convert a list of subscripts as returned 637 # by p_subscript_list into a list of ExprNodes, 638 # creating SliceNodes for elements with 2 or 639 # more components. 640 result = [] 641 for subscript in subscripts: 642 if len(subscript) == 1: 643 result.append(subscript[0]) 644 else: 645 result.append(make_slice_node(pos, *subscript)) 646 return result 647 648def make_slice_node(pos, start, stop = None, step = None): 649 if not start: 650 start = ExprNodes.NoneNode(pos) 651 if not stop: 652 stop = ExprNodes.NoneNode(pos) 653 if not step: 654 step = ExprNodes.NoneNode(pos) 655 return ExprNodes.SliceNode(pos, 656 start = start, stop = stop, step = step) 657 658#atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+ 659 660def p_atom(s): 661 pos = s.position() 662 sy = s.sy 663 if sy == '(': 664 s.next() 665 if s.sy == ')': 666 result = ExprNodes.TupleNode(pos, args = []) 667 elif s.sy == 'yield': 668 result = p_yield_expression(s) 669 else: 670 result = p_testlist_comp(s) 671 s.expect(')') 672 return result 673 elif sy == '[': 674 return p_list_maker(s) 675 elif sy == '{': 676 return p_dict_or_set_maker(s) 677 elif sy == '`': 678 return p_backquote_expr(s) 679 elif sy == '.': 680 expect_ellipsis(s) 681 return ExprNodes.EllipsisNode(pos) 682 elif sy == 'INT': 683 return p_int_literal(s) 684 elif sy == 'FLOAT': 685 value = s.systring 686 s.next() 687 return ExprNodes.FloatNode(pos, value = value) 688 elif sy == 'IMAG': 689 value = s.systring[:-1] 690 s.next() 691 return ExprNodes.ImagNode(pos, value = value) 692 elif sy == 'BEGIN_STRING': 693 kind, bytes_value, unicode_value = p_cat_string_literal(s) 694 if kind == 'c': 695 return ExprNodes.CharNode(pos, value = bytes_value) 696 elif kind == 'u': 697 return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value) 698 elif kind == 'b': 699 return ExprNodes.BytesNode(pos, value = bytes_value) 700 elif kind == 'f': 701 return ExprNodes.JoinedStrNode(pos, values = unicode_value) 702 elif kind == '': 703 return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value) 704 else: 705 s.error("invalid string kind '%s'" % kind) 706 elif sy == 'IDENT': 707 name = s.systring 708 if name == "None": 709 result = ExprNodes.NoneNode(pos) 710 elif name == "True": 711 result = ExprNodes.BoolNode(pos, value=True) 712 elif name == "False": 713 result = ExprNodes.BoolNode(pos, value=False) 714 elif name == "NULL" and not s.in_python_file: 715 result = ExprNodes.NullNode(pos) 716 else: 717 result = p_name(s, name) 718 s.next() 719 return result 720 else: 721 s.error("Expected an identifier or literal") 722 723def p_int_literal(s): 724 pos = s.position() 725 value = s.systring 726 s.next() 727 unsigned = "" 728 longness = "" 729 while value[-1] in u"UuLl": 730 if value[-1] in u"Ll": 731 longness += "L" 732 else: 733 unsigned += "U" 734 value = value[:-1] 735 # '3L' is ambiguous in Py2 but not in Py3. '3U' and '3LL' are 736 # illegal in Py2 Python files. All suffixes are illegal in Py3 737 # Python files. 738 is_c_literal = None 739 if unsigned: 740 is_c_literal = True 741 elif longness: 742 if longness == 'LL' or s.context.language_level >= 3: 743 is_c_literal = True 744 if s.in_python_file: 745 if is_c_literal: 746 error(pos, "illegal integer literal syntax in Python source file") 747 is_c_literal = False 748 return ExprNodes.IntNode(pos, 749 is_c_literal = is_c_literal, 750 value = value, 751 unsigned = unsigned, 752 longness = longness) 753 754 755def p_name(s, name): 756 pos = s.position() 757 if not s.compile_time_expr and name in s.compile_time_env: 758 value = s.compile_time_env.lookup_here(name) 759 node = wrap_compile_time_constant(pos, value) 760 if node is not None: 761 return node 762 return ExprNodes.NameNode(pos, name=name) 763 764 765def wrap_compile_time_constant(pos, value): 766 rep = repr(value) 767 if value is None: 768 return ExprNodes.NoneNode(pos) 769 elif value is Ellipsis: 770 return ExprNodes.EllipsisNode(pos) 771 elif isinstance(value, bool): 772 return ExprNodes.BoolNode(pos, value=value) 773 elif isinstance(value, int): 774 return ExprNodes.IntNode(pos, value=rep, constant_result=value) 775 elif isinstance(value, float): 776 return ExprNodes.FloatNode(pos, value=rep, constant_result=value) 777 elif isinstance(value, complex): 778 node = ExprNodes.ImagNode(pos, value=repr(value.imag), constant_result=complex(0.0, value.imag)) 779 if value.real: 780 # FIXME: should we care about -0.0 ? 781 # probably not worth using the '-' operator for negative imag values 782 node = ExprNodes.binop_node( 783 pos, '+', ExprNodes.FloatNode(pos, value=repr(value.real), constant_result=value.real), node, 784 constant_result=value) 785 return node 786 elif isinstance(value, _unicode): 787 return ExprNodes.UnicodeNode(pos, value=EncodedString(value)) 788 elif isinstance(value, _bytes): 789 bvalue = bytes_literal(value, 'ascii') # actually: unknown encoding, but BytesLiteral requires one 790 return ExprNodes.BytesNode(pos, value=bvalue, constant_result=value) 791 elif isinstance(value, tuple): 792 args = [wrap_compile_time_constant(pos, arg) 793 for arg in value] 794 if None not in args: 795 return ExprNodes.TupleNode(pos, args=args) 796 else: 797 # error already reported 798 return None 799 elif not _IS_PY3 and isinstance(value, long): 800 return ExprNodes.IntNode(pos, value=rep.rstrip('L'), constant_result=value) 801 error(pos, "Invalid type for compile-time constant: %r (type %s)" 802 % (value, value.__class__.__name__)) 803 return None 804 805 806def p_cat_string_literal(s): 807 # A sequence of one or more adjacent string literals. 808 # Returns (kind, bytes_value, unicode_value) 809 # where kind in ('b', 'c', 'u', 'f', '') 810 pos = s.position() 811 kind, bytes_value, unicode_value = p_string_literal(s) 812 if kind == 'c' or s.sy != 'BEGIN_STRING': 813 return kind, bytes_value, unicode_value 814 bstrings, ustrings, positions = [bytes_value], [unicode_value], [pos] 815 bytes_value = unicode_value = None 816 while s.sy == 'BEGIN_STRING': 817 pos = s.position() 818 next_kind, next_bytes_value, next_unicode_value = p_string_literal(s) 819 if next_kind == 'c': 820 error(pos, "Cannot concatenate char literal with another string or char literal") 821 continue 822 elif next_kind != kind: 823 # concatenating f strings and normal strings is allowed and leads to an f string 824 if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])): 825 kind = 'f' 826 else: 827 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % ( 828 kind, next_kind)) 829 continue 830 bstrings.append(next_bytes_value) 831 ustrings.append(next_unicode_value) 832 positions.append(pos) 833 # join and rewrap the partial literals 834 if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings: 835 # Py3 enforced unicode literals are parsed as bytes/unicode combination 836 bytes_value = bytes_literal(StringEncoding.join_bytes(bstrings), s.source_encoding) 837 if kind in ('u', ''): 838 unicode_value = EncodedString(u''.join([u for u in ustrings if u is not None])) 839 if kind == 'f': 840 unicode_value = [] 841 for u, pos in zip(ustrings, positions): 842 if isinstance(u, list): 843 unicode_value += u 844 else: 845 # non-f-string concatenated into the f-string 846 unicode_value.append(ExprNodes.UnicodeNode(pos, value=EncodedString(u))) 847 return kind, bytes_value, unicode_value 848 849 850def p_opt_string_literal(s, required_type='u'): 851 if s.sy != 'BEGIN_STRING': 852 return None 853 pos = s.position() 854 kind, bytes_value, unicode_value = p_string_literal(s, required_type) 855 if required_type == 'u': 856 if kind == 'f': 857 s.error("f-string not allowed here", pos) 858 return unicode_value 859 elif required_type == 'b': 860 return bytes_value 861 else: 862 s.error("internal parser configuration error") 863 864 865def check_for_non_ascii_characters(string): 866 for c in string: 867 if c >= u'\x80': 868 return True 869 return False 870 871 872def p_string_literal(s, kind_override=None): 873 # A single string or char literal. Returns (kind, bvalue, uvalue) 874 # where kind in ('b', 'c', 'u', 'f', ''). The 'bvalue' is the source 875 # code byte sequence of the string literal, 'uvalue' is the 876 # decoded Unicode string. Either of the two may be None depending 877 # on the 'kind' of string, only unprefixed strings have both 878 # representations. In f-strings, the uvalue is a list of the Unicode 879 # strings and f-string expressions that make up the f-string. 880 881 # s.sy == 'BEGIN_STRING' 882 pos = s.position() 883 is_python3_source = s.context.language_level >= 3 884 has_non_ascii_literal_characters = False 885 string_start_pos = (pos[0], pos[1], pos[2] + len(s.systring)) 886 kind_string = s.systring.rstrip('"\'').lower() 887 if len(kind_string) > 1: 888 if len(set(kind_string)) != len(kind_string): 889 error(pos, 'Duplicate string prefix character') 890 if 'b' in kind_string and 'u' in kind_string: 891 error(pos, 'String prefixes b and u cannot be combined') 892 if 'b' in kind_string and 'f' in kind_string: 893 error(pos, 'String prefixes b and f cannot be combined') 894 if 'u' in kind_string and 'f' in kind_string: 895 error(pos, 'String prefixes u and f cannot be combined') 896 897 is_raw = 'r' in kind_string 898 899 if 'c' in kind_string: 900 # this should never happen, since the lexer does not allow combining c 901 # with other prefix characters 902 if len(kind_string) != 1: 903 error(pos, 'Invalid string prefix for character literal') 904 kind = 'c' 905 elif 'f' in kind_string: 906 kind = 'f' # u is ignored 907 is_raw = True # postpone the escape resolution 908 elif 'b' in kind_string: 909 kind = 'b' 910 elif 'u' in kind_string: 911 kind = 'u' 912 else: 913 kind = '' 914 915 if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives: 916 chars = StringEncoding.StrLiteralBuilder(s.source_encoding) 917 kind = 'u' 918 else: 919 if kind_override is not None and kind_override in 'ub': 920 kind = kind_override 921 if kind in ('u', 'f'): # f-strings are scanned exactly like Unicode literals, but are parsed further later 922 chars = StringEncoding.UnicodeLiteralBuilder() 923 elif kind == '': 924 chars = StringEncoding.StrLiteralBuilder(s.source_encoding) 925 else: 926 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding) 927 928 while 1: 929 s.next() 930 sy = s.sy 931 systr = s.systring 932 # print "p_string_literal: sy =", sy, repr(s.systring) ### 933 if sy == 'CHARS': 934 chars.append(systr) 935 if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr): 936 has_non_ascii_literal_characters = True 937 elif sy == 'ESCAPE': 938 # in Py2, 'ur' raw unicode strings resolve unicode escapes but nothing else 939 if is_raw and (is_python3_source or kind != 'u' or systr[1] not in u'Uu'): 940 chars.append(systr) 941 if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr): 942 has_non_ascii_literal_characters = True 943 else: 944 _append_escape_sequence(kind, chars, systr, s) 945 elif sy == 'NEWLINE': 946 chars.append(u'\n') 947 elif sy == 'END_STRING': 948 break 949 elif sy == 'EOF': 950 s.error("Unclosed string literal", pos=pos) 951 else: 952 s.error("Unexpected token %r:%r in string literal" % ( 953 sy, s.systring)) 954 955 if kind == 'c': 956 unicode_value = None 957 bytes_value = chars.getchar() 958 if len(bytes_value) != 1: 959 error(pos, u"invalid character literal: %r" % bytes_value) 960 else: 961 bytes_value, unicode_value = chars.getstrings() 962 if (has_non_ascii_literal_characters 963 and is_python3_source and Future.unicode_literals in s.context.future_directives): 964 # Python 3 forbids literal non-ASCII characters in byte strings 965 if kind == 'b': 966 s.error("bytes can only contain ASCII literal characters.", pos=pos) 967 bytes_value = None 968 if kind == 'f': 969 unicode_value = p_f_string(s, unicode_value, string_start_pos, is_raw='r' in kind_string) 970 s.next() 971 return (kind, bytes_value, unicode_value) 972 973 974def _append_escape_sequence(kind, builder, escape_sequence, s): 975 c = escape_sequence[1] 976 if c in u"01234567": 977 builder.append_charval(int(escape_sequence[1:], 8)) 978 elif c in u"'\"\\": 979 builder.append(c) 980 elif c in u"abfnrtv": 981 builder.append(StringEncoding.char_from_escape_sequence(escape_sequence)) 982 elif c == u'\n': 983 pass # line continuation 984 elif c == u'x': # \xXX 985 if len(escape_sequence) == 4: 986 builder.append_charval(int(escape_sequence[2:], 16)) 987 else: 988 s.error("Invalid hex escape '%s'" % escape_sequence, fatal=False) 989 elif c in u'NUu' and kind in ('u', 'f', ''): # \uxxxx, \Uxxxxxxxx, \N{...} 990 chrval = -1 991 if c == u'N': 992 uchar = None 993 try: 994 uchar = lookup_unicodechar(escape_sequence[3:-1]) 995 chrval = ord(uchar) 996 except KeyError: 997 s.error("Unknown Unicode character name %s" % 998 repr(escape_sequence[3:-1]).lstrip('u'), fatal=False) 999 except TypeError: 1000 # 2-byte unicode build of CPython? 1001 if (uchar is not None and _IS_2BYTE_UNICODE and len(uchar) == 2 and 1002 unicode_category(uchar[0]) == 'Cs' and unicode_category(uchar[1]) == 'Cs'): 1003 # surrogate pair instead of single character 1004 chrval = 0x10000 + (ord(uchar[0]) - 0xd800) >> 10 + (ord(uchar[1]) - 0xdc00) 1005 else: 1006 raise 1007 elif len(escape_sequence) in (6, 10): 1008 chrval = int(escape_sequence[2:], 16) 1009 if chrval > 1114111: # sys.maxunicode: 1010 s.error("Invalid unicode escape '%s'" % escape_sequence) 1011 chrval = -1 1012 else: 1013 s.error("Invalid unicode escape '%s'" % escape_sequence, fatal=False) 1014 if chrval >= 0: 1015 builder.append_uescape(chrval, escape_sequence) 1016 else: 1017 builder.append(escape_sequence) 1018 1019 1020_parse_escape_sequences_raw, _parse_escape_sequences = [re.compile(( 1021 # escape sequences: 1022 br'(\\(?:' + 1023 (br'\\?' if is_raw else ( 1024 br'[\\abfnrtv"\'{]|' 1025 br'[0-7]{2,3}|' 1026 br'N\{[^}]*\}|' 1027 br'x[0-9a-fA-F]{2}|' 1028 br'u[0-9a-fA-F]{4}|' 1029 br'U[0-9a-fA-F]{8}|' 1030 br'[NxuU]|' # detect invalid escape sequences that do not match above 1031 )) + 1032 br')?|' 1033 # non-escape sequences: 1034 br'\{\{?|' 1035 br'\}\}?|' 1036 br'[^\\{}]+)' 1037 ).decode('us-ascii')).match 1038 for is_raw in (True, False)] 1039 1040 1041def _f_string_error_pos(pos, string, i): 1042 return (pos[0], pos[1], pos[2] + i + 1) # FIXME: handle newlines in string 1043 1044 1045def p_f_string(s, unicode_value, pos, is_raw): 1046 # Parses a PEP 498 f-string literal into a list of nodes. Nodes are either UnicodeNodes 1047 # or FormattedValueNodes. 1048 values = [] 1049 next_start = 0 1050 size = len(unicode_value) 1051 builder = StringEncoding.UnicodeLiteralBuilder() 1052 _parse_seq = _parse_escape_sequences_raw if is_raw else _parse_escape_sequences 1053 1054 while next_start < size: 1055 end = next_start 1056 match = _parse_seq(unicode_value, next_start) 1057 if match is None: 1058 error(_f_string_error_pos(pos, unicode_value, next_start), "Invalid escape sequence") 1059 1060 next_start = match.end() 1061 part = match.group() 1062 c = part[0] 1063 if c == '\\': 1064 if not is_raw and len(part) > 1: 1065 _append_escape_sequence('f', builder, part, s) 1066 else: 1067 builder.append(part) 1068 elif c == '{': 1069 if part == '{{': 1070 builder.append('{') 1071 else: 1072 # start of an expression 1073 if builder.chars: 1074 values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring())) 1075 builder = StringEncoding.UnicodeLiteralBuilder() 1076 next_start, expr_node = p_f_string_expr(s, unicode_value, pos, next_start, is_raw) 1077 values.append(expr_node) 1078 elif c == '}': 1079 if part == '}}': 1080 builder.append('}') 1081 else: 1082 error(_f_string_error_pos(pos, unicode_value, end), 1083 "f-string: single '}' is not allowed") 1084 else: 1085 builder.append(part) 1086 1087 if builder.chars: 1088 values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring())) 1089 return values 1090 1091 1092def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): 1093 # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode 1094 # and the index in the string that follows the expression. 1095 i = starting_index 1096 size = len(unicode_value) 1097 conversion_char = terminal_char = format_spec = None 1098 format_spec_str = None 1099 NO_CHAR = 2**30 1100 1101 nested_depth = 0 1102 quote_char = NO_CHAR 1103 in_triple_quotes = False 1104 backslash_reported = False 1105 1106 while True: 1107 if i >= size: 1108 break # error will be reported below 1109 c = unicode_value[i] 1110 1111 if quote_char != NO_CHAR: 1112 if c == '\\': 1113 # avoid redundant error reports along '\' sequences 1114 if not backslash_reported: 1115 error(_f_string_error_pos(pos, unicode_value, i), 1116 "backslashes not allowed in f-strings") 1117 backslash_reported = True 1118 elif c == quote_char: 1119 if in_triple_quotes: 1120 if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c: 1121 in_triple_quotes = False 1122 quote_char = NO_CHAR 1123 i += 2 1124 else: 1125 quote_char = NO_CHAR 1126 elif c in '\'"': 1127 quote_char = c 1128 if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c: 1129 in_triple_quotes = True 1130 i += 2 1131 elif c in '{[(': 1132 nested_depth += 1 1133 elif nested_depth != 0 and c in '}])': 1134 nested_depth -= 1 1135 elif c == '#': 1136 error(_f_string_error_pos(pos, unicode_value, i), 1137 "format string cannot include #") 1138 elif nested_depth == 0 and c in '!:}': 1139 # allow != as a special case 1140 if c == '!' and i + 1 < size and unicode_value[i + 1] == '=': 1141 i += 1 1142 continue 1143 1144 terminal_char = c 1145 break 1146 i += 1 1147 1148 # normalise line endings as the parser expects that 1149 expr_str = unicode_value[starting_index:i].replace('\r\n', '\n').replace('\r', '\n') 1150 expr_pos = (pos[0], pos[1], pos[2] + starting_index + 2) # TODO: find exact code position (concat, multi-line, ...) 1151 1152 if not expr_str.strip(): 1153 error(_f_string_error_pos(pos, unicode_value, starting_index), 1154 "empty expression not allowed in f-string") 1155 1156 if terminal_char == '!': 1157 i += 1 1158 if i + 2 > size: 1159 pass # error will be reported below 1160 else: 1161 conversion_char = unicode_value[i] 1162 i += 1 1163 terminal_char = unicode_value[i] 1164 1165 if terminal_char == ':': 1166 in_triple_quotes = False 1167 in_string = False 1168 nested_depth = 0 1169 start_format_spec = i + 1 1170 while True: 1171 if i >= size: 1172 break # error will be reported below 1173 c = unicode_value[i] 1174 if not in_triple_quotes and not in_string: 1175 if c == '{': 1176 nested_depth += 1 1177 elif c == '}': 1178 if nested_depth > 0: 1179 nested_depth -= 1 1180 else: 1181 terminal_char = c 1182 break 1183 if c in '\'"': 1184 if not in_string and i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c: 1185 in_triple_quotes = not in_triple_quotes 1186 i += 2 1187 elif not in_triple_quotes: 1188 in_string = not in_string 1189 i += 1 1190 1191 format_spec_str = unicode_value[start_format_spec:i] 1192 1193 if terminal_char != '}': 1194 error(_f_string_error_pos(pos, unicode_value, i), 1195 "missing '}' in format string expression" + ( 1196 ", found '%s'" % terminal_char if terminal_char else "")) 1197 1198 # parse the expression as if it was surrounded by parentheses 1199 buf = StringIO('(%s)' % expr_str) 1200 scanner = PyrexScanner(buf, expr_pos[0], parent_scanner=s, source_encoding=s.source_encoding, initial_pos=expr_pos) 1201 expr = p_testlist(scanner) # TODO is testlist right here? 1202 1203 # validate the conversion char 1204 if conversion_char is not None and not ExprNodes.FormattedValueNode.find_conversion_func(conversion_char): 1205 error(expr_pos, "invalid conversion character '%s'" % conversion_char) 1206 1207 # the format spec is itself treated like an f-string 1208 if format_spec_str: 1209 format_spec = ExprNodes.JoinedStrNode(pos, values=p_f_string(s, format_spec_str, pos, is_raw)) 1210 1211 return i + 1, ExprNodes.FormattedValueNode( 1212 pos, value=expr, conversion_char=conversion_char, format_spec=format_spec) 1213 1214 1215# since PEP 448: 1216# list_display ::= "[" [listmaker] "]" 1217# listmaker ::= (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 1218# comp_iter ::= comp_for | comp_if 1219# comp_for ::= ["async"] "for" expression_list "in" testlist [comp_iter] 1220# comp_if ::= "if" test [comp_iter] 1221 1222def p_list_maker(s): 1223 # s.sy == '[' 1224 pos = s.position() 1225 s.next() 1226 if s.sy == ']': 1227 s.expect(']') 1228 return ExprNodes.ListNode(pos, args=[]) 1229 1230 expr = p_test_or_starred_expr(s) 1231 if s.sy in ('for', 'async'): 1232 if expr.is_starred: 1233 s.error("iterable unpacking cannot be used in comprehension") 1234 append = ExprNodes.ComprehensionAppendNode(pos, expr=expr) 1235 loop = p_comp_for(s, append) 1236 s.expect(']') 1237 return ExprNodes.ComprehensionNode( 1238 pos, loop=loop, append=append, type=Builtin.list_type, 1239 # list comprehensions leak their loop variable in Py2 1240 has_local_scope=s.context.language_level >= 3) 1241 1242 # (merged) list literal 1243 if s.sy == ',': 1244 s.next() 1245 exprs = p_test_or_starred_expr_list(s, expr) 1246 else: 1247 exprs = [expr] 1248 s.expect(']') 1249 return ExprNodes.ListNode(pos, args=exprs) 1250 1251 1252def p_comp_iter(s, body): 1253 if s.sy in ('for', 'async'): 1254 return p_comp_for(s, body) 1255 elif s.sy == 'if': 1256 return p_comp_if(s, body) 1257 else: 1258 # insert the 'append' operation into the loop 1259 return body 1260 1261def p_comp_for(s, body): 1262 pos = s.position() 1263 # [async] for ... 1264 is_async = False 1265 if s.sy == 'async': 1266 is_async = True 1267 s.next() 1268 1269 # s.sy == 'for' 1270 s.expect('for') 1271 kw = p_for_bounds(s, allow_testlist=False, is_async=is_async) 1272 kw.update(else_clause=None, body=p_comp_iter(s, body), is_async=is_async) 1273 return Nodes.ForStatNode(pos, **kw) 1274 1275def p_comp_if(s, body): 1276 # s.sy == 'if' 1277 pos = s.position() 1278 s.next() 1279 test = p_test_nocond(s) 1280 return Nodes.IfStatNode(pos, 1281 if_clauses = [Nodes.IfClauseNode(pos, condition = test, 1282 body = p_comp_iter(s, body))], 1283 else_clause = None ) 1284 1285 1286# since PEP 448: 1287#dictorsetmaker: ( ((test ':' test | '**' expr) 1288# (comp_for | (',' (test ':' test | '**' expr))* [','])) | 1289# ((test | star_expr) 1290# (comp_for | (',' (test | star_expr))* [','])) ) 1291 1292def p_dict_or_set_maker(s): 1293 # s.sy == '{' 1294 pos = s.position() 1295 s.next() 1296 if s.sy == '}': 1297 s.next() 1298 return ExprNodes.DictNode(pos, key_value_pairs=[]) 1299 1300 parts = [] 1301 target_type = 0 1302 last_was_simple_item = False 1303 while True: 1304 if s.sy in ('*', '**'): 1305 # merged set/dict literal 1306 if target_type == 0: 1307 target_type = 1 if s.sy == '*' else 2 # 'stars' 1308 elif target_type != len(s.sy): 1309 s.error("unexpected %sitem found in %s literal" % ( 1310 s.sy, 'set' if target_type == 1 else 'dict')) 1311 s.next() 1312 if s.sy == '*': 1313 s.error("expected expression, found '*'") 1314 item = p_starred_expr(s) 1315 parts.append(item) 1316 last_was_simple_item = False 1317 else: 1318 item = p_test(s) 1319 if target_type == 0: 1320 target_type = 2 if s.sy == ':' else 1 # dict vs. set 1321 if target_type == 2: 1322 # dict literal 1323 s.expect(':') 1324 key = item 1325 value = p_test(s) 1326 item = ExprNodes.DictItemNode(key.pos, key=key, value=value) 1327 if last_was_simple_item: 1328 parts[-1].append(item) 1329 else: 1330 parts.append([item]) 1331 last_was_simple_item = True 1332 1333 if s.sy == ',': 1334 s.next() 1335 if s.sy == '}': 1336 break 1337 else: 1338 break 1339 1340 if s.sy in ('for', 'async'): 1341 # dict/set comprehension 1342 if len(parts) == 1 and isinstance(parts[0], list) and len(parts[0]) == 1: 1343 item = parts[0][0] 1344 if target_type == 2: 1345 assert isinstance(item, ExprNodes.DictItemNode), type(item) 1346 comprehension_type = Builtin.dict_type 1347 append = ExprNodes.DictComprehensionAppendNode( 1348 item.pos, key_expr=item.key, value_expr=item.value) 1349 else: 1350 comprehension_type = Builtin.set_type 1351 append = ExprNodes.ComprehensionAppendNode(item.pos, expr=item) 1352 loop = p_comp_for(s, append) 1353 s.expect('}') 1354 return ExprNodes.ComprehensionNode(pos, loop=loop, append=append, type=comprehension_type) 1355 else: 1356 # syntax error, try to find a good error message 1357 if len(parts) == 1 and not isinstance(parts[0], list): 1358 s.error("iterable unpacking cannot be used in comprehension") 1359 else: 1360 # e.g. "{1,2,3 for ..." 1361 s.expect('}') 1362 return ExprNodes.DictNode(pos, key_value_pairs=[]) 1363 1364 s.expect('}') 1365 if target_type == 1: 1366 # (merged) set literal 1367 items = [] 1368 set_items = [] 1369 for part in parts: 1370 if isinstance(part, list): 1371 set_items.extend(part) 1372 else: 1373 if set_items: 1374 items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items)) 1375 set_items = [] 1376 items.append(part) 1377 if set_items: 1378 items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items)) 1379 if len(items) == 1 and items[0].is_set_literal: 1380 return items[0] 1381 return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type) 1382 else: 1383 # (merged) dict literal 1384 items = [] 1385 dict_items = [] 1386 for part in parts: 1387 if isinstance(part, list): 1388 dict_items.extend(part) 1389 else: 1390 if dict_items: 1391 items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items)) 1392 dict_items = [] 1393 items.append(part) 1394 if dict_items: 1395 items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items)) 1396 if len(items) == 1 and items[0].is_dict_literal: 1397 return items[0] 1398 return ExprNodes.MergedDictNode(pos, keyword_args=items, reject_duplicates=False) 1399 1400 1401# NOTE: no longer in Py3 :) 1402def p_backquote_expr(s): 1403 # s.sy == '`' 1404 pos = s.position() 1405 s.next() 1406 args = [p_test(s)] 1407 while s.sy == ',': 1408 s.next() 1409 args.append(p_test(s)) 1410 s.expect('`') 1411 if len(args) == 1: 1412 arg = args[0] 1413 else: 1414 arg = ExprNodes.TupleNode(pos, args = args) 1415 return ExprNodes.BackquoteNode(pos, arg = arg) 1416 1417def p_simple_expr_list(s, expr=None): 1418 exprs = expr is not None and [expr] or [] 1419 while s.sy not in expr_terminators: 1420 exprs.append( p_test(s) ) 1421 if s.sy != ',': 1422 break 1423 s.next() 1424 return exprs 1425 1426 1427def p_test_or_starred_expr_list(s, expr=None): 1428 exprs = expr is not None and [expr] or [] 1429 while s.sy not in expr_terminators: 1430 exprs.append(p_test_or_starred_expr(s)) 1431 if s.sy != ',': 1432 break 1433 s.next() 1434 return exprs 1435 1436 1437#testlist: test (',' test)* [','] 1438 1439def p_testlist(s): 1440 pos = s.position() 1441 expr = p_test(s) 1442 if s.sy == ',': 1443 s.next() 1444 exprs = p_simple_expr_list(s, expr) 1445 return ExprNodes.TupleNode(pos, args = exprs) 1446 else: 1447 return expr 1448 1449# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 1450 1451def p_testlist_star_expr(s): 1452 pos = s.position() 1453 expr = p_test_or_starred_expr(s) 1454 if s.sy == ',': 1455 s.next() 1456 exprs = p_test_or_starred_expr_list(s, expr) 1457 return ExprNodes.TupleNode(pos, args = exprs) 1458 else: 1459 return expr 1460 1461# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 1462 1463def p_testlist_comp(s): 1464 pos = s.position() 1465 expr = p_test_or_starred_expr(s) 1466 if s.sy == ',': 1467 s.next() 1468 exprs = p_test_or_starred_expr_list(s, expr) 1469 return ExprNodes.TupleNode(pos, args = exprs) 1470 elif s.sy in ('for', 'async'): 1471 return p_genexp(s, expr) 1472 else: 1473 return expr 1474 1475def p_genexp(s, expr): 1476 # s.sy == 'async' | 'for' 1477 loop = p_comp_for(s, Nodes.ExprStatNode( 1478 expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr))) 1479 return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop) 1480 1481expr_terminators = cython.declare(set, set([ 1482 ')', ']', '}', ':', '=', 'NEWLINE'])) 1483 1484 1485#------------------------------------------------------- 1486# 1487# Statements 1488# 1489#------------------------------------------------------- 1490 1491def p_global_statement(s): 1492 # assume s.sy == 'global' 1493 pos = s.position() 1494 s.next() 1495 names = p_ident_list(s) 1496 return Nodes.GlobalNode(pos, names = names) 1497 1498 1499def p_nonlocal_statement(s): 1500 pos = s.position() 1501 s.next() 1502 names = p_ident_list(s) 1503 return Nodes.NonlocalNode(pos, names = names) 1504 1505 1506def p_expression_or_assignment(s): 1507 expr = p_testlist_star_expr(s) 1508 if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute): 1509 s.next() 1510 expr.annotation = p_test(s) 1511 if s.sy == '=' and expr.is_starred: 1512 # This is a common enough error to make when learning Cython to let 1513 # it fail as early as possible and give a very clear error message. 1514 s.error("a starred assignment target must be in a list or tuple" 1515 " - maybe you meant to use an index assignment: var[0] = ...", 1516 pos=expr.pos) 1517 expr_list = [expr] 1518 while s.sy == '=': 1519 s.next() 1520 if s.sy == 'yield': 1521 expr = p_yield_expression(s) 1522 else: 1523 expr = p_testlist_star_expr(s) 1524 expr_list.append(expr) 1525 if len(expr_list) == 1: 1526 if re.match(r"([-+*/%^&|]|<<|>>|\*\*|//|@)=", s.sy): 1527 lhs = expr_list[0] 1528 if isinstance(lhs, ExprNodes.SliceIndexNode): 1529 # implementation requires IndexNode 1530 lhs = ExprNodes.IndexNode( 1531 lhs.pos, 1532 base=lhs.base, 1533 index=make_slice_node(lhs.pos, lhs.start, lhs.stop)) 1534 elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode)): 1535 error(lhs.pos, "Illegal operand for inplace operation.") 1536 operator = s.sy[:-1] 1537 s.next() 1538 if s.sy == 'yield': 1539 rhs = p_yield_expression(s) 1540 else: 1541 rhs = p_testlist(s) 1542 return Nodes.InPlaceAssignmentNode(lhs.pos, operator=operator, lhs=lhs, rhs=rhs) 1543 expr = expr_list[0] 1544 return Nodes.ExprStatNode(expr.pos, expr=expr) 1545 1546 rhs = expr_list[-1] 1547 if len(expr_list) == 2: 1548 return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs) 1549 else: 1550 return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs) 1551 1552 1553def p_print_statement(s): 1554 # s.sy == 'print' 1555 pos = s.position() 1556 ends_with_comma = 0 1557 s.next() 1558 if s.sy == '>>': 1559 s.next() 1560 stream = p_test(s) 1561 if s.sy == ',': 1562 s.next() 1563 ends_with_comma = s.sy in ('NEWLINE', 'EOF') 1564 else: 1565 stream = None 1566 args = [] 1567 if s.sy not in ('NEWLINE', 'EOF'): 1568 args.append(p_test(s)) 1569 while s.sy == ',': 1570 s.next() 1571 if s.sy in ('NEWLINE', 'EOF'): 1572 ends_with_comma = 1 1573 break 1574 args.append(p_test(s)) 1575 arg_tuple = ExprNodes.TupleNode(pos, args=args) 1576 return Nodes.PrintStatNode(pos, 1577 arg_tuple=arg_tuple, stream=stream, 1578 append_newline=not ends_with_comma) 1579 1580 1581def p_exec_statement(s): 1582 # s.sy == 'exec' 1583 pos = s.position() 1584 s.next() 1585 code = p_bit_expr(s) 1586 if isinstance(code, ExprNodes.TupleNode): 1587 # Py3 compatibility syntax 1588 tuple_variant = True 1589 args = code.args 1590 if len(args) not in (2, 3): 1591 s.error("expected tuple of length 2 or 3, got length %d" % len(args), 1592 pos=pos, fatal=False) 1593 args = [code] 1594 else: 1595 tuple_variant = False 1596 args = [code] 1597 if s.sy == 'in': 1598 if tuple_variant: 1599 s.error("tuple variant of exec does not support additional 'in' arguments", 1600 fatal=False) 1601 s.next() 1602 args.append(p_test(s)) 1603 if s.sy == ',': 1604 s.next() 1605 args.append(p_test(s)) 1606 return Nodes.ExecStatNode(pos, args=args) 1607 1608def p_del_statement(s): 1609 # s.sy == 'del' 1610 pos = s.position() 1611 s.next() 1612 # FIXME: 'exprlist' in Python 1613 args = p_simple_expr_list(s) 1614 return Nodes.DelStatNode(pos, args = args) 1615 1616def p_pass_statement(s, with_newline = 0): 1617 pos = s.position() 1618 s.expect('pass') 1619 if with_newline: 1620 s.expect_newline("Expected a newline", ignore_semicolon=True) 1621 return Nodes.PassStatNode(pos) 1622 1623def p_break_statement(s): 1624 # s.sy == 'break' 1625 pos = s.position() 1626 s.next() 1627 return Nodes.BreakStatNode(pos) 1628 1629def p_continue_statement(s): 1630 # s.sy == 'continue' 1631 pos = s.position() 1632 s.next() 1633 return Nodes.ContinueStatNode(pos) 1634 1635def p_return_statement(s): 1636 # s.sy == 'return' 1637 pos = s.position() 1638 s.next() 1639 if s.sy not in statement_terminators: 1640 value = p_testlist(s) 1641 else: 1642 value = None 1643 return Nodes.ReturnStatNode(pos, value = value) 1644 1645def p_raise_statement(s): 1646 # s.sy == 'raise' 1647 pos = s.position() 1648 s.next() 1649 exc_type = None 1650 exc_value = None 1651 exc_tb = None 1652 cause = None 1653 if s.sy not in statement_terminators: 1654 exc_type = p_test(s) 1655 if s.sy == ',': 1656 s.next() 1657 exc_value = p_test(s) 1658 if s.sy == ',': 1659 s.next() 1660 exc_tb = p_test(s) 1661 elif s.sy == 'from': 1662 s.next() 1663 cause = p_test(s) 1664 if exc_type or exc_value or exc_tb: 1665 return Nodes.RaiseStatNode(pos, 1666 exc_type = exc_type, 1667 exc_value = exc_value, 1668 exc_tb = exc_tb, 1669 cause = cause) 1670 else: 1671 return Nodes.ReraiseStatNode(pos) 1672 1673 1674def p_import_statement(s): 1675 # s.sy in ('import', 'cimport') 1676 pos = s.position() 1677 kind = s.sy 1678 s.next() 1679 items = [p_dotted_name(s, as_allowed=1)] 1680 while s.sy == ',': 1681 s.next() 1682 items.append(p_dotted_name(s, as_allowed=1)) 1683 stats = [] 1684 is_absolute = Future.absolute_import in s.context.future_directives 1685 for pos, target_name, dotted_name, as_name in items: 1686 if kind == 'cimport': 1687 stat = Nodes.CImportStatNode( 1688 pos, 1689 module_name=dotted_name, 1690 as_name=as_name, 1691 is_absolute=is_absolute) 1692 else: 1693 if as_name and "." in dotted_name: 1694 name_list = ExprNodes.ListNode(pos, args=[ 1695 ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))]) 1696 else: 1697 name_list = None 1698 stat = Nodes.SingleAssignmentNode( 1699 pos, 1700 lhs=ExprNodes.NameNode(pos, name=as_name or target_name), 1701 rhs=ExprNodes.ImportNode( 1702 pos, 1703 module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name), 1704 level=0 if is_absolute else None, 1705 name_list=name_list)) 1706 stats.append(stat) 1707 return Nodes.StatListNode(pos, stats=stats) 1708 1709 1710def p_from_import_statement(s, first_statement = 0): 1711 # s.sy == 'from' 1712 pos = s.position() 1713 s.next() 1714 if s.sy == '.': 1715 # count relative import level 1716 level = 0 1717 while s.sy == '.': 1718 level += 1 1719 s.next() 1720 else: 1721 level = None 1722 if level is not None and s.sy in ('import', 'cimport'): 1723 # we are dealing with "from .. import foo, bar" 1724 dotted_name_pos, dotted_name = s.position(), s.context.intern_ustring('') 1725 else: 1726 if level is None and Future.absolute_import in s.context.future_directives: 1727 level = 0 1728 (dotted_name_pos, _, dotted_name, _) = p_dotted_name(s, as_allowed=False) 1729 if s.sy not in ('import', 'cimport'): 1730 s.error("Expected 'import' or 'cimport'") 1731 kind = s.sy 1732 s.next() 1733 1734 is_cimport = kind == 'cimport' 1735 is_parenthesized = False 1736 if s.sy == '*': 1737 imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)] 1738 s.next() 1739 else: 1740 if s.sy == '(': 1741 is_parenthesized = True 1742 s.next() 1743 imported_names = [p_imported_name(s, is_cimport)] 1744 while s.sy == ',': 1745 s.next() 1746 if is_parenthesized and s.sy == ')': 1747 break 1748 imported_names.append(p_imported_name(s, is_cimport)) 1749 if is_parenthesized: 1750 s.expect(')') 1751 if dotted_name == '__future__': 1752 if not first_statement: 1753 s.error("from __future__ imports must occur at the beginning of the file") 1754 elif level: 1755 s.error("invalid syntax") 1756 else: 1757 for (name_pos, name, as_name, kind) in imported_names: 1758 if name == "braces": 1759 s.error("not a chance", name_pos) 1760 break 1761 try: 1762 directive = getattr(Future, name) 1763 except AttributeError: 1764 s.error("future feature %s is not defined" % name, name_pos) 1765 break 1766 s.context.future_directives.add(directive) 1767 return Nodes.PassStatNode(pos) 1768 elif kind == 'cimport': 1769 return Nodes.FromCImportStatNode( 1770 pos, module_name=dotted_name, 1771 relative_level=level, 1772 imported_names=imported_names) 1773 else: 1774 imported_name_strings = [] 1775 items = [] 1776 for (name_pos, name, as_name, kind) in imported_names: 1777 imported_name_strings.append( 1778 ExprNodes.IdentifierStringNode(name_pos, value=name)) 1779 items.append( 1780 (name, ExprNodes.NameNode(name_pos, name=as_name or name))) 1781 import_list = ExprNodes.ListNode( 1782 imported_names[0][0], args=imported_name_strings) 1783 return Nodes.FromImportStatNode(pos, 1784 module = ExprNodes.ImportNode(dotted_name_pos, 1785 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name), 1786 level = level, 1787 name_list = import_list), 1788 items = items) 1789 1790 1791imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union'])) 1792 1793def p_imported_name(s, is_cimport): 1794 pos = s.position() 1795 kind = None 1796 if is_cimport and s.systring in imported_name_kinds: 1797 kind = s.systring 1798 s.next() 1799 name = p_ident(s) 1800 as_name = p_as_name(s) 1801 return (pos, name, as_name, kind) 1802 1803 1804def p_dotted_name(s, as_allowed): 1805 pos = s.position() 1806 target_name = p_ident(s) 1807 as_name = None 1808 names = [target_name] 1809 while s.sy == '.': 1810 s.next() 1811 names.append(p_ident(s)) 1812 if as_allowed: 1813 as_name = p_as_name(s) 1814 return (pos, target_name, s.context.intern_ustring(u'.'.join(names)), as_name) 1815 1816 1817def p_as_name(s): 1818 if s.sy == 'IDENT' and s.systring == 'as': 1819 s.next() 1820 return p_ident(s) 1821 else: 1822 return None 1823 1824 1825def p_assert_statement(s): 1826 # s.sy == 'assert' 1827 pos = s.position() 1828 s.next() 1829 cond = p_test(s) 1830 if s.sy == ',': 1831 s.next() 1832 value = p_test(s) 1833 else: 1834 value = None 1835 return Nodes.AssertStatNode(pos, cond = cond, value = value) 1836 1837 1838statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF'])) 1839 1840def p_if_statement(s): 1841 # s.sy == 'if' 1842 pos = s.position() 1843 s.next() 1844 if_clauses = [p_if_clause(s)] 1845 while s.sy == 'elif': 1846 s.next() 1847 if_clauses.append(p_if_clause(s)) 1848 else_clause = p_else_clause(s) 1849 return Nodes.IfStatNode(pos, 1850 if_clauses = if_clauses, else_clause = else_clause) 1851 1852def p_if_clause(s): 1853 pos = s.position() 1854 test = p_test(s) 1855 body = p_suite(s) 1856 return Nodes.IfClauseNode(pos, 1857 condition = test, body = body) 1858 1859def p_else_clause(s): 1860 if s.sy == 'else': 1861 s.next() 1862 return p_suite(s) 1863 else: 1864 return None 1865 1866def p_while_statement(s): 1867 # s.sy == 'while' 1868 pos = s.position() 1869 s.next() 1870 test = p_test(s) 1871 body = p_suite(s) 1872 else_clause = p_else_clause(s) 1873 return Nodes.WhileStatNode(pos, 1874 condition = test, body = body, 1875 else_clause = else_clause) 1876 1877 1878def p_for_statement(s, is_async=False): 1879 # s.sy == 'for' 1880 pos = s.position() 1881 s.next() 1882 kw = p_for_bounds(s, allow_testlist=True, is_async=is_async) 1883 body = p_suite(s) 1884 else_clause = p_else_clause(s) 1885 kw.update(body=body, else_clause=else_clause, is_async=is_async) 1886 return Nodes.ForStatNode(pos, **kw) 1887 1888 1889def p_for_bounds(s, allow_testlist=True, is_async=False): 1890 target = p_for_target(s) 1891 if s.sy == 'in': 1892 s.next() 1893 iterator = p_for_iterator(s, allow_testlist, is_async=is_async) 1894 return dict(target=target, iterator=iterator) 1895 elif not s.in_python_file and not is_async: 1896 if s.sy == 'from': 1897 s.next() 1898 bound1 = p_bit_expr(s) 1899 else: 1900 # Support shorter "for a <= x < b" syntax 1901 bound1, target = target, None 1902 rel1 = p_for_from_relation(s) 1903 name2_pos = s.position() 1904 name2 = p_ident(s) 1905 rel2_pos = s.position() 1906 rel2 = p_for_from_relation(s) 1907 bound2 = p_bit_expr(s) 1908 step = p_for_from_step(s) 1909 if target is None: 1910 target = ExprNodes.NameNode(name2_pos, name = name2) 1911 else: 1912 if not target.is_name: 1913 error(target.pos, 1914 "Target of for-from statement must be a variable name") 1915 elif name2 != target.name: 1916 error(name2_pos, 1917 "Variable name in for-from range does not match target") 1918 if rel1[0] != rel2[0]: 1919 error(rel2_pos, 1920 "Relation directions in for-from do not match") 1921 return dict(target = target, 1922 bound1 = bound1, 1923 relation1 = rel1, 1924 relation2 = rel2, 1925 bound2 = bound2, 1926 step = step, 1927 ) 1928 else: 1929 s.expect('in') 1930 return {} 1931 1932def p_for_from_relation(s): 1933 if s.sy in inequality_relations: 1934 op = s.sy 1935 s.next() 1936 return op 1937 else: 1938 s.error("Expected one of '<', '<=', '>' '>='") 1939 1940def p_for_from_step(s): 1941 if s.sy == 'IDENT' and s.systring == 'by': 1942 s.next() 1943 step = p_bit_expr(s) 1944 return step 1945 else: 1946 return None 1947 1948inequality_relations = cython.declare(set, set(['<', '<=', '>', '>='])) 1949 1950def p_target(s, terminator): 1951 pos = s.position() 1952 expr = p_starred_expr(s) 1953 if s.sy == ',': 1954 s.next() 1955 exprs = [expr] 1956 while s.sy != terminator: 1957 exprs.append(p_starred_expr(s)) 1958 if s.sy != ',': 1959 break 1960 s.next() 1961 return ExprNodes.TupleNode(pos, args = exprs) 1962 else: 1963 return expr 1964 1965 1966def p_for_target(s): 1967 return p_target(s, 'in') 1968 1969 1970def p_for_iterator(s, allow_testlist=True, is_async=False): 1971 pos = s.position() 1972 if allow_testlist: 1973 expr = p_testlist(s) 1974 else: 1975 expr = p_or_test(s) 1976 return (ExprNodes.AsyncIteratorNode if is_async else ExprNodes.IteratorNode)(pos, sequence=expr) 1977 1978 1979def p_try_statement(s): 1980 # s.sy == 'try' 1981 pos = s.position() 1982 s.next() 1983 body = p_suite(s) 1984 except_clauses = [] 1985 else_clause = None 1986 if s.sy in ('except', 'else'): 1987 while s.sy == 'except': 1988 except_clauses.append(p_except_clause(s)) 1989 if s.sy == 'else': 1990 s.next() 1991 else_clause = p_suite(s) 1992 body = Nodes.TryExceptStatNode(pos, 1993 body = body, except_clauses = except_clauses, 1994 else_clause = else_clause) 1995 if s.sy != 'finally': 1996 return body 1997 # try-except-finally is equivalent to nested try-except/try-finally 1998 if s.sy == 'finally': 1999 s.next() 2000 finally_clause = p_suite(s) 2001 return Nodes.TryFinallyStatNode(pos, 2002 body = body, finally_clause = finally_clause) 2003 else: 2004 s.error("Expected 'except' or 'finally'") 2005 2006def p_except_clause(s): 2007 # s.sy == 'except' 2008 pos = s.position() 2009 s.next() 2010 exc_type = None 2011 exc_value = None 2012 is_except_as = False 2013 if s.sy != ':': 2014 exc_type = p_test(s) 2015 # normalise into list of single exception tests 2016 if isinstance(exc_type, ExprNodes.TupleNode): 2017 exc_type = exc_type.args 2018 else: 2019 exc_type = [exc_type] 2020 if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as' 2021 and s.context.language_level == 2): 2022 s.next() 2023 exc_value = p_test(s) 2024 elif s.sy == 'IDENT' and s.systring == 'as': 2025 # Py3 syntax requires a name here 2026 s.next() 2027 pos2 = s.position() 2028 name = p_ident(s) 2029 exc_value = ExprNodes.NameNode(pos2, name = name) 2030 is_except_as = True 2031 body = p_suite(s) 2032 return Nodes.ExceptClauseNode(pos, 2033 pattern = exc_type, target = exc_value, 2034 body = body, is_except_as=is_except_as) 2035 2036def p_include_statement(s, ctx): 2037 pos = s.position() 2038 s.next() # 'include' 2039 unicode_include_file_name = p_string_literal(s, 'u')[2] 2040 s.expect_newline("Syntax error in include statement") 2041 if s.compile_time_eval: 2042 include_file_name = unicode_include_file_name 2043 include_file_path = s.context.find_include_file(include_file_name, pos) 2044 if include_file_path: 2045 s.included_files.append(include_file_name) 2046 with Utils.open_source_file(include_file_path) as f: 2047 source_desc = FileSourceDescriptor(include_file_path) 2048 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments) 2049 tree = p_statement_list(s2, ctx) 2050 return tree 2051 else: 2052 return None 2053 else: 2054 return Nodes.PassStatNode(pos) 2055 2056 2057def p_with_statement(s): 2058 s.next() # 'with' 2059 if s.systring == 'template' and not s.in_python_file: 2060 node = p_with_template(s) 2061 else: 2062 node = p_with_items(s) 2063 return node 2064 2065 2066def p_with_items(s, is_async=False): 2067 pos = s.position() 2068 if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'): 2069 if is_async: 2070 s.error("with gil/nogil cannot be async") 2071 state = s.systring 2072 s.next() 2073 if s.sy == ',': 2074 s.next() 2075 body = p_with_items(s) 2076 else: 2077 body = p_suite(s) 2078 return Nodes.GILStatNode(pos, state=state, body=body) 2079 else: 2080 manager = p_test(s) 2081 target = None 2082 if s.sy == 'IDENT' and s.systring == 'as': 2083 s.next() 2084 target = p_starred_expr(s) 2085 if s.sy == ',': 2086 s.next() 2087 body = p_with_items(s, is_async=is_async) 2088 else: 2089 body = p_suite(s) 2090 return Nodes.WithStatNode(pos, manager=manager, target=target, body=body, is_async=is_async) 2091 2092 2093def p_with_template(s): 2094 pos = s.position() 2095 templates = [] 2096 s.next() 2097 s.expect('[') 2098 templates.append(s.systring) 2099 s.next() 2100 while s.systring == ',': 2101 s.next() 2102 templates.append(s.systring) 2103 s.next() 2104 s.expect(']') 2105 if s.sy == ':': 2106 s.next() 2107 s.expect_newline("Syntax error in template function declaration") 2108 s.expect_indent() 2109 body_ctx = Ctx() 2110 body_ctx.templates = templates 2111 func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx) 2112 s.expect_dedent() 2113 return func_or_var 2114 else: 2115 error(pos, "Syntax error in template function declaration") 2116 2117def p_simple_statement(s, first_statement = 0): 2118 #print "p_simple_statement:", s.sy, s.systring ### 2119 if s.sy == 'global': 2120 node = p_global_statement(s) 2121 elif s.sy == 'nonlocal': 2122 node = p_nonlocal_statement(s) 2123 elif s.sy == 'print': 2124 node = p_print_statement(s) 2125 elif s.sy == 'exec': 2126 node = p_exec_statement(s) 2127 elif s.sy == 'del': 2128 node = p_del_statement(s) 2129 elif s.sy == 'break': 2130 node = p_break_statement(s) 2131 elif s.sy == 'continue': 2132 node = p_continue_statement(s) 2133 elif s.sy == 'return': 2134 node = p_return_statement(s) 2135 elif s.sy == 'raise': 2136 node = p_raise_statement(s) 2137 elif s.sy in ('import', 'cimport'): 2138 node = p_import_statement(s) 2139 elif s.sy == 'from': 2140 node = p_from_import_statement(s, first_statement = first_statement) 2141 elif s.sy == 'yield': 2142 node = p_yield_statement(s) 2143 elif s.sy == 'assert': 2144 node = p_assert_statement(s) 2145 elif s.sy == 'pass': 2146 node = p_pass_statement(s) 2147 else: 2148 node = p_expression_or_assignment(s) 2149 return node 2150 2151def p_simple_statement_list(s, ctx, first_statement = 0): 2152 # Parse a series of simple statements on one line 2153 # separated by semicolons. 2154 stat = p_simple_statement(s, first_statement = first_statement) 2155 pos = stat.pos 2156 stats = [] 2157 if not isinstance(stat, Nodes.PassStatNode): 2158 stats.append(stat) 2159 while s.sy == ';': 2160 #print "p_simple_statement_list: maybe more to follow" ### 2161 s.next() 2162 if s.sy in ('NEWLINE', 'EOF'): 2163 break 2164 stat = p_simple_statement(s, first_statement = first_statement) 2165 if isinstance(stat, Nodes.PassStatNode): 2166 continue 2167 stats.append(stat) 2168 first_statement = False 2169 2170 if not stats: 2171 stat = Nodes.PassStatNode(pos) 2172 elif len(stats) == 1: 2173 stat = stats[0] 2174 else: 2175 stat = Nodes.StatListNode(pos, stats = stats) 2176 2177 if s.sy not in ('NEWLINE', 'EOF'): 2178 # provide a better error message for users who accidentally write Cython code in .py files 2179 if isinstance(stat, Nodes.ExprStatNode): 2180 if stat.expr.is_name and stat.expr.name == 'cdef': 2181 s.error("The 'cdef' keyword is only allowed in Cython files (pyx/pxi/pxd)", pos) 2182 s.expect_newline("Syntax error in simple statement list") 2183 2184 return stat 2185 2186def p_compile_time_expr(s): 2187 old = s.compile_time_expr 2188 s.compile_time_expr = 1 2189 expr = p_testlist(s) 2190 s.compile_time_expr = old 2191 return expr 2192 2193def p_DEF_statement(s): 2194 pos = s.position() 2195 denv = s.compile_time_env 2196 s.next() # 'DEF' 2197 name = p_ident(s) 2198 s.expect('=') 2199 expr = p_compile_time_expr(s) 2200 if s.compile_time_eval: 2201 value = expr.compile_time_value(denv) 2202 #print "p_DEF_statement: %s = %r" % (name, value) ### 2203 denv.declare(name, value) 2204 s.expect_newline("Expected a newline", ignore_semicolon=True) 2205 return Nodes.PassStatNode(pos) 2206 2207def p_IF_statement(s, ctx): 2208 pos = s.position() 2209 saved_eval = s.compile_time_eval 2210 current_eval = saved_eval 2211 denv = s.compile_time_env 2212 result = None 2213 while 1: 2214 s.next() # 'IF' or 'ELIF' 2215 expr = p_compile_time_expr(s) 2216 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) 2217 body = p_suite(s, ctx) 2218 if s.compile_time_eval: 2219 result = body 2220 current_eval = 0 2221 if s.sy != 'ELIF': 2222 break 2223 if s.sy == 'ELSE': 2224 s.next() 2225 s.compile_time_eval = current_eval 2226 body = p_suite(s, ctx) 2227 if current_eval: 2228 result = body 2229 if not result: 2230 result = Nodes.PassStatNode(pos) 2231 s.compile_time_eval = saved_eval 2232 return result 2233 2234def p_statement(s, ctx, first_statement = 0): 2235 cdef_flag = ctx.cdef_flag 2236 decorators = None 2237 if s.sy == 'ctypedef': 2238 if ctx.level not in ('module', 'module_pxd'): 2239 s.error("ctypedef statement not allowed here") 2240 #if ctx.api: 2241 # error(s.position(), "'api' not allowed with 'ctypedef'") 2242 return p_ctypedef_statement(s, ctx) 2243 elif s.sy == 'DEF': 2244 return p_DEF_statement(s) 2245 elif s.sy == 'IF': 2246 return p_IF_statement(s, ctx) 2247 elif s.sy == '@': 2248 if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'): 2249 s.error('decorator not allowed here') 2250 s.level = ctx.level 2251 decorators = p_decorators(s) 2252 if not ctx.allow_struct_enum_decorator and s.sy not in ('def', 'cdef', 'cpdef', 'class', 'async'): 2253 if s.sy == 'IDENT' and s.systring == 'async': 2254 pass # handled below 2255 else: 2256 s.error("Decorators can only be followed by functions or classes") 2257 elif s.sy == 'pass' and cdef_flag: 2258 # empty cdef block 2259 return p_pass_statement(s, with_newline=1) 2260 2261 overridable = 0 2262 if s.sy == 'cdef': 2263 cdef_flag = 1 2264 s.next() 2265 elif s.sy == 'cpdef': 2266 cdef_flag = 1 2267 overridable = 1 2268 s.next() 2269 if cdef_flag: 2270 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): 2271 s.error('cdef statement not allowed here') 2272 s.level = ctx.level 2273 node = p_cdef_statement(s, ctx(overridable=overridable)) 2274 if decorators is not None: 2275 tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode) 2276 if ctx.allow_struct_enum_decorator: 2277 tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode) 2278 if not isinstance(node, tup): 2279 s.error("Decorators can only be followed by functions or classes") 2280 node.decorators = decorators 2281 return node 2282 else: 2283 if ctx.api: 2284 s.error("'api' not allowed with this statement", fatal=False) 2285 elif s.sy == 'def': 2286 # def statements aren't allowed in pxd files, except 2287 # as part of a cdef class 2288 if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'): 2289 s.error('def statement not allowed here') 2290 s.level = ctx.level 2291 return p_def_statement(s, decorators) 2292 elif s.sy == 'class': 2293 if ctx.level not in ('module', 'function', 'class', 'other'): 2294 s.error("class definition not allowed here") 2295 return p_class_statement(s, decorators) 2296 elif s.sy == 'include': 2297 if ctx.level not in ('module', 'module_pxd'): 2298 s.error("include statement not allowed here") 2299 return p_include_statement(s, ctx) 2300 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': 2301 return p_property_decl(s) 2302 elif s.sy == 'pass' and ctx.level != 'property': 2303 return p_pass_statement(s, with_newline=True) 2304 else: 2305 if ctx.level in ('c_class_pxd', 'property'): 2306 node = p_ignorable_statement(s) 2307 if node is not None: 2308 return node 2309 s.error("Executable statement not allowed here") 2310 if s.sy == 'if': 2311 return p_if_statement(s) 2312 elif s.sy == 'while': 2313 return p_while_statement(s) 2314 elif s.sy == 'for': 2315 return p_for_statement(s) 2316 elif s.sy == 'try': 2317 return p_try_statement(s) 2318 elif s.sy == 'with': 2319 return p_with_statement(s) 2320 elif s.sy == 'async': 2321 s.next() 2322 return p_async_statement(s, ctx, decorators) 2323 else: 2324 if s.sy == 'IDENT' and s.systring == 'async': 2325 ident_name = s.systring 2326 # PEP 492 enables the async/await keywords when it spots "async def ..." 2327 s.next() 2328 if s.sy == 'def': 2329 return p_async_statement(s, ctx, decorators) 2330 elif decorators: 2331 s.error("Decorators can only be followed by functions or classes") 2332 s.put_back('IDENT', ident_name) # re-insert original token 2333 return p_simple_statement_list(s, ctx, first_statement=first_statement) 2334 2335 2336def p_statement_list(s, ctx, first_statement = 0): 2337 # Parse a series of statements separated by newlines. 2338 pos = s.position() 2339 stats = [] 2340 while s.sy not in ('DEDENT', 'EOF'): 2341 stat = p_statement(s, ctx, first_statement = first_statement) 2342 if isinstance(stat, Nodes.PassStatNode): 2343 continue 2344 stats.append(stat) 2345 first_statement = False 2346 if not stats: 2347 return Nodes.PassStatNode(pos) 2348 elif len(stats) == 1: 2349 return stats[0] 2350 else: 2351 return Nodes.StatListNode(pos, stats = stats) 2352 2353 2354def p_suite(s, ctx=Ctx()): 2355 return p_suite_with_docstring(s, ctx, with_doc_only=False)[1] 2356 2357 2358def p_suite_with_docstring(s, ctx, with_doc_only=False): 2359 s.expect(':') 2360 doc = None 2361 if s.sy == 'NEWLINE': 2362 s.next() 2363 s.expect_indent() 2364 if with_doc_only: 2365 doc = p_doc_string(s) 2366 body = p_statement_list(s, ctx) 2367 s.expect_dedent() 2368 else: 2369 if ctx.api: 2370 s.error("'api' not allowed with this statement", fatal=False) 2371 if ctx.level in ('module', 'class', 'function', 'other'): 2372 body = p_simple_statement_list(s, ctx) 2373 else: 2374 body = p_pass_statement(s) 2375 s.expect_newline("Syntax error in declarations", ignore_semicolon=True) 2376 if not with_doc_only: 2377 doc, body = _extract_docstring(body) 2378 return doc, body 2379 2380 2381def p_positional_and_keyword_args(s, end_sy_set, templates = None): 2382 """ 2383 Parses positional and keyword arguments. end_sy_set 2384 should contain any s.sy that terminate the argument list. 2385 Argument expansion (* and **) are not allowed. 2386 2387 Returns: (positional_args, keyword_args) 2388 """ 2389 positional_args = [] 2390 keyword_args = [] 2391 pos_idx = 0 2392 2393 while s.sy not in end_sy_set: 2394 if s.sy == '*' or s.sy == '**': 2395 s.error('Argument expansion not allowed here.', fatal=False) 2396 2397 parsed_type = False 2398 if s.sy == 'IDENT' and s.peek()[0] == '=': 2399 ident = s.systring 2400 s.next() # s.sy is '=' 2401 s.next() 2402 if looking_at_expr(s): 2403 arg = p_test(s) 2404 else: 2405 base_type = p_c_base_type(s, templates = templates) 2406 declarator = p_c_declarator(s, empty = 1) 2407 arg = Nodes.CComplexBaseTypeNode(base_type.pos, 2408 base_type = base_type, declarator = declarator) 2409 parsed_type = True 2410 keyword_node = ExprNodes.IdentifierStringNode(arg.pos, value=ident) 2411 keyword_args.append((keyword_node, arg)) 2412 was_keyword = True 2413 2414 else: 2415 if looking_at_expr(s): 2416 arg = p_test(s) 2417 else: 2418 base_type = p_c_base_type(s, templates = templates) 2419 declarator = p_c_declarator(s, empty = 1) 2420 arg = Nodes.CComplexBaseTypeNode(base_type.pos, 2421 base_type = base_type, declarator = declarator) 2422 parsed_type = True 2423 positional_args.append(arg) 2424 pos_idx += 1 2425 if len(keyword_args) > 0: 2426 s.error("Non-keyword arg following keyword arg", 2427 pos=arg.pos) 2428 2429 if s.sy != ',': 2430 if s.sy not in end_sy_set: 2431 if parsed_type: 2432 s.error("Unmatched %s" % " or ".join(end_sy_set)) 2433 break 2434 s.next() 2435 return positional_args, keyword_args 2436 2437def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None): 2438 # If self_flag is true, this is the base type for the 2439 # self argument of a C method of an extension type. 2440 if s.sy == '(': 2441 return p_c_complex_base_type(s, templates = templates) 2442 else: 2443 return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates) 2444 2445def p_calling_convention(s): 2446 if s.sy == 'IDENT' and s.systring in calling_convention_words: 2447 result = s.systring 2448 s.next() 2449 return result 2450 else: 2451 return "" 2452 2453 2454calling_convention_words = cython.declare( 2455 set, set(["__stdcall", "__cdecl", "__fastcall"])) 2456 2457 2458def p_c_complex_base_type(s, templates = None): 2459 # s.sy == '(' 2460 pos = s.position() 2461 s.next() 2462 base_type = p_c_base_type(s, templates=templates) 2463 declarator = p_c_declarator(s, empty=True) 2464 type_node = Nodes.CComplexBaseTypeNode( 2465 pos, base_type=base_type, declarator=declarator) 2466 if s.sy == ',': 2467 components = [type_node] 2468 while s.sy == ',': 2469 s.next() 2470 if s.sy == ')': 2471 break 2472 base_type = p_c_base_type(s, templates=templates) 2473 declarator = p_c_declarator(s, empty=True) 2474 components.append(Nodes.CComplexBaseTypeNode( 2475 pos, base_type=base_type, declarator=declarator)) 2476 type_node = Nodes.CTupleBaseTypeNode(pos, components = components) 2477 2478 s.expect(')') 2479 if s.sy == '[': 2480 if is_memoryviewslice_access(s): 2481 type_node = p_memoryviewslice_access(s, type_node) 2482 else: 2483 type_node = p_buffer_or_template(s, type_node, templates) 2484 return type_node 2485 2486 2487def p_c_simple_base_type(s, self_flag, nonempty, templates = None): 2488 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty 2489 is_basic = 0 2490 signed = 1 2491 longness = 0 2492 complex = 0 2493 module_path = [] 2494 pos = s.position() 2495 if not s.sy == 'IDENT': 2496 error(pos, "Expected an identifier, found '%s'" % s.sy) 2497 if s.systring == 'const': 2498 s.next() 2499 base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates) 2500 if isinstance(base_type, Nodes.MemoryViewSliceTypeNode): 2501 # reverse order to avoid having to write "(const int)[:]" 2502 base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node) 2503 return base_type 2504 return Nodes.CConstTypeNode(pos, base_type=base_type) 2505 if looking_at_base_type(s): 2506 #print "p_c_simple_base_type: looking_at_base_type at", s.position() 2507 is_basic = 1 2508 if s.sy == 'IDENT' and s.systring in special_basic_c_types: 2509 signed, longness = special_basic_c_types[s.systring] 2510 name = s.systring 2511 s.next() 2512 else: 2513 signed, longness = p_sign_and_longness(s) 2514 if s.sy == 'IDENT' and s.systring in basic_c_type_names: 2515 name = s.systring 2516 s.next() 2517 else: 2518 name = 'int' # long [int], short [int], long [int] complex, etc. 2519 if s.sy == 'IDENT' and s.systring == 'complex': 2520 complex = 1 2521 s.next() 2522 elif looking_at_dotted_name(s): 2523 #print "p_c_simple_base_type: looking_at_type_name at", s.position() 2524 name = s.systring 2525 s.next() 2526 while s.sy == '.': 2527 module_path.append(name) 2528 s.next() 2529 name = p_ident(s) 2530 else: 2531 name = s.systring 2532 s.next() 2533 if nonempty and s.sy != 'IDENT': 2534 # Make sure this is not a declaration of a variable or function. 2535 if s.sy == '(': 2536 s.next() 2537 if (s.sy == '*' or s.sy == '**' or s.sy == '&' 2538 or (s.sy == 'IDENT' and s.systring in calling_convention_words)): 2539 s.put_back('(', '(') 2540 else: 2541 s.put_back('(', '(') 2542 s.put_back('IDENT', name) 2543 name = None 2544 elif s.sy not in ('*', '**', '[', '&'): 2545 s.put_back('IDENT', name) 2546 name = None 2547 2548 type_node = Nodes.CSimpleBaseTypeNode(pos, 2549 name = name, module_path = module_path, 2550 is_basic_c_type = is_basic, signed = signed, 2551 complex = complex, longness = longness, 2552 is_self_arg = self_flag, templates = templates) 2553 2554 # declarations here. 2555 if s.sy == '[': 2556 if is_memoryviewslice_access(s): 2557 type_node = p_memoryviewslice_access(s, type_node) 2558 else: 2559 type_node = p_buffer_or_template(s, type_node, templates) 2560 2561 if s.sy == '.': 2562 s.next() 2563 name = p_ident(s) 2564 type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name) 2565 2566 return type_node 2567 2568def p_buffer_or_template(s, base_type_node, templates): 2569 # s.sy == '[' 2570 pos = s.position() 2571 s.next() 2572 # Note that buffer_positional_options_count=1, so the only positional argument is dtype. 2573 # For templated types, all parameters are types. 2574 positional_args, keyword_args = ( 2575 p_positional_and_keyword_args(s, (']',), templates) 2576 ) 2577 s.expect(']') 2578 2579 if s.sy == '[': 2580 base_type_node = p_buffer_or_template(s, base_type_node, templates) 2581 2582 keyword_dict = ExprNodes.DictNode(pos, 2583 key_value_pairs = [ 2584 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value) 2585 for key, value in keyword_args 2586 ]) 2587 result = Nodes.TemplatedTypeNode(pos, 2588 positional_args = positional_args, 2589 keyword_args = keyword_dict, 2590 base_type_node = base_type_node) 2591 return result 2592 2593def p_bracketed_base_type(s, base_type_node, nonempty, empty): 2594 # s.sy == '[' 2595 if empty and not nonempty: 2596 # sizeof-like thing. Only anonymous C arrays allowed (int[SIZE]). 2597 return base_type_node 2598 elif not empty and nonempty: 2599 # declaration of either memoryview slice or buffer. 2600 if is_memoryviewslice_access(s): 2601 return p_memoryviewslice_access(s, base_type_node) 2602 else: 2603 return p_buffer_or_template(s, base_type_node, None) 2604 # return p_buffer_access(s, base_type_node) 2605 elif not empty and not nonempty: 2606 # only anonymous C arrays and memoryview slice arrays here. We 2607 # disallow buffer declarations for now, due to ambiguity with anonymous 2608 # C arrays. 2609 if is_memoryviewslice_access(s): 2610 return p_memoryviewslice_access(s, base_type_node) 2611 else: 2612 return base_type_node 2613 2614def is_memoryviewslice_access(s): 2615 # s.sy == '[' 2616 # a memoryview slice declaration is distinguishable from a buffer access 2617 # declaration by the first entry in the bracketed list. The buffer will 2618 # not have an unnested colon in the first entry; the memoryview slice will. 2619 saved = [(s.sy, s.systring)] 2620 s.next() 2621 retval = False 2622 if s.systring == ':': 2623 retval = True 2624 elif s.sy == 'INT': 2625 saved.append((s.sy, s.systring)) 2626 s.next() 2627 if s.sy == ':': 2628 retval = True 2629 2630 for sv in saved[::-1]: 2631 s.put_back(*sv) 2632 2633 return retval 2634 2635def p_memoryviewslice_access(s, base_type_node): 2636 # s.sy == '[' 2637 pos = s.position() 2638 s.next() 2639 subscripts, _ = p_subscript_list(s) 2640 # make sure each entry in subscripts is a slice 2641 for subscript in subscripts: 2642 if len(subscript) < 2: 2643 s.error("An axis specification in memoryview declaration does not have a ':'.") 2644 s.expect(']') 2645 indexes = make_slice_nodes(pos, subscripts) 2646 result = Nodes.MemoryViewSliceTypeNode(pos, 2647 base_type_node = base_type_node, 2648 axes = indexes) 2649 return result 2650 2651def looking_at_name(s): 2652 return s.sy == 'IDENT' and not s.systring in calling_convention_words 2653 2654def looking_at_expr(s): 2655 if s.systring in base_type_start_words: 2656 return False 2657 elif s.sy == 'IDENT': 2658 is_type = False 2659 name = s.systring 2660 dotted_path = [] 2661 s.next() 2662 2663 while s.sy == '.': 2664 s.next() 2665 dotted_path.append(s.systring) 2666 s.expect('IDENT') 2667 2668 saved = s.sy, s.systring 2669 if s.sy == 'IDENT': 2670 is_type = True 2671 elif s.sy == '*' or s.sy == '**': 2672 s.next() 2673 is_type = s.sy in (')', ']') 2674 s.put_back(*saved) 2675 elif s.sy == '(': 2676 s.next() 2677 is_type = s.sy == '*' 2678 s.put_back(*saved) 2679 elif s.sy == '[': 2680 s.next() 2681 is_type = s.sy == ']' or not looking_at_expr(s) # could be a nested template type 2682 s.put_back(*saved) 2683 2684 dotted_path.reverse() 2685 for p in dotted_path: 2686 s.put_back('IDENT', p) 2687 s.put_back('.', '.') 2688 2689 s.put_back('IDENT', name) 2690 return not is_type and saved[0] 2691 else: 2692 return True 2693 2694def looking_at_base_type(s): 2695 #print "looking_at_base_type?", s.sy, s.systring, s.position() 2696 return s.sy == 'IDENT' and s.systring in base_type_start_words 2697 2698def looking_at_dotted_name(s): 2699 if s.sy == 'IDENT': 2700 name = s.systring 2701 s.next() 2702 result = s.sy == '.' 2703 s.put_back('IDENT', name) 2704 return result 2705 else: 2706 return 0 2707 2708def looking_at_call(s): 2709 "See if we're looking at a.b.c(" 2710 # Don't mess up the original position, so save and restore it. 2711 # Unfortunately there's no good way to handle this, as a subsequent call 2712 # to next() will not advance the position until it reads a new token. 2713 position = s.start_line, s.start_col 2714 result = looking_at_expr(s) == u'(' 2715 if not result: 2716 s.start_line, s.start_col = position 2717 return result 2718 2719basic_c_type_names = cython.declare( 2720 set, set(["void", "char", "int", "float", "double", "bint"])) 2721 2722special_basic_c_types = cython.declare(dict, { 2723 # name : (signed, longness) 2724 "Py_UNICODE" : (0, 0), 2725 "Py_UCS4" : (0, 0), 2726 "Py_hash_t" : (2, 0), 2727 "Py_ssize_t" : (2, 0), 2728 "ssize_t" : (2, 0), 2729 "size_t" : (0, 0), 2730 "ptrdiff_t" : (2, 0), 2731 "Py_tss_t" : (1, 0), 2732}) 2733 2734sign_and_longness_words = cython.declare( 2735 set, set(["short", "long", "signed", "unsigned"])) 2736 2737base_type_start_words = cython.declare( 2738 set, 2739 basic_c_type_names 2740 | sign_and_longness_words 2741 | set(special_basic_c_types)) 2742 2743struct_enum_union = cython.declare( 2744 set, set(["struct", "union", "enum", "packed"])) 2745 2746def p_sign_and_longness(s): 2747 signed = 1 2748 longness = 0 2749 while s.sy == 'IDENT' and s.systring in sign_and_longness_words: 2750 if s.systring == 'unsigned': 2751 signed = 0 2752 elif s.systring == 'signed': 2753 signed = 2 2754 elif s.systring == 'short': 2755 longness = -1 2756 elif s.systring == 'long': 2757 longness += 1 2758 s.next() 2759 return signed, longness 2760 2761def p_opt_cname(s): 2762 literal = p_opt_string_literal(s, 'u') 2763 if literal is not None: 2764 cname = EncodedString(literal) 2765 cname.encoding = s.source_encoding 2766 else: 2767 cname = None 2768 return cname 2769 2770def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, 2771 assignable = 0, nonempty = 0, 2772 calling_convention_allowed = 0): 2773 # If empty is true, the declarator must be empty. If nonempty is true, 2774 # the declarator must be nonempty. Otherwise we don't care. 2775 # If cmethod_flag is true, then if this declarator declares 2776 # a function, it's a C method of an extension type. 2777 pos = s.position() 2778 if s.sy == '(': 2779 s.next() 2780 if s.sy == ')' or looking_at_name(s): 2781 base = Nodes.CNameDeclaratorNode(pos, name=s.context.intern_ustring(u""), cname=None) 2782 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag) 2783 else: 2784 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2785 cmethod_flag = cmethod_flag, 2786 nonempty = nonempty, 2787 calling_convention_allowed = 1) 2788 s.expect(')') 2789 else: 2790 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, 2791 assignable, nonempty) 2792 if not calling_convention_allowed and result.calling_convention and s.sy != '(': 2793 error(s.position(), "%s on something that is not a function" 2794 % result.calling_convention) 2795 while s.sy in ('[', '('): 2796 pos = s.position() 2797 if s.sy == '[': 2798 result = p_c_array_declarator(s, result) 2799 else: # sy == '(' 2800 s.next() 2801 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) 2802 cmethod_flag = 0 2803 return result 2804 2805def p_c_array_declarator(s, base): 2806 pos = s.position() 2807 s.next() # '[' 2808 if s.sy != ']': 2809 dim = p_testlist(s) 2810 else: 2811 dim = None 2812 s.expect(']') 2813 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) 2814 2815def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): 2816 # Opening paren has already been skipped 2817 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, 2818 nonempty_declarators = 0) 2819 ellipsis = p_optional_ellipsis(s) 2820 s.expect(')') 2821 nogil = p_nogil(s) 2822 exc_val, exc_check = p_exception_value_clause(s) 2823 with_gil = p_with_gil(s) 2824 return Nodes.CFuncDeclaratorNode(pos, 2825 base = base, args = args, has_varargs = ellipsis, 2826 exception_value = exc_val, exception_check = exc_check, 2827 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) 2828 2829supported_overloaded_operators = cython.declare(set, set([ 2830 '+', '-', '*', '/', '%', 2831 '++', '--', '~', '|', '&', '^', '<<', '>>', ',', 2832 '==', '!=', '>=', '>', '<=', '<', 2833 '[]', '()', '!', '=', 2834 'bool', 2835])) 2836 2837def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, 2838 assignable, nonempty): 2839 pos = s.position() 2840 calling_convention = p_calling_convention(s) 2841 if s.sy == '*': 2842 s.next() 2843 if s.systring == 'const': 2844 const_pos = s.position() 2845 s.next() 2846 const_base = p_c_declarator(s, ctx, empty = empty, 2847 is_type = is_type, 2848 cmethod_flag = cmethod_flag, 2849 assignable = assignable, 2850 nonempty = nonempty) 2851 base = Nodes.CConstDeclaratorNode(const_pos, base = const_base) 2852 else: 2853 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2854 cmethod_flag = cmethod_flag, 2855 assignable = assignable, nonempty = nonempty) 2856 result = Nodes.CPtrDeclaratorNode(pos, 2857 base = base) 2858 elif s.sy == '**': # scanner returns this as a single token 2859 s.next() 2860 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2861 cmethod_flag = cmethod_flag, 2862 assignable = assignable, nonempty = nonempty) 2863 result = Nodes.CPtrDeclaratorNode(pos, 2864 base = Nodes.CPtrDeclaratorNode(pos, 2865 base = base)) 2866 elif s.sy == '&': 2867 s.next() 2868 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2869 cmethod_flag = cmethod_flag, 2870 assignable = assignable, nonempty = nonempty) 2871 result = Nodes.CReferenceDeclaratorNode(pos, base = base) 2872 else: 2873 rhs = None 2874 if s.sy == 'IDENT': 2875 name = s.systring 2876 if empty: 2877 error(s.position(), "Declarator should be empty") 2878 s.next() 2879 cname = p_opt_cname(s) 2880 if name != 'operator' and s.sy == '=' and assignable: 2881 s.next() 2882 rhs = p_test(s) 2883 else: 2884 if nonempty: 2885 error(s.position(), "Empty declarator") 2886 name = "" 2887 cname = None 2888 if cname is None and ctx.namespace is not None and nonempty: 2889 cname = ctx.namespace + "::" + name 2890 if name == 'operator' and ctx.visibility == 'extern' and nonempty: 2891 op = s.sy 2892 if [1 for c in op if c in '+-*/<=>!%&|([^~,']: 2893 s.next() 2894 # Handle diphthong operators. 2895 if op == '(': 2896 s.expect(')') 2897 op = '()' 2898 elif op == '[': 2899 s.expect(']') 2900 op = '[]' 2901 elif op in ('-', '+', '|', '&') and s.sy == op: 2902 op *= 2 # ++, --, ... 2903 s.next() 2904 elif s.sy == '=': 2905 op += s.sy # +=, -=, ... 2906 s.next() 2907 if op not in supported_overloaded_operators: 2908 s.error("Overloading operator '%s' not yet supported." % op, 2909 fatal=False) 2910 name += op 2911 elif op == 'IDENT': 2912 op = s.systring; 2913 if op not in supported_overloaded_operators: 2914 s.error("Overloading operator '%s' not yet supported." % op, 2915 fatal=False) 2916 name = name + ' ' + op 2917 s.next() 2918 result = Nodes.CNameDeclaratorNode(pos, 2919 name = name, cname = cname, default = rhs) 2920 result.calling_convention = calling_convention 2921 return result 2922 2923def p_nogil(s): 2924 if s.sy == 'IDENT' and s.systring == 'nogil': 2925 s.next() 2926 return 1 2927 else: 2928 return 0 2929 2930def p_with_gil(s): 2931 if s.sy == 'with': 2932 s.next() 2933 s.expect_keyword('gil') 2934 return 1 2935 else: 2936 return 0 2937 2938def p_exception_value_clause(s): 2939 exc_val = None 2940 exc_check = 0 2941 if s.sy == 'except': 2942 s.next() 2943 if s.sy == '*': 2944 exc_check = 1 2945 s.next() 2946 elif s.sy == '+': 2947 exc_check = '+' 2948 s.next() 2949 if s.sy == 'IDENT': 2950 name = s.systring 2951 s.next() 2952 exc_val = p_name(s, name) 2953 elif s.sy == '*': 2954 exc_val = ExprNodes.CharNode(s.position(), value=u'*') 2955 s.next() 2956 else: 2957 if s.sy == '?': 2958 exc_check = 1 2959 s.next() 2960 exc_val = p_test(s) 2961 return exc_val, exc_check 2962 2963c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':'])) 2964 2965def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, 2966 nonempty_declarators = 0, kw_only = 0, annotated = 1): 2967 # Comma-separated list of C argument declarations, possibly empty. 2968 # May have a trailing comma. 2969 args = [] 2970 is_self_arg = cmethod_flag 2971 while s.sy not in c_arg_list_terminators: 2972 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg, 2973 nonempty = nonempty_declarators, kw_only = kw_only, 2974 annotated = annotated)) 2975 if s.sy != ',': 2976 break 2977 s.next() 2978 is_self_arg = 0 2979 return args 2980 2981def p_optional_ellipsis(s): 2982 if s.sy == '.': 2983 expect_ellipsis(s) 2984 return 1 2985 else: 2986 return 0 2987 2988def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, 2989 kw_only = 0, annotated = 1): 2990 pos = s.position() 2991 not_none = or_none = 0 2992 default = None 2993 annotation = None 2994 if s.in_python_file: 2995 # empty type declaration 2996 base_type = Nodes.CSimpleBaseTypeNode(pos, 2997 name = None, module_path = [], 2998 is_basic_c_type = 0, signed = 0, 2999 complex = 0, longness = 0, 3000 is_self_arg = cmethod_flag, templates = None) 3001 else: 3002 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty) 3003 declarator = p_c_declarator(s, ctx, nonempty = nonempty) 3004 if s.sy in ('not', 'or') and not s.in_python_file: 3005 kind = s.sy 3006 s.next() 3007 if s.sy == 'IDENT' and s.systring == 'None': 3008 s.next() 3009 else: 3010 s.error("Expected 'None'") 3011 if not in_pyfunc: 3012 error(pos, "'%s None' only allowed in Python functions" % kind) 3013 or_none = kind == 'or' 3014 not_none = kind == 'not' 3015 if annotated and s.sy == ':': 3016 s.next() 3017 annotation = p_test(s) 3018 if s.sy == '=': 3019 s.next() 3020 if 'pxd' in ctx.level: 3021 if s.sy in ['*', '?']: 3022 # TODO(github/1736): Make this an error for inline declarations. 3023 default = ExprNodes.NoneNode(pos) 3024 s.next() 3025 elif 'inline' in ctx.modifiers: 3026 default = p_test(s) 3027 else: 3028 error(pos, "default values cannot be specified in pxd files, use ? or *") 3029 else: 3030 default = p_test(s) 3031 return Nodes.CArgDeclNode(pos, 3032 base_type = base_type, 3033 declarator = declarator, 3034 not_none = not_none, 3035 or_none = or_none, 3036 default = default, 3037 annotation = annotation, 3038 kw_only = kw_only) 3039 3040def p_api(s): 3041 if s.sy == 'IDENT' and s.systring == 'api': 3042 s.next() 3043 return 1 3044 else: 3045 return 0 3046 3047def p_cdef_statement(s, ctx): 3048 pos = s.position() 3049 ctx.visibility = p_visibility(s, ctx.visibility) 3050 ctx.api = ctx.api or p_api(s) 3051 if ctx.api: 3052 if ctx.visibility not in ('private', 'public'): 3053 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility) 3054 if (ctx.visibility == 'extern') and s.sy == 'from': 3055 return p_cdef_extern_block(s, pos, ctx) 3056 elif s.sy == 'import': 3057 s.next() 3058 return p_cdef_extern_block(s, pos, ctx) 3059 elif p_nogil(s): 3060 ctx.nogil = 1 3061 if ctx.overridable: 3062 error(pos, "cdef blocks cannot be declared cpdef") 3063 return p_cdef_block(s, ctx) 3064 elif s.sy == ':': 3065 if ctx.overridable: 3066 error(pos, "cdef blocks cannot be declared cpdef") 3067 return p_cdef_block(s, ctx) 3068 elif s.sy == 'class': 3069 if ctx.level not in ('module', 'module_pxd'): 3070 error(pos, "Extension type definition not allowed here") 3071 if ctx.overridable: 3072 error(pos, "Extension types cannot be declared cpdef") 3073 return p_c_class_definition(s, pos, ctx) 3074 elif s.sy == 'IDENT' and s.systring == 'cppclass': 3075 return p_cpp_class_definition(s, pos, ctx) 3076 elif s.sy == 'IDENT' and s.systring in struct_enum_union: 3077 if ctx.level not in ('module', 'module_pxd'): 3078 error(pos, "C struct/union/enum definition not allowed here") 3079 if ctx.overridable: 3080 if s.systring != 'enum': 3081 error(pos, "C struct/union cannot be declared cpdef") 3082 return p_struct_enum(s, pos, ctx) 3083 elif s.sy == 'IDENT' and s.systring == 'fused': 3084 return p_fused_definition(s, pos, ctx) 3085 else: 3086 return p_c_func_or_var_declaration(s, pos, ctx) 3087 3088def p_cdef_block(s, ctx): 3089 return p_suite(s, ctx(cdef_flag = 1)) 3090 3091def p_cdef_extern_block(s, pos, ctx): 3092 if ctx.overridable: 3093 error(pos, "cdef extern blocks cannot be declared cpdef") 3094 include_file = None 3095 s.expect('from') 3096 if s.sy == '*': 3097 s.next() 3098 else: 3099 include_file = p_string_literal(s, 'u')[2] 3100 ctx = ctx(cdef_flag = 1, visibility = 'extern') 3101 if s.systring == "namespace": 3102 s.next() 3103 ctx.namespace = p_string_literal(s, 'u')[2] 3104 if p_nogil(s): 3105 ctx.nogil = 1 3106 3107 # Use "docstring" as verbatim string to include 3108 verbatim_include, body = p_suite_with_docstring(s, ctx, True) 3109 3110 return Nodes.CDefExternNode(pos, 3111 include_file = include_file, 3112 verbatim_include = verbatim_include, 3113 body = body, 3114 namespace = ctx.namespace) 3115 3116def p_c_enum_definition(s, pos, ctx): 3117 # s.sy == ident 'enum' 3118 s.next() 3119 if s.sy == 'IDENT': 3120 name = s.systring 3121 s.next() 3122 cname = p_opt_cname(s) 3123 if cname is None and ctx.namespace is not None: 3124 cname = ctx.namespace + "::" + name 3125 else: 3126 name = None 3127 cname = None 3128 items = None 3129 s.expect(':') 3130 items = [] 3131 if s.sy != 'NEWLINE': 3132 p_c_enum_line(s, ctx, items) 3133 else: 3134 s.next() # 'NEWLINE' 3135 s.expect_indent() 3136 while s.sy not in ('DEDENT', 'EOF'): 3137 p_c_enum_line(s, ctx, items) 3138 s.expect_dedent() 3139 return Nodes.CEnumDefNode( 3140 pos, name = name, cname = cname, items = items, 3141 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, 3142 create_wrapper = ctx.overridable, 3143 api = ctx.api, in_pxd = ctx.level == 'module_pxd') 3144 3145def p_c_enum_line(s, ctx, items): 3146 if s.sy != 'pass': 3147 p_c_enum_item(s, ctx, items) 3148 while s.sy == ',': 3149 s.next() 3150 if s.sy in ('NEWLINE', 'EOF'): 3151 break 3152 p_c_enum_item(s, ctx, items) 3153 else: 3154 s.next() 3155 s.expect_newline("Syntax error in enum item list") 3156 3157def p_c_enum_item(s, ctx, items): 3158 pos = s.position() 3159 name = p_ident(s) 3160 cname = p_opt_cname(s) 3161 if cname is None and ctx.namespace is not None: 3162 cname = ctx.namespace + "::" + name 3163 value = None 3164 if s.sy == '=': 3165 s.next() 3166 value = p_test(s) 3167 items.append(Nodes.CEnumDefItemNode(pos, 3168 name = name, cname = cname, value = value)) 3169 3170def p_c_struct_or_union_definition(s, pos, ctx): 3171 packed = False 3172 if s.systring == 'packed': 3173 packed = True 3174 s.next() 3175 if s.sy != 'IDENT' or s.systring != 'struct': 3176 s.expected('struct') 3177 # s.sy == ident 'struct' or 'union' 3178 kind = s.systring 3179 s.next() 3180 name = p_ident(s) 3181 cname = p_opt_cname(s) 3182 if cname is None and ctx.namespace is not None: 3183 cname = ctx.namespace + "::" + name 3184 attributes = None 3185 if s.sy == ':': 3186 s.next() 3187 s.expect('NEWLINE') 3188 s.expect_indent() 3189 attributes = [] 3190 body_ctx = Ctx() 3191 while s.sy != 'DEDENT': 3192 if s.sy != 'pass': 3193 attributes.append( 3194 p_c_func_or_var_declaration(s, s.position(), body_ctx)) 3195 else: 3196 s.next() 3197 s.expect_newline("Expected a newline") 3198 s.expect_dedent() 3199 else: 3200 s.expect_newline("Syntax error in struct or union definition") 3201 return Nodes.CStructOrUnionDefNode(pos, 3202 name = name, cname = cname, kind = kind, attributes = attributes, 3203 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, 3204 api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed) 3205 3206def p_fused_definition(s, pos, ctx): 3207 """ 3208 c(type)def fused my_fused_type: 3209 ... 3210 """ 3211 # s.systring == 'fused' 3212 3213 if ctx.level not in ('module', 'module_pxd'): 3214 error(pos, "Fused type definition not allowed here") 3215 3216 s.next() 3217 name = p_ident(s) 3218 3219 s.expect(":") 3220 s.expect_newline() 3221 s.expect_indent() 3222 3223 types = [] 3224 while s.sy != 'DEDENT': 3225 if s.sy != 'pass': 3226 #types.append(p_c_declarator(s)) 3227 types.append(p_c_base_type(s)) #, nonempty=1)) 3228 else: 3229 s.next() 3230 3231 s.expect_newline() 3232 3233 s.expect_dedent() 3234 3235 if not types: 3236 error(pos, "Need at least one type") 3237 3238 return Nodes.FusedTypeNode(pos, name=name, types=types) 3239 3240def p_struct_enum(s, pos, ctx): 3241 if s.systring == 'enum': 3242 return p_c_enum_definition(s, pos, ctx) 3243 else: 3244 return p_c_struct_or_union_definition(s, pos, ctx) 3245 3246def p_visibility(s, prev_visibility): 3247 pos = s.position() 3248 visibility = prev_visibility 3249 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'): 3250 visibility = s.systring 3251 if prev_visibility != 'private' and visibility != prev_visibility: 3252 s.error("Conflicting visibility options '%s' and '%s'" 3253 % (prev_visibility, visibility), fatal=False) 3254 s.next() 3255 return visibility 3256 3257def p_c_modifiers(s): 3258 if s.sy == 'IDENT' and s.systring in ('inline',): 3259 modifier = s.systring 3260 s.next() 3261 return [modifier] + p_c_modifiers(s) 3262 return [] 3263 3264def p_c_func_or_var_declaration(s, pos, ctx): 3265 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd') 3266 modifiers = p_c_modifiers(s) 3267 base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates) 3268 declarator = p_c_declarator(s, ctx(modifiers=modifiers), cmethod_flag = cmethod_flag, 3269 assignable = 1, nonempty = 1) 3270 declarator.overridable = ctx.overridable 3271 if s.sy == 'IDENT' and s.systring == 'const' and ctx.level == 'cpp_class': 3272 s.next() 3273 is_const_method = 1 3274 else: 3275 is_const_method = 0 3276 if s.sy == '->': 3277 # Special enough to give a better error message and keep going. 3278 s.error( 3279 "Return type annotation is not allowed in cdef/cpdef signatures. " 3280 "Please define it before the function name, as in C signatures.", 3281 fatal=False) 3282 s.next() 3283 p_test(s) # Keep going, but ignore result. 3284 if s.sy == ':': 3285 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates: 3286 s.error("C function definition not allowed here") 3287 doc, suite = p_suite_with_docstring(s, Ctx(level='function')) 3288 result = Nodes.CFuncDefNode(pos, 3289 visibility = ctx.visibility, 3290 base_type = base_type, 3291 declarator = declarator, 3292 body = suite, 3293 doc = doc, 3294 modifiers = modifiers, 3295 api = ctx.api, 3296 overridable = ctx.overridable, 3297 is_const_method = is_const_method) 3298 else: 3299 #if api: 3300 # s.error("'api' not allowed with variable declaration") 3301 if is_const_method: 3302 declarator.is_const_method = is_const_method 3303 declarators = [declarator] 3304 while s.sy == ',': 3305 s.next() 3306 if s.sy == 'NEWLINE': 3307 break 3308 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, 3309 assignable = 1, nonempty = 1) 3310 declarators.append(declarator) 3311 doc_line = s.start_line + 1 3312 s.expect_newline("Syntax error in C variable declaration", ignore_semicolon=True) 3313 if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line: 3314 doc = p_doc_string(s) 3315 else: 3316 doc = None 3317 result = Nodes.CVarDefNode(pos, 3318 visibility = ctx.visibility, 3319 base_type = base_type, 3320 declarators = declarators, 3321 in_pxd = ctx.level in ('module_pxd', 'c_class_pxd'), 3322 doc = doc, 3323 api = ctx.api, 3324 modifiers = modifiers, 3325 overridable = ctx.overridable) 3326 return result 3327 3328def p_ctypedef_statement(s, ctx): 3329 # s.sy == 'ctypedef' 3330 pos = s.position() 3331 s.next() 3332 visibility = p_visibility(s, ctx.visibility) 3333 api = p_api(s) 3334 ctx = ctx(typedef_flag = 1, visibility = visibility) 3335 if api: 3336 ctx.api = 1 3337 if s.sy == 'class': 3338 return p_c_class_definition(s, pos, ctx) 3339 elif s.sy == 'IDENT' and s.systring in struct_enum_union: 3340 return p_struct_enum(s, pos, ctx) 3341 elif s.sy == 'IDENT' and s.systring == 'fused': 3342 return p_fused_definition(s, pos, ctx) 3343 else: 3344 base_type = p_c_base_type(s, nonempty = 1) 3345 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1) 3346 s.expect_newline("Syntax error in ctypedef statement", ignore_semicolon=True) 3347 return Nodes.CTypeDefNode( 3348 pos, base_type = base_type, 3349 declarator = declarator, 3350 visibility = visibility, api = api, 3351 in_pxd = ctx.level == 'module_pxd') 3352 3353def p_decorators(s): 3354 decorators = [] 3355 while s.sy == '@': 3356 pos = s.position() 3357 s.next() 3358 decstring = p_dotted_name(s, as_allowed=0)[2] 3359 names = decstring.split('.') 3360 decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0])) 3361 for name in names[1:]: 3362 decorator = ExprNodes.AttributeNode( 3363 pos, attribute=s.context.intern_ustring(name), obj=decorator) 3364 if s.sy == '(': 3365 decorator = p_call(s, decorator) 3366 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator)) 3367 s.expect_newline("Expected a newline after decorator") 3368 return decorators 3369 3370 3371def _reject_cdef_modifier_in_py(s, name): 3372 """Step over incorrectly placed cdef modifiers (@see _CDEF_MODIFIERS) to provide a good error message for them. 3373 """ 3374 if s.sy == 'IDENT' and name in _CDEF_MODIFIERS: 3375 # Special enough to provide a good error message. 3376 s.error("Cannot use cdef modifier '%s' in Python function signature. Use a decorator instead." % name, fatal=False) 3377 return p_ident(s) # Keep going, in case there are other errors. 3378 return name 3379 3380 3381def p_def_statement(s, decorators=None, is_async_def=False): 3382 # s.sy == 'def' 3383 pos = s.position() 3384 # PEP 492 switches the async/await keywords on in "async def" functions 3385 if is_async_def: 3386 s.enter_async() 3387 s.next() 3388 name = _reject_cdef_modifier_in_py(s, p_ident(s)) 3389 s.expect( 3390 '(', 3391 "Expected '(', found '%s'. Did you use cdef syntax in a Python declaration? " 3392 "Use decorators and Python type annotations instead." % ( 3393 s.systring if s.sy == 'IDENT' else s.sy)) 3394 args, star_arg, starstar_arg = p_varargslist(s, terminator=')') 3395 s.expect(')') 3396 _reject_cdef_modifier_in_py(s, s.systring) 3397 return_type_annotation = None 3398 if s.sy == '->': 3399 s.next() 3400 return_type_annotation = p_test(s) 3401 _reject_cdef_modifier_in_py(s, s.systring) 3402 3403 doc, body = p_suite_with_docstring(s, Ctx(level='function')) 3404 if is_async_def: 3405 s.exit_async() 3406 3407 return Nodes.DefNode( 3408 pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg, 3409 doc=doc, body=body, decorators=decorators, is_async_def=is_async_def, 3410 return_type_annotation=return_type_annotation) 3411 3412 3413def p_varargslist(s, terminator=')', annotated=1): 3414 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1, 3415 annotated = annotated) 3416 star_arg = None 3417 starstar_arg = None 3418 if s.sy == '*': 3419 s.next() 3420 if s.sy == 'IDENT': 3421 star_arg = p_py_arg_decl(s, annotated=annotated) 3422 if s.sy == ',': 3423 s.next() 3424 args.extend(p_c_arg_list(s, in_pyfunc = 1, 3425 nonempty_declarators = 1, kw_only = 1, annotated = annotated)) 3426 elif s.sy != terminator: 3427 s.error("Syntax error in Python function argument list") 3428 if s.sy == '**': 3429 s.next() 3430 starstar_arg = p_py_arg_decl(s, annotated=annotated) 3431 if s.sy == ',': 3432 s.next() 3433 return (args, star_arg, starstar_arg) 3434 3435def p_py_arg_decl(s, annotated = 1): 3436 pos = s.position() 3437 name = p_ident(s) 3438 annotation = None 3439 if annotated and s.sy == ':': 3440 s.next() 3441 annotation = p_test(s) 3442 return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation) 3443 3444 3445def p_class_statement(s, decorators): 3446 # s.sy == 'class' 3447 pos = s.position() 3448 s.next() 3449 class_name = EncodedString(p_ident(s)) 3450 class_name.encoding = s.source_encoding # FIXME: why is this needed? 3451 arg_tuple = None 3452 keyword_dict = None 3453 if s.sy == '(': 3454 positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False) 3455 arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args) 3456 if arg_tuple is None: 3457 # XXX: empty arg_tuple 3458 arg_tuple = ExprNodes.TupleNode(pos, args=[]) 3459 doc, body = p_suite_with_docstring(s, Ctx(level='class')) 3460 return Nodes.PyClassDefNode( 3461 pos, name=class_name, 3462 bases=arg_tuple, 3463 keyword_args=keyword_dict, 3464 doc=doc, body=body, decorators=decorators, 3465 force_py3_semantics=s.context.language_level >= 3) 3466 3467 3468def p_c_class_definition(s, pos, ctx): 3469 # s.sy == 'class' 3470 s.next() 3471 module_path = [] 3472 class_name = p_ident(s) 3473 while s.sy == '.': 3474 s.next() 3475 module_path.append(class_name) 3476 class_name = p_ident(s) 3477 if module_path and ctx.visibility != 'extern': 3478 error(pos, "Qualified class name only allowed for 'extern' C class") 3479 if module_path and s.sy == 'IDENT' and s.systring == 'as': 3480 s.next() 3481 as_name = p_ident(s) 3482 else: 3483 as_name = class_name 3484 objstruct_name = None 3485 typeobj_name = None 3486 bases = None 3487 check_size = None 3488 if s.sy == '(': 3489 positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False) 3490 if keyword_args: 3491 s.error("C classes cannot take keyword bases.") 3492 bases, _ = p_call_build_packed_args(pos, positional_args, keyword_args) 3493 if bases is None: 3494 bases = ExprNodes.TupleNode(pos, args=[]) 3495 3496 if s.sy == '[': 3497 if ctx.visibility not in ('public', 'extern') and not ctx.api: 3498 error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class") 3499 objstruct_name, typeobj_name, check_size = p_c_class_options(s) 3500 if s.sy == ':': 3501 if ctx.level == 'module_pxd': 3502 body_level = 'c_class_pxd' 3503 else: 3504 body_level = 'c_class' 3505 doc, body = p_suite_with_docstring(s, Ctx(level=body_level)) 3506 else: 3507 s.expect_newline("Syntax error in C class definition") 3508 doc = None 3509 body = None 3510 if ctx.visibility == 'extern': 3511 if not module_path: 3512 error(pos, "Module name required for 'extern' C class") 3513 if typeobj_name: 3514 error(pos, "Type object name specification not allowed for 'extern' C class") 3515 elif ctx.visibility == 'public': 3516 if not objstruct_name: 3517 error(pos, "Object struct name specification required for 'public' C class") 3518 if not typeobj_name: 3519 error(pos, "Type object name specification required for 'public' C class") 3520 elif ctx.visibility == 'private': 3521 if ctx.api: 3522 if not objstruct_name: 3523 error(pos, "Object struct name specification required for 'api' C class") 3524 if not typeobj_name: 3525 error(pos, "Type object name specification required for 'api' C class") 3526 else: 3527 error(pos, "Invalid class visibility '%s'" % ctx.visibility) 3528 return Nodes.CClassDefNode(pos, 3529 visibility = ctx.visibility, 3530 typedef_flag = ctx.typedef_flag, 3531 api = ctx.api, 3532 module_name = ".".join(module_path), 3533 class_name = class_name, 3534 as_name = as_name, 3535 bases = bases, 3536 objstruct_name = objstruct_name, 3537 typeobj_name = typeobj_name, 3538 check_size = check_size, 3539 in_pxd = ctx.level == 'module_pxd', 3540 doc = doc, 3541 body = body) 3542 3543 3544def p_c_class_options(s): 3545 objstruct_name = None 3546 typeobj_name = None 3547 check_size = None 3548 s.expect('[') 3549 while 1: 3550 if s.sy != 'IDENT': 3551 break 3552 if s.systring == 'object': 3553 s.next() 3554 objstruct_name = p_ident(s) 3555 elif s.systring == 'type': 3556 s.next() 3557 typeobj_name = p_ident(s) 3558 elif s.systring == 'check_size': 3559 s.next() 3560 check_size = p_ident(s) 3561 if check_size not in ('ignore', 'warn', 'error'): 3562 s.error("Expected one of ignore, warn or error, found %r" % check_size) 3563 if s.sy != ',': 3564 break 3565 s.next() 3566 s.expect(']', "Expected 'object', 'type' or 'check_size'") 3567 return objstruct_name, typeobj_name, check_size 3568 3569 3570def p_property_decl(s): 3571 pos = s.position() 3572 s.next() # 'property' 3573 name = p_ident(s) 3574 doc, body = p_suite_with_docstring( 3575 s, Ctx(level='property'), with_doc_only=True) 3576 return Nodes.PropertyNode(pos, name=name, doc=doc, body=body) 3577 3578 3579def p_ignorable_statement(s): 3580 """ 3581 Parses any kind of ignorable statement that is allowed in .pxd files. 3582 """ 3583 if s.sy == 'BEGIN_STRING': 3584 pos = s.position() 3585 string_node = p_atom(s) 3586 s.expect_newline("Syntax error in string", ignore_semicolon=True) 3587 return Nodes.ExprStatNode(pos, expr=string_node) 3588 return None 3589 3590 3591def p_doc_string(s): 3592 if s.sy == 'BEGIN_STRING': 3593 pos = s.position() 3594 kind, bytes_result, unicode_result = p_cat_string_literal(s) 3595 s.expect_newline("Syntax error in doc string", ignore_semicolon=True) 3596 if kind in ('u', ''): 3597 return unicode_result 3598 warning(pos, "Python 3 requires docstrings to be unicode strings") 3599 return bytes_result 3600 else: 3601 return None 3602 3603 3604def _extract_docstring(node): 3605 """ 3606 Extract a docstring from a statement or from the first statement 3607 in a list. Remove the statement if found. Return a tuple 3608 (plain-docstring or None, node). 3609 """ 3610 doc_node = None 3611 if node is None: 3612 pass 3613 elif isinstance(node, Nodes.ExprStatNode): 3614 if node.expr.is_string_literal: 3615 doc_node = node.expr 3616 node = Nodes.StatListNode(node.pos, stats=[]) 3617 elif isinstance(node, Nodes.StatListNode) and node.stats: 3618 stats = node.stats 3619 if isinstance(stats[0], Nodes.ExprStatNode): 3620 if stats[0].expr.is_string_literal: 3621 doc_node = stats[0].expr 3622 del stats[0] 3623 3624 if doc_node is None: 3625 doc = None 3626 elif isinstance(doc_node, ExprNodes.BytesNode): 3627 warning(node.pos, 3628 "Python 3 requires docstrings to be unicode strings") 3629 doc = doc_node.value 3630 elif isinstance(doc_node, ExprNodes.StringNode): 3631 doc = doc_node.unicode_value 3632 if doc is None: 3633 doc = doc_node.value 3634 else: 3635 doc = doc_node.value 3636 return doc, node 3637 3638 3639def p_code(s, level=None, ctx=Ctx): 3640 body = p_statement_list(s, ctx(level = level), first_statement = 1) 3641 if s.sy != 'EOF': 3642 s.error("Syntax error in statement [%s,%s]" % ( 3643 repr(s.sy), repr(s.systring))) 3644 return body 3645 3646 3647_match_compiler_directive_comment = cython.declare(object, re.compile( 3648 r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match) 3649 3650 3651def p_compiler_directive_comments(s): 3652 result = {} 3653 while s.sy == 'commentline': 3654 pos = s.position() 3655 m = _match_compiler_directive_comment(s.systring) 3656 if m: 3657 directives_string = m.group(1).strip() 3658 try: 3659 new_directives = Options.parse_directive_list(directives_string, ignore_unknown=True) 3660 except ValueError as e: 3661 s.error(e.args[0], fatal=False) 3662 s.next() 3663 continue 3664 3665 for name in new_directives: 3666 if name not in result: 3667 pass 3668 elif new_directives[name] == result[name]: 3669 warning(pos, "Duplicate directive found: %s" % (name,)) 3670 else: 3671 s.error("Conflicting settings found for top-level directive %s: %r and %r" % ( 3672 name, result[name], new_directives[name]), pos=pos) 3673 3674 if 'language_level' in new_directives: 3675 # Make sure we apply the language level already to the first token that follows the comments. 3676 s.context.set_language_level(new_directives['language_level']) 3677 3678 result.update(new_directives) 3679 3680 s.next() 3681 return result 3682 3683 3684def p_module(s, pxd, full_module_name, ctx=Ctx): 3685 pos = s.position() 3686 3687 directive_comments = p_compiler_directive_comments(s) 3688 s.parse_comments = False 3689 3690 if s.context.language_level is None: 3691 s.context.set_language_level(2) 3692 if pos[0].filename: 3693 import warnings 3694 warnings.warn( 3695 "Cython directive 'language_level' not set, using 2 for now (Py2). " 3696 "This will change in a later release! File: %s" % pos[0].filename, 3697 FutureWarning, 3698 stacklevel=1 if cython.compiled else 2, 3699 ) 3700 3701 doc = p_doc_string(s) 3702 if pxd: 3703 level = 'module_pxd' 3704 else: 3705 level = 'module' 3706 3707 body = p_statement_list(s, ctx(level=level), first_statement = 1) 3708 if s.sy != 'EOF': 3709 s.error("Syntax error in statement [%s,%s]" % ( 3710 repr(s.sy), repr(s.systring))) 3711 return ModuleNode(pos, doc = doc, body = body, 3712 full_module_name = full_module_name, 3713 directive_comments = directive_comments) 3714 3715def p_template_definition(s): 3716 name = p_ident(s) 3717 if s.sy == '=': 3718 s.expect('=') 3719 s.expect('*') 3720 required = False 3721 else: 3722 required = True 3723 return name, required 3724 3725def p_cpp_class_definition(s, pos, ctx): 3726 # s.sy == 'cppclass' 3727 s.next() 3728 module_path = [] 3729 class_name = p_ident(s) 3730 cname = p_opt_cname(s) 3731 if cname is None and ctx.namespace is not None: 3732 cname = ctx.namespace + "::" + class_name 3733 if s.sy == '.': 3734 error(pos, "Qualified class name not allowed C++ class") 3735 if s.sy == '[': 3736 s.next() 3737 templates = [p_template_definition(s)] 3738 while s.sy == ',': 3739 s.next() 3740 templates.append(p_template_definition(s)) 3741 s.expect(']') 3742 template_names = [name for name, required in templates] 3743 else: 3744 templates = None 3745 template_names = None 3746 if s.sy == '(': 3747 s.next() 3748 base_classes = [p_c_base_type(s, templates = template_names)] 3749 while s.sy == ',': 3750 s.next() 3751 base_classes.append(p_c_base_type(s, templates = template_names)) 3752 s.expect(')') 3753 else: 3754 base_classes = [] 3755 if s.sy == '[': 3756 error(s.position(), "Name options not allowed for C++ class") 3757 nogil = p_nogil(s) 3758 if s.sy == ':': 3759 s.next() 3760 s.expect('NEWLINE') 3761 s.expect_indent() 3762 attributes = [] 3763 body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil) 3764 body_ctx.templates = template_names 3765 while s.sy != 'DEDENT': 3766 if s.sy != 'pass': 3767 attributes.append(p_cpp_class_attribute(s, body_ctx)) 3768 else: 3769 s.next() 3770 s.expect_newline("Expected a newline") 3771 s.expect_dedent() 3772 else: 3773 attributes = None 3774 s.expect_newline("Syntax error in C++ class definition") 3775 return Nodes.CppClassNode(pos, 3776 name = class_name, 3777 cname = cname, 3778 base_classes = base_classes, 3779 visibility = ctx.visibility, 3780 in_pxd = ctx.level == 'module_pxd', 3781 attributes = attributes, 3782 templates = templates) 3783 3784def p_cpp_class_attribute(s, ctx): 3785 decorators = None 3786 if s.sy == '@': 3787 decorators = p_decorators(s) 3788 if s.systring == 'cppclass': 3789 return p_cpp_class_definition(s, s.position(), ctx) 3790 elif s.systring == 'ctypedef': 3791 return p_ctypedef_statement(s, ctx) 3792 elif s.sy == 'IDENT' and s.systring in struct_enum_union: 3793 if s.systring != 'enum': 3794 return p_cpp_class_definition(s, s.position(), ctx) 3795 else: 3796 return p_struct_enum(s, s.position(), ctx) 3797 else: 3798 node = p_c_func_or_var_declaration(s, s.position(), ctx) 3799 if decorators is not None: 3800 tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode 3801 if ctx.allow_struct_enum_decorator: 3802 tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode 3803 if not isinstance(node, tup): 3804 s.error("Decorators can only be followed by functions or classes") 3805 node.decorators = decorators 3806 return node 3807 3808 3809#---------------------------------------------- 3810# 3811# Debugging 3812# 3813#---------------------------------------------- 3814 3815def print_parse_tree(f, node, level, key = None): 3816 ind = " " * level 3817 if node: 3818 f.write(ind) 3819 if key: 3820 f.write("%s: " % key) 3821 t = type(node) 3822 if t is tuple: 3823 f.write("(%s @ %s\n" % (node[0], node[1])) 3824 for i in range(2, len(node)): 3825 print_parse_tree(f, node[i], level+1) 3826 f.write("%s)\n" % ind) 3827 return 3828 elif isinstance(node, Nodes.Node): 3829 try: 3830 tag = node.tag 3831 except AttributeError: 3832 tag = node.__class__.__name__ 3833 f.write("%s @ %s\n" % (tag, node.pos)) 3834 for name, value in node.__dict__.items(): 3835 if name != 'tag' and name != 'pos': 3836 print_parse_tree(f, value, level+1, name) 3837 return 3838 elif t is list: 3839 f.write("[\n") 3840 for i in range(len(node)): 3841 print_parse_tree(f, node[i], level+1) 3842 f.write("%s]\n" % ind) 3843 return 3844 f.write("%s%s\n" % (ind, node)) 3845