1from __future__ import absolute_import 2 3import cython 4cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object, 5 Options=object, UtilNodes=object, LetNode=object, 6 LetRefNode=object, TreeFragment=object, EncodedString=object, 7 error=object, warning=object, copy=object, _unicode=object) 8 9import copy 10import hashlib 11 12from . import PyrexTypes 13from . import Naming 14from . import ExprNodes 15from . import Nodes 16from . import Options 17from . import Builtin 18from . import Errors 19 20from .Visitor import VisitorTransform, TreeVisitor 21from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform 22from .UtilNodes import LetNode, LetRefNode 23from .TreeFragment import TreeFragment 24from .StringEncoding import EncodedString, _unicode 25from .Errors import error, warning, CompileError, InternalError 26from .Code import UtilityCode 27 28 29class SkipDeclarations(object): 30 """ 31 Variable and function declarations can often have a deep tree structure, 32 and yet most transformations don't need to descend to this depth. 33 34 Declaration nodes are removed after AnalyseDeclarationsTransform, so there 35 is no need to use this for transformations after that point. 36 """ 37 def visit_CTypeDefNode(self, node): 38 return node 39 40 def visit_CVarDefNode(self, node): 41 return node 42 43 def visit_CDeclaratorNode(self, node): 44 return node 45 46 def visit_CBaseTypeNode(self, node): 47 return node 48 49 def visit_CEnumDefNode(self, node): 50 return node 51 52 def visit_CStructOrUnionDefNode(self, node): 53 return node 54 55 56class NormalizeTree(CythonTransform): 57 """ 58 This transform fixes up a few things after parsing 59 in order to make the parse tree more suitable for 60 transforms. 61 62 a) After parsing, blocks with only one statement will 63 be represented by that statement, not by a StatListNode. 64 When doing transforms this is annoying and inconsistent, 65 as one cannot in general remove a statement in a consistent 66 way and so on. This transform wraps any single statements 67 in a StatListNode containing a single statement. 68 69 b) The PassStatNode is a noop and serves no purpose beyond 70 plugging such one-statement blocks; i.e., once parsed a 71` "pass" can just as well be represented using an empty 72 StatListNode. This means less special cases to worry about 73 in subsequent transforms (one always checks to see if a 74 StatListNode has no children to see if the block is empty). 75 """ 76 77 def __init__(self, context): 78 super(NormalizeTree, self).__init__(context) 79 self.is_in_statlist = False 80 self.is_in_expr = False 81 82 def visit_ExprNode(self, node): 83 stacktmp = self.is_in_expr 84 self.is_in_expr = True 85 self.visitchildren(node) 86 self.is_in_expr = stacktmp 87 return node 88 89 def visit_StatNode(self, node, is_listcontainer=False): 90 stacktmp = self.is_in_statlist 91 self.is_in_statlist = is_listcontainer 92 self.visitchildren(node) 93 self.is_in_statlist = stacktmp 94 if not self.is_in_statlist and not self.is_in_expr: 95 return Nodes.StatListNode(pos=node.pos, stats=[node]) 96 else: 97 return node 98 99 def visit_StatListNode(self, node): 100 self.is_in_statlist = True 101 self.visitchildren(node) 102 self.is_in_statlist = False 103 return node 104 105 def visit_ParallelAssignmentNode(self, node): 106 return self.visit_StatNode(node, True) 107 108 def visit_CEnumDefNode(self, node): 109 return self.visit_StatNode(node, True) 110 111 def visit_CStructOrUnionDefNode(self, node): 112 return self.visit_StatNode(node, True) 113 114 def visit_PassStatNode(self, node): 115 """Eliminate PassStatNode""" 116 if not self.is_in_statlist: 117 return Nodes.StatListNode(pos=node.pos, stats=[]) 118 else: 119 return [] 120 121 def visit_ExprStatNode(self, node): 122 """Eliminate useless string literals""" 123 if node.expr.is_string_literal: 124 return self.visit_PassStatNode(node) 125 else: 126 return self.visit_StatNode(node) 127 128 def visit_CDeclaratorNode(self, node): 129 return node 130 131 132class PostParseError(CompileError): pass 133 134# error strings checked by unit tests, so define them 135ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions' 136ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)' 137ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared' 138class PostParse(ScopeTrackingTransform): 139 """ 140 Basic interpretation of the parse tree, as well as validity 141 checking that can be done on a very basic level on the parse 142 tree (while still not being a problem with the basic syntax, 143 as such). 144 145 Specifically: 146 - Default values to cdef assignments are turned into single 147 assignments following the declaration (everywhere but in class 148 bodies, where they raise a compile error) 149 150 - Interpret some node structures into Python runtime values. 151 Some nodes take compile-time arguments (currently: 152 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}), 153 which should be interpreted. This happens in a general way 154 and other steps should be taken to ensure validity. 155 156 Type arguments cannot be interpreted in this way. 157 158 - For __cythonbufferdefaults__ the arguments are checked for 159 validity. 160 161 TemplatedTypeNode has its directives interpreted: 162 Any first positional argument goes into the "dtype" attribute, 163 any "ndim" keyword argument goes into the "ndim" attribute and 164 so on. Also it is checked that the directive combination is valid. 165 - __cythonbufferdefaults__ attributes are parsed and put into the 166 type information. 167 168 Note: Currently Parsing.py does a lot of interpretation and 169 reorganization that can be refactored into this transform 170 if a more pure Abstract Syntax Tree is wanted. 171 """ 172 173 def __init__(self, context): 174 super(PostParse, self).__init__(context) 175 self.specialattribute_handlers = { 176 '__cythonbufferdefaults__' : self.handle_bufferdefaults 177 } 178 179 def visit_LambdaNode(self, node): 180 # unpack a lambda expression into the corresponding DefNode 181 collector = YieldNodeCollector() 182 collector.visitchildren(node.result_expr) 183 if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode): 184 body = Nodes.ExprStatNode( 185 node.result_expr.pos, expr=node.result_expr) 186 else: 187 body = Nodes.ReturnStatNode( 188 node.result_expr.pos, value=node.result_expr) 189 node.def_node = Nodes.DefNode( 190 node.pos, name=node.name, 191 args=node.args, star_arg=node.star_arg, 192 starstar_arg=node.starstar_arg, 193 body=body, doc=None) 194 self.visitchildren(node) 195 return node 196 197 def visit_GeneratorExpressionNode(self, node): 198 # unpack a generator expression into the corresponding DefNode 199 collector = YieldNodeCollector() 200 collector.visitchildren(node.loop) 201 node.def_node = Nodes.DefNode( 202 node.pos, name=node.name, doc=None, 203 args=[], star_arg=None, starstar_arg=None, 204 body=node.loop, is_async_def=collector.has_await) 205 self.visitchildren(node) 206 return node 207 208 def visit_ComprehensionNode(self, node): 209 # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...) 210 if not node.has_local_scope: 211 collector = YieldNodeCollector() 212 collector.visitchildren(node.loop) 213 if collector.has_await: 214 node.has_local_scope = True 215 self.visitchildren(node) 216 return node 217 218 # cdef variables 219 def handle_bufferdefaults(self, decl): 220 if not isinstance(decl.default, ExprNodes.DictNode): 221 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS) 222 self.scope_node.buffer_defaults_node = decl.default 223 self.scope_node.buffer_defaults_pos = decl.pos 224 225 def visit_CVarDefNode(self, node): 226 # This assumes only plain names and pointers are assignable on 227 # declaration. Also, it makes use of the fact that a cdef decl 228 # must appear before the first use, so we don't have to deal with 229 # "i = 3; cdef int i = i" and can simply move the nodes around. 230 try: 231 self.visitchildren(node) 232 stats = [node] 233 newdecls = [] 234 for decl in node.declarators: 235 declbase = decl 236 while isinstance(declbase, Nodes.CPtrDeclaratorNode): 237 declbase = declbase.base 238 if isinstance(declbase, Nodes.CNameDeclaratorNode): 239 if declbase.default is not None: 240 if self.scope_type in ('cclass', 'pyclass', 'struct'): 241 if isinstance(self.scope_node, Nodes.CClassDefNode): 242 handler = self.specialattribute_handlers.get(decl.name) 243 if handler: 244 if decl is not declbase: 245 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE) 246 handler(decl) 247 continue # Remove declaration 248 raise PostParseError(decl.pos, ERR_CDEF_INCLASS) 249 first_assignment = self.scope_type != 'module' 250 stats.append(Nodes.SingleAssignmentNode(node.pos, 251 lhs=ExprNodes.NameNode(node.pos, name=declbase.name), 252 rhs=declbase.default, first=first_assignment)) 253 declbase.default = None 254 newdecls.append(decl) 255 node.declarators = newdecls 256 return stats 257 except PostParseError as e: 258 # An error in a cdef clause is ok, simply remove the declaration 259 # and try to move on to report more errors 260 self.context.nonfatal_error(e) 261 return None 262 263 # Split parallel assignments (a,b = b,a) into separate partial 264 # assignments that are executed rhs-first using temps. This 265 # restructuring must be applied before type analysis so that known 266 # types on rhs and lhs can be matched directly. It is required in 267 # the case that the types cannot be coerced to a Python type in 268 # order to assign from a tuple. 269 270 def visit_SingleAssignmentNode(self, node): 271 self.visitchildren(node) 272 return self._visit_assignment_node(node, [node.lhs, node.rhs]) 273 274 def visit_CascadedAssignmentNode(self, node): 275 self.visitchildren(node) 276 return self._visit_assignment_node(node, node.lhs_list + [node.rhs]) 277 278 def _visit_assignment_node(self, node, expr_list): 279 """Flatten parallel assignments into separate single 280 assignments or cascaded assignments. 281 """ 282 if sum([ 1 for expr in expr_list 283 if expr.is_sequence_constructor or expr.is_string_literal ]) < 2: 284 # no parallel assignments => nothing to do 285 return node 286 287 expr_list_list = [] 288 flatten_parallel_assignments(expr_list, expr_list_list) 289 temp_refs = [] 290 eliminate_rhs_duplicates(expr_list_list, temp_refs) 291 292 nodes = [] 293 for expr_list in expr_list_list: 294 lhs_list = expr_list[:-1] 295 rhs = expr_list[-1] 296 if len(lhs_list) == 1: 297 node = Nodes.SingleAssignmentNode(rhs.pos, 298 lhs = lhs_list[0], rhs = rhs) 299 else: 300 node = Nodes.CascadedAssignmentNode(rhs.pos, 301 lhs_list = lhs_list, rhs = rhs) 302 nodes.append(node) 303 304 if len(nodes) == 1: 305 assign_node = nodes[0] 306 else: 307 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes) 308 309 if temp_refs: 310 duplicates_and_temps = [ (temp.expression, temp) 311 for temp in temp_refs ] 312 sort_common_subsequences(duplicates_and_temps) 313 for _, temp_ref in duplicates_and_temps[::-1]: 314 assign_node = LetNode(temp_ref, assign_node) 315 316 return assign_node 317 318 def _flatten_sequence(self, seq, result): 319 for arg in seq.args: 320 if arg.is_sequence_constructor: 321 self._flatten_sequence(arg, result) 322 else: 323 result.append(arg) 324 return result 325 326 def visit_DelStatNode(self, node): 327 self.visitchildren(node) 328 node.args = self._flatten_sequence(node, []) 329 return node 330 331 def visit_ExceptClauseNode(self, node): 332 if node.is_except_as: 333 # except-as must delete NameNode target at the end 334 del_target = Nodes.DelStatNode( 335 node.pos, 336 args=[ExprNodes.NameNode( 337 node.target.pos, name=node.target.name)], 338 ignore_nonexisting=True) 339 node.body = Nodes.StatListNode( 340 node.pos, 341 stats=[Nodes.TryFinallyStatNode( 342 node.pos, 343 body=node.body, 344 finally_clause=Nodes.StatListNode( 345 node.pos, 346 stats=[del_target]))]) 347 self.visitchildren(node) 348 return node 349 350 351def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence): 352 """Replace rhs items by LetRefNodes if they appear more than once. 353 Creates a sequence of LetRefNodes that set up the required temps 354 and appends them to ref_node_sequence. The input list is modified 355 in-place. 356 """ 357 seen_nodes = set() 358 ref_nodes = {} 359 def find_duplicates(node): 360 if node.is_literal or node.is_name: 361 # no need to replace those; can't include attributes here 362 # as their access is not necessarily side-effect free 363 return 364 if node in seen_nodes: 365 if node not in ref_nodes: 366 ref_node = LetRefNode(node) 367 ref_nodes[node] = ref_node 368 ref_node_sequence.append(ref_node) 369 else: 370 seen_nodes.add(node) 371 if node.is_sequence_constructor: 372 for item in node.args: 373 find_duplicates(item) 374 375 for expr_list in expr_list_list: 376 rhs = expr_list[-1] 377 find_duplicates(rhs) 378 if not ref_nodes: 379 return 380 381 def substitute_nodes(node): 382 if node in ref_nodes: 383 return ref_nodes[node] 384 elif node.is_sequence_constructor: 385 node.args = list(map(substitute_nodes, node.args)) 386 return node 387 388 # replace nodes inside of the common subexpressions 389 for node in ref_nodes: 390 if node.is_sequence_constructor: 391 node.args = list(map(substitute_nodes, node.args)) 392 393 # replace common subexpressions on all rhs items 394 for expr_list in expr_list_list: 395 expr_list[-1] = substitute_nodes(expr_list[-1]) 396 397def sort_common_subsequences(items): 398 """Sort items/subsequences so that all items and subsequences that 399 an item contains appear before the item itself. This is needed 400 because each rhs item must only be evaluated once, so its value 401 must be evaluated first and then reused when packing sequences 402 that contain it. 403 404 This implies a partial order, and the sort must be stable to 405 preserve the original order as much as possible, so we use a 406 simple insertion sort (which is very fast for short sequences, the 407 normal case in practice). 408 """ 409 def contains(seq, x): 410 for item in seq: 411 if item is x: 412 return True 413 elif item.is_sequence_constructor and contains(item.args, x): 414 return True 415 return False 416 def lower_than(a,b): 417 return b.is_sequence_constructor and contains(b.args, a) 418 419 for pos, item in enumerate(items): 420 key = item[1] # the ResultRefNode which has already been injected into the sequences 421 new_pos = pos 422 for i in range(pos-1, -1, -1): 423 if lower_than(key, items[i][0]): 424 new_pos = i 425 if new_pos != pos: 426 for i in range(pos, new_pos, -1): 427 items[i] = items[i-1] 428 items[new_pos] = item 429 430def unpack_string_to_character_literals(literal): 431 chars = [] 432 pos = literal.pos 433 stype = literal.__class__ 434 sval = literal.value 435 sval_type = sval.__class__ 436 for char in sval: 437 cval = sval_type(char) 438 chars.append(stype(pos, value=cval, constant_result=cval)) 439 return chars 440 441def flatten_parallel_assignments(input, output): 442 # The input is a list of expression nodes, representing the LHSs 443 # and RHS of one (possibly cascaded) assignment statement. For 444 # sequence constructors, rearranges the matching parts of both 445 # sides into a list of equivalent assignments between the 446 # individual elements. This transformation is applied 447 # recursively, so that nested structures get matched as well. 448 rhs = input[-1] 449 if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode)) 450 or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): 451 output.append(input) 452 return 453 454 complete_assignments = [] 455 456 if rhs.is_sequence_constructor: 457 rhs_args = rhs.args 458 elif rhs.is_string_literal: 459 rhs_args = unpack_string_to_character_literals(rhs) 460 461 rhs_size = len(rhs_args) 462 lhs_targets = [[] for _ in range(rhs_size)] 463 starred_assignments = [] 464 for lhs in input[:-1]: 465 if not lhs.is_sequence_constructor: 466 if lhs.is_starred: 467 error(lhs.pos, "starred assignment target must be in a list or tuple") 468 complete_assignments.append(lhs) 469 continue 470 lhs_size = len(lhs.args) 471 starred_targets = sum([1 for expr in lhs.args if expr.is_starred]) 472 if starred_targets > 1: 473 error(lhs.pos, "more than 1 starred expression in assignment") 474 output.append([lhs,rhs]) 475 continue 476 elif lhs_size - starred_targets > rhs_size: 477 error(lhs.pos, "need more than %d value%s to unpack" 478 % (rhs_size, (rhs_size != 1) and 's' or '')) 479 output.append([lhs,rhs]) 480 continue 481 elif starred_targets: 482 map_starred_assignment(lhs_targets, starred_assignments, 483 lhs.args, rhs_args) 484 elif lhs_size < rhs_size: 485 error(lhs.pos, "too many values to unpack (expected %d, got %d)" 486 % (lhs_size, rhs_size)) 487 output.append([lhs,rhs]) 488 continue 489 else: 490 for targets, expr in zip(lhs_targets, lhs.args): 491 targets.append(expr) 492 493 if complete_assignments: 494 complete_assignments.append(rhs) 495 output.append(complete_assignments) 496 497 # recursively flatten partial assignments 498 for cascade, rhs in zip(lhs_targets, rhs_args): 499 if cascade: 500 cascade.append(rhs) 501 flatten_parallel_assignments(cascade, output) 502 503 # recursively flatten starred assignments 504 for cascade in starred_assignments: 505 if cascade[0].is_sequence_constructor: 506 flatten_parallel_assignments(cascade, output) 507 else: 508 output.append(cascade) 509 510def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args): 511 # Appends the fixed-position LHS targets to the target list that 512 # appear left and right of the starred argument. 513 # 514 # The starred_assignments list receives a new tuple 515 # (lhs_target, rhs_values_list) that maps the remaining arguments 516 # (those that match the starred target) to a list. 517 518 # left side of the starred target 519 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)): 520 if expr.is_starred: 521 starred = i 522 lhs_remaining = len(lhs_args) - i - 1 523 break 524 targets.append(expr) 525 else: 526 raise InternalError("no starred arg found when splitting starred assignment") 527 528 # right side of the starred target 529 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:], 530 lhs_args[starred + 1:])): 531 targets.append(expr) 532 533 # the starred target itself, must be assigned a (potentially empty) list 534 target = lhs_args[starred].target # unpack starred node 535 starred_rhs = rhs_args[starred:] 536 if lhs_remaining: 537 starred_rhs = starred_rhs[:-lhs_remaining] 538 if starred_rhs: 539 pos = starred_rhs[0].pos 540 else: 541 pos = target.pos 542 starred_assignments.append([ 543 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)]) 544 545 546class PxdPostParse(CythonTransform, SkipDeclarations): 547 """ 548 Basic interpretation/validity checking that should only be 549 done on pxd trees. 550 551 A lot of this checking currently happens in the parser; but 552 what is listed below happens here. 553 554 - "def" functions are let through only if they fill the 555 getbuffer/releasebuffer slots 556 557 - cdef functions are let through only if they are on the 558 top level and are declared "inline" 559 """ 560 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'" 561 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'" 562 563 def __call__(self, node): 564 self.scope_type = 'pxd' 565 return super(PxdPostParse, self).__call__(node) 566 567 def visit_CClassDefNode(self, node): 568 old = self.scope_type 569 self.scope_type = 'cclass' 570 self.visitchildren(node) 571 self.scope_type = old 572 return node 573 574 def visit_FuncDefNode(self, node): 575 # FuncDefNode always come with an implementation (without 576 # an imp they are CVarDefNodes..) 577 err = self.ERR_INLINE_ONLY 578 579 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass' 580 and node.name in ('__getbuffer__', '__releasebuffer__')): 581 err = None # allow these slots 582 583 if isinstance(node, Nodes.CFuncDefNode): 584 if (u'inline' in node.modifiers and 585 self.scope_type in ('pxd', 'cclass')): 586 node.inline_in_pxd = True 587 if node.visibility != 'private': 588 err = self.ERR_NOGO_WITH_INLINE % node.visibility 589 elif node.api: 590 err = self.ERR_NOGO_WITH_INLINE % 'api' 591 else: 592 err = None # allow inline function 593 else: 594 err = self.ERR_INLINE_ONLY 595 596 if err: 597 self.context.nonfatal_error(PostParseError(node.pos, err)) 598 return None 599 else: 600 return node 601 602 603class TrackNumpyAttributes(VisitorTransform, SkipDeclarations): 604 # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two. 605 def __init__(self): 606 super(TrackNumpyAttributes, self).__init__() 607 self.numpy_module_names = set() 608 609 def visit_CImportStatNode(self, node): 610 if node.module_name == u"numpy": 611 self.numpy_module_names.add(node.as_name or u"numpy") 612 return node 613 614 def visit_AttributeNode(self, node): 615 self.visitchildren(node) 616 obj = node.obj 617 if (obj.is_name and obj.name in self.numpy_module_names) or obj.is_numpy_attribute: 618 node.is_numpy_attribute = True 619 return node 620 621 visit_Node = VisitorTransform.recurse_to_children 622 623 624class InterpretCompilerDirectives(CythonTransform): 625 """ 626 After parsing, directives can be stored in a number of places: 627 - #cython-comments at the top of the file (stored in ModuleNode) 628 - Command-line arguments overriding these 629 - @cython.directivename decorators 630 - with cython.directivename: statements 631 632 This transform is responsible for interpreting these various sources 633 and store the directive in two ways: 634 - Set the directives attribute of the ModuleNode for global directives. 635 - Use a CompilerDirectivesNode to override directives for a subtree. 636 637 (The first one is primarily to not have to modify with the tree 638 structure, so that ModuleNode stay on top.) 639 640 The directives are stored in dictionaries from name to value in effect. 641 Each such dictionary is always filled in for all possible directives, 642 using default values where no value is given by the user. 643 644 The available directives are controlled in Options.py. 645 646 Note that we have to run this prior to analysis, and so some minor 647 duplication of functionality has to occur: We manually track cimports 648 and which names the "cython" module may have been imported to. 649 """ 650 unop_method_nodes = { 651 'typeof': ExprNodes.TypeofNode, 652 653 'operator.address': ExprNodes.AmpersandNode, 654 'operator.dereference': ExprNodes.DereferenceNode, 655 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'), 656 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'), 657 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'), 658 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'), 659 'operator.typeid' : ExprNodes.TypeidNode, 660 661 # For backwards compatibility. 662 'address': ExprNodes.AmpersandNode, 663 } 664 665 binop_method_nodes = { 666 'operator.comma' : ExprNodes.c_binop_constructor(','), 667 } 668 669 special_methods = set(['declare', 'union', 'struct', 'typedef', 670 'sizeof', 'cast', 'pointer', 'compiled', 671 'NULL', 'fused_type', 'parallel']) 672 special_methods.update(unop_method_nodes) 673 674 valid_parallel_directives = set([ 675 "parallel", 676 "prange", 677 "threadid", 678 #"threadsavailable", 679 ]) 680 681 def __init__(self, context, compilation_directive_defaults): 682 super(InterpretCompilerDirectives, self).__init__(context) 683 self.cython_module_names = set() 684 self.directive_names = {'staticmethod': 'staticmethod'} 685 self.parallel_directives = {} 686 directives = copy.deepcopy(Options.get_directive_defaults()) 687 for key, value in compilation_directive_defaults.items(): 688 directives[_unicode(key)] = copy.deepcopy(value) 689 self.directives = directives 690 691 def check_directive_scope(self, pos, directive, scope): 692 legal_scopes = Options.directive_scopes.get(directive, None) 693 if legal_scopes and scope not in legal_scopes: 694 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive ' 695 'is not allowed in %s scope' % (directive, scope))) 696 return False 697 else: 698 if directive not in Options.directive_types: 699 error(pos, "Invalid directive: '%s'." % (directive,)) 700 return True 701 702 # Set up processing and handle the cython: comments. 703 def visit_ModuleNode(self, node): 704 for key in sorted(node.directive_comments): 705 if not self.check_directive_scope(node.pos, key, 'module'): 706 self.wrong_scope_error(node.pos, key, 'module') 707 del node.directive_comments[key] 708 709 self.module_scope = node.scope 710 711 self.directives.update(node.directive_comments) 712 node.directives = self.directives 713 node.parallel_directives = self.parallel_directives 714 self.visitchildren(node) 715 node.cython_module_names = self.cython_module_names 716 return node 717 718 # The following four functions track imports and cimports that 719 # begin with "cython" 720 def is_cython_directive(self, name): 721 return (name in Options.directive_types or 722 name in self.special_methods or 723 PyrexTypes.parse_basic_type(name)) 724 725 def is_parallel_directive(self, full_name, pos): 726 """ 727 Checks to see if fullname (e.g. cython.parallel.prange) is a valid 728 parallel directive. If it is a star import it also updates the 729 parallel_directives. 730 """ 731 result = (full_name + ".").startswith("cython.parallel.") 732 733 if result: 734 directive = full_name.split('.') 735 if full_name == u"cython.parallel": 736 self.parallel_directives[u"parallel"] = u"cython.parallel" 737 elif full_name == u"cython.parallel.*": 738 for name in self.valid_parallel_directives: 739 self.parallel_directives[name] = u"cython.parallel.%s" % name 740 elif (len(directive) != 3 or 741 directive[-1] not in self.valid_parallel_directives): 742 error(pos, "No such directive: %s" % full_name) 743 744 self.module_scope.use_utility_code( 745 UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c")) 746 747 return result 748 749 def visit_CImportStatNode(self, node): 750 if node.module_name == u"cython": 751 self.cython_module_names.add(node.as_name or u"cython") 752 elif node.module_name.startswith(u"cython."): 753 if node.module_name.startswith(u"cython.parallel."): 754 error(node.pos, node.module_name + " is not a module") 755 if node.module_name == u"cython.parallel": 756 if node.as_name and node.as_name != u"cython": 757 self.parallel_directives[node.as_name] = node.module_name 758 else: 759 self.cython_module_names.add(u"cython") 760 self.parallel_directives[ 761 u"cython.parallel"] = node.module_name 762 self.module_scope.use_utility_code( 763 UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c")) 764 elif node.as_name: 765 self.directive_names[node.as_name] = node.module_name[7:] 766 else: 767 self.cython_module_names.add(u"cython") 768 # if this cimport was a compiler directive, we don't 769 # want to leave the cimport node sitting in the tree 770 return None 771 return node 772 773 def visit_FromCImportStatNode(self, node): 774 if not node.relative_level and ( 775 node.module_name == u"cython" or node.module_name.startswith(u"cython.")): 776 submodule = (node.module_name + u".")[7:] 777 newimp = [] 778 779 for pos, name, as_name, kind in node.imported_names: 780 full_name = submodule + name 781 qualified_name = u"cython." + full_name 782 783 if self.is_parallel_directive(qualified_name, node.pos): 784 # from cython cimport parallel, or 785 # from cython.parallel cimport parallel, prange, ... 786 self.parallel_directives[as_name or name] = qualified_name 787 elif self.is_cython_directive(full_name): 788 self.directive_names[as_name or name] = full_name 789 if kind is not None: 790 self.context.nonfatal_error(PostParseError(pos, 791 "Compiler directive imports must be plain imports")) 792 else: 793 newimp.append((pos, name, as_name, kind)) 794 795 if not newimp: 796 return None 797 798 node.imported_names = newimp 799 return node 800 801 def visit_FromImportStatNode(self, node): 802 if (node.module.module_name.value == u"cython") or \ 803 node.module.module_name.value.startswith(u"cython."): 804 submodule = (node.module.module_name.value + u".")[7:] 805 newimp = [] 806 for name, name_node in node.items: 807 full_name = submodule + name 808 qualified_name = u"cython." + full_name 809 if self.is_parallel_directive(qualified_name, node.pos): 810 self.parallel_directives[name_node.name] = qualified_name 811 elif self.is_cython_directive(full_name): 812 self.directive_names[name_node.name] = full_name 813 else: 814 newimp.append((name, name_node)) 815 if not newimp: 816 return None 817 node.items = newimp 818 return node 819 820 def visit_SingleAssignmentNode(self, node): 821 if isinstance(node.rhs, ExprNodes.ImportNode): 822 module_name = node.rhs.module_name.value 823 is_parallel = (module_name + u".").startswith(u"cython.parallel.") 824 825 if module_name != u"cython" and not is_parallel: 826 return node 827 828 module_name = node.rhs.module_name.value 829 as_name = node.lhs.name 830 831 node = Nodes.CImportStatNode(node.pos, 832 module_name = module_name, 833 as_name = as_name) 834 node = self.visit_CImportStatNode(node) 835 else: 836 self.visitchildren(node) 837 838 return node 839 840 def visit_NameNode(self, node): 841 if node.name in self.cython_module_names: 842 node.is_cython_module = True 843 else: 844 directive = self.directive_names.get(node.name) 845 if directive is not None: 846 node.cython_attribute = directive 847 return node 848 849 def visit_NewExprNode(self, node): 850 self.visit(node.cppclass) 851 self.visitchildren(node) 852 return node 853 854 def try_to_parse_directives(self, node): 855 # If node is the contents of an directive (in a with statement or 856 # decorator), returns a list of (directivename, value) pairs. 857 # Otherwise, returns None 858 if isinstance(node, ExprNodes.CallNode): 859 self.visit(node.function) 860 optname = node.function.as_cython_attribute() 861 if optname: 862 directivetype = Options.directive_types.get(optname) 863 if directivetype: 864 args, kwds = node.explicit_args_kwds() 865 directives = [] 866 key_value_pairs = [] 867 if kwds is not None and directivetype is not dict: 868 for keyvalue in kwds.key_value_pairs: 869 key, value = keyvalue 870 sub_optname = "%s.%s" % (optname, key.value) 871 if Options.directive_types.get(sub_optname): 872 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos)) 873 else: 874 key_value_pairs.append(keyvalue) 875 if not key_value_pairs: 876 kwds = None 877 else: 878 kwds.key_value_pairs = key_value_pairs 879 if directives and not kwds and not args: 880 return directives 881 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos)) 882 return directives 883 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)): 884 self.visit(node) 885 optname = node.as_cython_attribute() 886 if optname: 887 directivetype = Options.directive_types.get(optname) 888 if directivetype is bool: 889 arg = ExprNodes.BoolNode(node.pos, value=True) 890 return [self.try_to_parse_directive(optname, [arg], None, node.pos)] 891 elif directivetype is None: 892 return [(optname, None)] 893 else: 894 raise PostParseError( 895 node.pos, "The '%s' directive should be used as a function call." % optname) 896 return None 897 898 def try_to_parse_directive(self, optname, args, kwds, pos): 899 if optname == 'np_pythran' and not self.context.cpp: 900 raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname) 901 elif optname == 'exceptval': 902 # default: exceptval(None, check=True) 903 arg_error = len(args) > 1 904 check = True 905 if kwds and kwds.key_value_pairs: 906 kw = kwds.key_value_pairs[0] 907 if (len(kwds.key_value_pairs) == 1 and 908 kw.key.is_string_literal and kw.key.value == 'check' and 909 isinstance(kw.value, ExprNodes.BoolNode)): 910 check = kw.value.value 911 else: 912 arg_error = True 913 if arg_error: 914 raise PostParseError( 915 pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"') 916 return ('exceptval', (args[0] if args else None, check)) 917 918 directivetype = Options.directive_types.get(optname) 919 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode): 920 return optname, Options.get_directive_defaults()[optname] 921 elif directivetype is bool: 922 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode): 923 raise PostParseError(pos, 924 'The %s directive takes one compile-time boolean argument' % optname) 925 return (optname, args[0].value) 926 elif directivetype is int: 927 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode): 928 raise PostParseError(pos, 929 'The %s directive takes one compile-time integer argument' % optname) 930 return (optname, int(args[0].value)) 931 elif directivetype is str: 932 if kwds is not None or len(args) != 1 or not isinstance( 933 args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)): 934 raise PostParseError(pos, 935 'The %s directive takes one compile-time string argument' % optname) 936 return (optname, str(args[0].value)) 937 elif directivetype is type: 938 if kwds is not None or len(args) != 1: 939 raise PostParseError(pos, 940 'The %s directive takes one type argument' % optname) 941 return (optname, args[0]) 942 elif directivetype is dict: 943 if len(args) != 0: 944 raise PostParseError(pos, 945 'The %s directive takes no prepositional arguments' % optname) 946 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs]) 947 elif directivetype is list: 948 if kwds and len(kwds.key_value_pairs) != 0: 949 raise PostParseError(pos, 950 'The %s directive takes no keyword arguments' % optname) 951 return optname, [ str(arg.value) for arg in args ] 952 elif callable(directivetype): 953 if kwds is not None or len(args) != 1 or not isinstance( 954 args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)): 955 raise PostParseError(pos, 956 'The %s directive takes one compile-time string argument' % optname) 957 return (optname, directivetype(optname, str(args[0].value))) 958 else: 959 assert False 960 961 def visit_with_directives(self, node, directives): 962 if not directives: 963 return self.visit_Node(node) 964 965 old_directives = self.directives 966 new_directives = dict(old_directives) 967 new_directives.update(directives) 968 969 if new_directives == old_directives: 970 return self.visit_Node(node) 971 972 self.directives = new_directives 973 retbody = self.visit_Node(node) 974 self.directives = old_directives 975 976 if not isinstance(retbody, Nodes.StatListNode): 977 retbody = Nodes.StatListNode(node.pos, stats=[retbody]) 978 return Nodes.CompilerDirectivesNode( 979 pos=retbody.pos, body=retbody, directives=new_directives) 980 981 # Handle decorators 982 def visit_FuncDefNode(self, node): 983 directives = self._extract_directives(node, 'function') 984 return self.visit_with_directives(node, directives) 985 986 def visit_CVarDefNode(self, node): 987 directives = self._extract_directives(node, 'function') 988 for name, value in directives.items(): 989 if name == 'locals': 990 node.directive_locals = value 991 elif name not in ('final', 'staticmethod'): 992 self.context.nonfatal_error(PostParseError( 993 node.pos, 994 "Cdef functions can only take cython.locals(), " 995 "staticmethod, or final decorators, got %s." % name)) 996 return self.visit_with_directives(node, directives) 997 998 def visit_CClassDefNode(self, node): 999 directives = self._extract_directives(node, 'cclass') 1000 return self.visit_with_directives(node, directives) 1001 1002 def visit_CppClassNode(self, node): 1003 directives = self._extract_directives(node, 'cppclass') 1004 return self.visit_with_directives(node, directives) 1005 1006 def visit_PyClassDefNode(self, node): 1007 directives = self._extract_directives(node, 'class') 1008 return self.visit_with_directives(node, directives) 1009 1010 def _extract_directives(self, node, scope_name): 1011 if not node.decorators: 1012 return {} 1013 # Split the decorators into two lists -- real decorators and directives 1014 directives = [] 1015 realdecs = [] 1016 both = [] 1017 # Decorators coming first take precedence. 1018 for dec in node.decorators[::-1]: 1019 new_directives = self.try_to_parse_directives(dec.decorator) 1020 if new_directives is not None: 1021 for directive in new_directives: 1022 if self.check_directive_scope(node.pos, directive[0], scope_name): 1023 name, value = directive 1024 if self.directives.get(name, object()) != value: 1025 directives.append(directive) 1026 if directive[0] == 'staticmethod': 1027 both.append(dec) 1028 # Adapt scope type based on decorators that change it. 1029 if directive[0] == 'cclass' and scope_name == 'class': 1030 scope_name = 'cclass' 1031 else: 1032 realdecs.append(dec) 1033 if realdecs and (scope_name == 'cclass' or 1034 isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): 1035 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.") 1036 node.decorators = realdecs[::-1] + both[::-1] 1037 # merge or override repeated directives 1038 optdict = {} 1039 for directive in directives: 1040 name, value = directive 1041 if name in optdict: 1042 old_value = optdict[name] 1043 # keywords and arg lists can be merged, everything 1044 # else overrides completely 1045 if isinstance(old_value, dict): 1046 old_value.update(value) 1047 elif isinstance(old_value, list): 1048 old_value.extend(value) 1049 else: 1050 optdict[name] = value 1051 else: 1052 optdict[name] = value 1053 return optdict 1054 1055 # Handle with-statements 1056 def visit_WithStatNode(self, node): 1057 directive_dict = {} 1058 for directive in self.try_to_parse_directives(node.manager) or []: 1059 if directive is not None: 1060 if node.target is not None: 1061 self.context.nonfatal_error( 1062 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'")) 1063 else: 1064 name, value = directive 1065 if name in ('nogil', 'gil'): 1066 # special case: in pure mode, "with nogil" spells "with cython.nogil" 1067 node = Nodes.GILStatNode(node.pos, state = name, body = node.body) 1068 return self.visit_Node(node) 1069 if self.check_directive_scope(node.pos, name, 'with statement'): 1070 directive_dict[name] = value 1071 if directive_dict: 1072 return self.visit_with_directives(node.body, directive_dict) 1073 return self.visit_Node(node) 1074 1075 1076class ParallelRangeTransform(CythonTransform, SkipDeclarations): 1077 """ 1078 Transform cython.parallel stuff. The parallel_directives come from the 1079 module node, set there by InterpretCompilerDirectives. 1080 1081 x = cython.parallel.threadavailable() -> ParallelThreadAvailableNode 1082 with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode 1083 print cython.parallel.threadid() -> ParallelThreadIdNode 1084 for i in cython.parallel.prange(...): -> ParallelRangeNode 1085 ... 1086 """ 1087 1088 # a list of names, maps 'cython.parallel.prange' in the code to 1089 # ['cython', 'parallel', 'prange'] 1090 parallel_directive = None 1091 1092 # Indicates whether a namenode in an expression is the cython module 1093 namenode_is_cython_module = False 1094 1095 # Keep track of whether we are the context manager of a 'with' statement 1096 in_context_manager_section = False 1097 1098 # One of 'prange' or 'with parallel'. This is used to disallow closely 1099 # nested 'with parallel:' blocks 1100 state = None 1101 1102 directive_to_node = { 1103 u"cython.parallel.parallel": Nodes.ParallelWithBlockNode, 1104 # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode, 1105 u"cython.parallel.threadid": ExprNodes.ParallelThreadIdNode, 1106 u"cython.parallel.prange": Nodes.ParallelRangeNode, 1107 } 1108 1109 def node_is_parallel_directive(self, node): 1110 return node.name in self.parallel_directives or node.is_cython_module 1111 1112 def get_directive_class_node(self, node): 1113 """ 1114 Figure out which parallel directive was used and return the associated 1115 Node class. 1116 1117 E.g. for a cython.parallel.prange() call we return ParallelRangeNode 1118 """ 1119 if self.namenode_is_cython_module: 1120 directive = '.'.join(self.parallel_directive) 1121 else: 1122 directive = self.parallel_directives[self.parallel_directive[0]] 1123 directive = '%s.%s' % (directive, 1124 '.'.join(self.parallel_directive[1:])) 1125 directive = directive.rstrip('.') 1126 1127 cls = self.directive_to_node.get(directive) 1128 if cls is None and not (self.namenode_is_cython_module and 1129 self.parallel_directive[0] != 'parallel'): 1130 error(node.pos, "Invalid directive: %s" % directive) 1131 1132 self.namenode_is_cython_module = False 1133 self.parallel_directive = None 1134 1135 return cls 1136 1137 def visit_ModuleNode(self, node): 1138 """ 1139 If any parallel directives were imported, copy them over and visit 1140 the AST 1141 """ 1142 if node.parallel_directives: 1143 self.parallel_directives = node.parallel_directives 1144 return self.visit_Node(node) 1145 1146 # No parallel directives were imported, so they can't be used :) 1147 return node 1148 1149 def visit_NameNode(self, node): 1150 if self.node_is_parallel_directive(node): 1151 self.parallel_directive = [node.name] 1152 self.namenode_is_cython_module = node.is_cython_module 1153 return node 1154 1155 def visit_AttributeNode(self, node): 1156 self.visitchildren(node) 1157 if self.parallel_directive: 1158 self.parallel_directive.append(node.attribute) 1159 return node 1160 1161 def visit_CallNode(self, node): 1162 self.visit(node.function) 1163 if not self.parallel_directive: 1164 self.visitchildren(node, exclude=('function',)) 1165 return node 1166 1167 # We are a parallel directive, replace this node with the 1168 # corresponding ParallelSomethingSomething node 1169 1170 if isinstance(node, ExprNodes.GeneralCallNode): 1171 args = node.positional_args.args 1172 kwargs = node.keyword_args 1173 else: 1174 args = node.args 1175 kwargs = {} 1176 1177 parallel_directive_class = self.get_directive_class_node(node) 1178 if parallel_directive_class: 1179 # Note: in case of a parallel() the body is set by 1180 # visit_WithStatNode 1181 node = parallel_directive_class(node.pos, args=args, kwargs=kwargs) 1182 1183 return node 1184 1185 def visit_WithStatNode(self, node): 1186 "Rewrite with cython.parallel.parallel() blocks" 1187 newnode = self.visit(node.manager) 1188 1189 if isinstance(newnode, Nodes.ParallelWithBlockNode): 1190 if self.state == 'parallel with': 1191 error(node.manager.pos, 1192 "Nested parallel with blocks are disallowed") 1193 1194 self.state = 'parallel with' 1195 body = self.visit(node.body) 1196 self.state = None 1197 1198 newnode.body = body 1199 return newnode 1200 elif self.parallel_directive: 1201 parallel_directive_class = self.get_directive_class_node(node) 1202 1203 if not parallel_directive_class: 1204 # There was an error, stop here and now 1205 return None 1206 1207 if parallel_directive_class is Nodes.ParallelWithBlockNode: 1208 error(node.pos, "The parallel directive must be called") 1209 return None 1210 1211 node.body = self.visit(node.body) 1212 return node 1213 1214 def visit_ForInStatNode(self, node): 1215 "Rewrite 'for i in cython.parallel.prange(...):'" 1216 self.visit(node.iterator) 1217 self.visit(node.target) 1218 1219 in_prange = isinstance(node.iterator.sequence, 1220 Nodes.ParallelRangeNode) 1221 previous_state = self.state 1222 1223 if in_prange: 1224 # This will replace the entire ForInStatNode, so copy the 1225 # attributes 1226 parallel_range_node = node.iterator.sequence 1227 1228 parallel_range_node.target = node.target 1229 parallel_range_node.body = node.body 1230 parallel_range_node.else_clause = node.else_clause 1231 1232 node = parallel_range_node 1233 1234 if not isinstance(node.target, ExprNodes.NameNode): 1235 error(node.target.pos, 1236 "Can only iterate over an iteration variable") 1237 1238 self.state = 'prange' 1239 1240 self.visit(node.body) 1241 self.state = previous_state 1242 self.visit(node.else_clause) 1243 return node 1244 1245 def visit(self, node): 1246 "Visit a node that may be None" 1247 if node is not None: 1248 return super(ParallelRangeTransform, self).visit(node) 1249 1250 1251class WithTransform(CythonTransform, SkipDeclarations): 1252 def visit_WithStatNode(self, node): 1253 self.visitchildren(node, 'body') 1254 pos = node.pos 1255 is_async = node.is_async 1256 body, target, manager = node.body, node.target, node.manager 1257 node.enter_call = ExprNodes.SimpleCallNode( 1258 pos, function=ExprNodes.AttributeNode( 1259 pos, obj=ExprNodes.CloneNode(manager), 1260 attribute=EncodedString('__aenter__' if is_async else '__enter__'), 1261 is_special_lookup=True), 1262 args=[], 1263 is_temp=True) 1264 1265 if is_async: 1266 node.enter_call = ExprNodes.AwaitExprNode(pos, arg=node.enter_call) 1267 1268 if target is not None: 1269 body = Nodes.StatListNode( 1270 pos, stats=[ 1271 Nodes.WithTargetAssignmentStatNode( 1272 pos, lhs=target, with_node=node), 1273 body]) 1274 1275 excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[ 1276 ExprNodes.ExcValueNode(pos) for _ in range(3)]) 1277 except_clause = Nodes.ExceptClauseNode( 1278 pos, body=Nodes.IfStatNode( 1279 pos, if_clauses=[ 1280 Nodes.IfClauseNode( 1281 pos, condition=ExprNodes.NotNode( 1282 pos, operand=ExprNodes.WithExitCallNode( 1283 pos, with_stat=node, 1284 test_if_run=False, 1285 args=excinfo_target, 1286 await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), 1287 body=Nodes.ReraiseStatNode(pos), 1288 ), 1289 ], 1290 else_clause=None), 1291 pattern=None, 1292 target=None, 1293 excinfo_target=excinfo_target, 1294 ) 1295 1296 node.body = Nodes.TryFinallyStatNode( 1297 pos, body=Nodes.TryExceptStatNode( 1298 pos, body=body, 1299 except_clauses=[except_clause], 1300 else_clause=None, 1301 ), 1302 finally_clause=Nodes.ExprStatNode( 1303 pos, expr=ExprNodes.WithExitCallNode( 1304 pos, with_stat=node, 1305 test_if_run=True, 1306 args=ExprNodes.TupleNode( 1307 pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]), 1308 await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), 1309 handle_error_case=False, 1310 ) 1311 return node 1312 1313 def visit_ExprNode(self, node): 1314 # With statements are never inside expressions. 1315 return node 1316 1317 1318class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): 1319 """ 1320 Transforms method decorators in cdef classes into nested calls or properties. 1321 1322 Python-style decorator properties are transformed into a PropertyNode 1323 with up to the three getter, setter and deleter DefNodes. 1324 The functional style isn't supported yet. 1325 """ 1326 _properties = None 1327 1328 _map_property_attribute = { 1329 'getter': '__get__', 1330 'setter': '__set__', 1331 'deleter': '__del__', 1332 }.get 1333 1334 def visit_CClassDefNode(self, node): 1335 if self._properties is None: 1336 self._properties = [] 1337 self._properties.append({}) 1338 super(DecoratorTransform, self).visit_CClassDefNode(node) 1339 self._properties.pop() 1340 return node 1341 1342 def visit_PropertyNode(self, node): 1343 # Low-level warning for other code until we can convert all our uses over. 1344 level = 2 if isinstance(node.pos[0], str) else 0 1345 warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level) 1346 return node 1347 1348 def visit_DefNode(self, node): 1349 scope_type = self.scope_type 1350 node = self.visit_FuncDefNode(node) 1351 if scope_type != 'cclass' or not node.decorators: 1352 return node 1353 1354 # transform @property decorators 1355 properties = self._properties[-1] 1356 for decorator_node in node.decorators[::-1]: 1357 decorator = decorator_node.decorator 1358 if decorator.is_name and decorator.name == 'property': 1359 if len(node.decorators) > 1: 1360 return self._reject_decorated_property(node, decorator_node) 1361 name = node.name 1362 node.name = EncodedString('__get__') 1363 node.decorators.remove(decorator_node) 1364 stat_list = [node] 1365 if name in properties: 1366 prop = properties[name] 1367 prop.pos = node.pos 1368 prop.doc = node.doc 1369 prop.body.stats = stat_list 1370 return [] 1371 prop = Nodes.PropertyNode(node.pos, name=name) 1372 prop.doc = node.doc 1373 prop.body = Nodes.StatListNode(node.pos, stats=stat_list) 1374 properties[name] = prop 1375 return [prop] 1376 elif decorator.is_attribute and decorator.obj.name in properties: 1377 handler_name = self._map_property_attribute(decorator.attribute) 1378 if handler_name: 1379 if decorator.obj.name != node.name: 1380 # CPython does not generate an error or warning, but not something useful either. 1381 error(decorator_node.pos, 1382 "Mismatching property names, expected '%s', got '%s'" % ( 1383 decorator.obj.name, node.name)) 1384 elif len(node.decorators) > 1: 1385 return self._reject_decorated_property(node, decorator_node) 1386 else: 1387 return self._add_to_property(properties, node, handler_name, decorator_node) 1388 1389 # we clear node.decorators, so we need to set the 1390 # is_staticmethod/is_classmethod attributes now 1391 for decorator in node.decorators: 1392 func = decorator.decorator 1393 if func.is_name: 1394 node.is_classmethod |= func.name == 'classmethod' 1395 node.is_staticmethod |= func.name == 'staticmethod' 1396 1397 # transform normal decorators 1398 decs = node.decorators 1399 node.decorators = None 1400 return self.chain_decorators(node, decs, node.name) 1401 1402 @staticmethod 1403 def _reject_decorated_property(node, decorator_node): 1404 # restrict transformation to outermost decorator as wrapped properties will probably not work 1405 for deco in node.decorators: 1406 if deco != decorator_node: 1407 error(deco.pos, "Property methods with additional decorators are not supported") 1408 return node 1409 1410 @staticmethod 1411 def _add_to_property(properties, node, name, decorator): 1412 prop = properties[node.name] 1413 node.name = name 1414 node.decorators.remove(decorator) 1415 stats = prop.body.stats 1416 for i, stat in enumerate(stats): 1417 if stat.name == name: 1418 stats[i] = node 1419 break 1420 else: 1421 stats.append(node) 1422 return [] 1423 1424 @staticmethod 1425 def chain_decorators(node, decorators, name): 1426 """ 1427 Decorators are applied directly in DefNode and PyClassDefNode to avoid 1428 reassignments to the function/class name - except for cdef class methods. 1429 For those, the reassignment is required as methods are originally 1430 defined in the PyMethodDef struct. 1431 1432 The IndirectionNode allows DefNode to override the decorator. 1433 """ 1434 decorator_result = ExprNodes.NameNode(node.pos, name=name) 1435 for decorator in decorators[::-1]: 1436 decorator_result = ExprNodes.SimpleCallNode( 1437 decorator.pos, 1438 function=decorator.decorator, 1439 args=[decorator_result]) 1440 1441 name_node = ExprNodes.NameNode(node.pos, name=name) 1442 reassignment = Nodes.SingleAssignmentNode( 1443 node.pos, 1444 lhs=name_node, 1445 rhs=decorator_result) 1446 1447 reassignment = Nodes.IndirectionNode([reassignment]) 1448 node.decorator_indirection = reassignment 1449 return [node, reassignment] 1450 1451 1452class CnameDirectivesTransform(CythonTransform, SkipDeclarations): 1453 """ 1454 Only part of the CythonUtilityCode pipeline. Must be run before 1455 DecoratorTransform in case this is a decorator for a cdef class. 1456 It filters out @cname('my_cname') decorators and rewrites them to 1457 CnameDecoratorNodes. 1458 """ 1459 1460 def handle_function(self, node): 1461 if not getattr(node, 'decorators', None): 1462 return self.visit_Node(node) 1463 1464 for i, decorator in enumerate(node.decorators): 1465 decorator = decorator.decorator 1466 1467 if (isinstance(decorator, ExprNodes.CallNode) and 1468 decorator.function.is_name and 1469 decorator.function.name == 'cname'): 1470 args, kwargs = decorator.explicit_args_kwds() 1471 1472 if kwargs: 1473 raise AssertionError( 1474 "cname decorator does not take keyword arguments") 1475 1476 if len(args) != 1: 1477 raise AssertionError( 1478 "cname decorator takes exactly one argument") 1479 1480 if not (args[0].is_literal and 1481 args[0].type == Builtin.str_type): 1482 raise AssertionError( 1483 "argument to cname decorator must be a string literal") 1484 1485 cname = args[0].compile_time_value(None) 1486 del node.decorators[i] 1487 node = Nodes.CnameDecoratorNode(pos=node.pos, node=node, 1488 cname=cname) 1489 break 1490 1491 return self.visit_Node(node) 1492 1493 visit_FuncDefNode = handle_function 1494 visit_CClassDefNode = handle_function 1495 visit_CEnumDefNode = handle_function 1496 visit_CStructOrUnionDefNode = handle_function 1497 1498 1499class ForwardDeclareTypes(CythonTransform): 1500 1501 def visit_CompilerDirectivesNode(self, node): 1502 env = self.module_scope 1503 old = env.directives 1504 env.directives = node.directives 1505 self.visitchildren(node) 1506 env.directives = old 1507 return node 1508 1509 def visit_ModuleNode(self, node): 1510 self.module_scope = node.scope 1511 self.module_scope.directives = node.directives 1512 self.visitchildren(node) 1513 return node 1514 1515 def visit_CDefExternNode(self, node): 1516 old_cinclude_flag = self.module_scope.in_cinclude 1517 self.module_scope.in_cinclude = 1 1518 self.visitchildren(node) 1519 self.module_scope.in_cinclude = old_cinclude_flag 1520 return node 1521 1522 def visit_CEnumDefNode(self, node): 1523 node.declare(self.module_scope) 1524 return node 1525 1526 def visit_CStructOrUnionDefNode(self, node): 1527 if node.name not in self.module_scope.entries: 1528 node.declare(self.module_scope) 1529 return node 1530 1531 def visit_CClassDefNode(self, node): 1532 if node.class_name not in self.module_scope.entries: 1533 node.declare(self.module_scope) 1534 # Expand fused methods of .pxd declared types to construct the final vtable order. 1535 type = self.module_scope.entries[node.class_name].type 1536 if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope: 1537 scope = type.scope 1538 for entry in scope.cfunc_entries: 1539 if entry.type and entry.type.is_fused: 1540 entry.type.get_all_specialized_function_types() 1541 return node 1542 1543 1544class AnalyseDeclarationsTransform(EnvTransform): 1545 1546 basic_property = TreeFragment(u""" 1547property NAME: 1548 def __get__(self): 1549 return ATTR 1550 def __set__(self, value): 1551 ATTR = value 1552 """, level='c_class', pipeline=[NormalizeTree(None)]) 1553 basic_pyobject_property = TreeFragment(u""" 1554property NAME: 1555 def __get__(self): 1556 return ATTR 1557 def __set__(self, value): 1558 ATTR = value 1559 def __del__(self): 1560 ATTR = None 1561 """, level='c_class', pipeline=[NormalizeTree(None)]) 1562 basic_property_ro = TreeFragment(u""" 1563property NAME: 1564 def __get__(self): 1565 return ATTR 1566 """, level='c_class', pipeline=[NormalizeTree(None)]) 1567 1568 struct_or_union_wrapper = TreeFragment(u""" 1569cdef class NAME: 1570 cdef TYPE value 1571 def __init__(self, MEMBER=None): 1572 cdef int count 1573 count = 0 1574 INIT_ASSIGNMENTS 1575 if IS_UNION and count > 1: 1576 raise ValueError, "At most one union member should be specified." 1577 def __str__(self): 1578 return STR_FORMAT % MEMBER_TUPLE 1579 def __repr__(self): 1580 return REPR_FORMAT % MEMBER_TUPLE 1581 """, pipeline=[NormalizeTree(None)]) 1582 1583 init_assignment = TreeFragment(u""" 1584if VALUE is not None: 1585 ATTR = VALUE 1586 count += 1 1587 """, pipeline=[NormalizeTree(None)]) 1588 1589 fused_function = None 1590 in_lambda = 0 1591 1592 def __call__(self, root): 1593 # needed to determine if a cdef var is declared after it's used. 1594 self.seen_vars_stack = [] 1595 self.fused_error_funcs = set() 1596 super_class = super(AnalyseDeclarationsTransform, self) 1597 self._super_visit_FuncDefNode = super_class.visit_FuncDefNode 1598 return super_class.__call__(root) 1599 1600 def visit_NameNode(self, node): 1601 self.seen_vars_stack[-1].add(node.name) 1602 return node 1603 1604 def visit_ModuleNode(self, node): 1605 # Pickling support requires injecting module-level nodes. 1606 self.extra_module_declarations = [] 1607 self.seen_vars_stack.append(set()) 1608 node.analyse_declarations(self.current_env()) 1609 self.visitchildren(node) 1610 self.seen_vars_stack.pop() 1611 node.body.stats.extend(self.extra_module_declarations) 1612 return node 1613 1614 def visit_LambdaNode(self, node): 1615 self.in_lambda += 1 1616 node.analyse_declarations(self.current_env()) 1617 self.visitchildren(node) 1618 self.in_lambda -= 1 1619 return node 1620 1621 def visit_CClassDefNode(self, node): 1622 node = self.visit_ClassDefNode(node) 1623 if node.scope and node.scope.implemented and node.body: 1624 stats = [] 1625 for entry in node.scope.var_entries: 1626 if entry.needs_property: 1627 property = self.create_Property(entry) 1628 property.analyse_declarations(node.scope) 1629 self.visit(property) 1630 stats.append(property) 1631 if stats: 1632 node.body.stats += stats 1633 if (node.visibility != 'extern' 1634 and not node.scope.lookup('__reduce__') 1635 and not node.scope.lookup('__reduce_ex__')): 1636 self._inject_pickle_methods(node) 1637 return node 1638 1639 def _inject_pickle_methods(self, node): 1640 env = self.current_env() 1641 if node.scope.directives['auto_pickle'] is False: # None means attempt it. 1642 # Old behavior of not doing anything. 1643 return 1644 auto_pickle_forced = node.scope.directives['auto_pickle'] is True 1645 1646 all_members = [] 1647 cls = node.entry.type 1648 cinit = None 1649 inherited_reduce = None 1650 while cls is not None: 1651 all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__')) 1652 cinit = cinit or cls.scope.lookup('__cinit__') 1653 inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__') 1654 cls = cls.base_type 1655 all_members.sort(key=lambda e: e.name) 1656 1657 if inherited_reduce: 1658 # This is not failsafe, as we may not know whether a cimported class defines a __reduce__. 1659 # This is why we define __reduce_cython__ and only replace __reduce__ 1660 # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation. 1661 return 1662 1663 non_py = [ 1664 e for e in all_members 1665 if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env) 1666 or not e.type.can_coerce_from_pyobject(env)) 1667 ] 1668 1669 structs = [e for e in all_members if e.type.is_struct_or_union] 1670 1671 if cinit or non_py or (structs and not auto_pickle_forced): 1672 if cinit: 1673 # TODO(robertwb): We could allow this if __cinit__ has no require arguments. 1674 msg = 'no default __reduce__ due to non-trivial __cinit__' 1675 elif non_py: 1676 msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py) 1677 else: 1678 # Extern structs may be only partially defined. 1679 # TODO(robertwb): Limit the restriction to extern 1680 # (and recursively extern-containing) structs. 1681 msg = ("Pickling of struct members such as %s must be explicitly requested " 1682 "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs)) 1683 1684 if auto_pickle_forced: 1685 error(node.pos, msg) 1686 1687 pickle_func = TreeFragment(u""" 1688 def __reduce_cython__(self): 1689 raise TypeError("%(msg)s") 1690 def __setstate_cython__(self, __pyx_state): 1691 raise TypeError("%(msg)s") 1692 """ % {'msg': msg}, 1693 level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) 1694 pickle_func.analyse_declarations(node.scope) 1695 self.visit(pickle_func) 1696 node.body.stats.append(pickle_func) 1697 1698 else: 1699 for e in all_members: 1700 if not e.type.is_pyobject: 1701 e.type.create_to_py_utility_code(env) 1702 e.type.create_from_py_utility_code(env) 1703 all_members_names = sorted([e.name for e in all_members]) 1704 checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] 1705 unpickle_func_name = '__pyx_unpickle_%s' % node.class_name 1706 1707 # TODO(robertwb): Move the state into the third argument 1708 # so it can be pickled *after* self is memoized. 1709 unpickle_func = TreeFragment(u""" 1710 def %(unpickle_func_name)s(__pyx_type, long __pyx_checksum, __pyx_state): 1711 cdef object __pyx_PickleError 1712 cdef object __pyx_result 1713 if __pyx_checksum != %(checksum)s: 1714 from pickle import PickleError as __pyx_PickleError 1715 raise __pyx_PickleError("Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum) 1716 __pyx_result = %(class_name)s.__new__(__pyx_type) 1717 if __pyx_state is not None: 1718 %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) 1719 return __pyx_result 1720 1721 cdef %(unpickle_func_name)s__set_state(%(class_name)s __pyx_result, tuple __pyx_state): 1722 %(assignments)s 1723 if len(__pyx_state) > %(num_members)d and hasattr(__pyx_result, '__dict__'): 1724 __pyx_result.__dict__.update(__pyx_state[%(num_members)d]) 1725 """ % { 1726 'unpickle_func_name': unpickle_func_name, 1727 'checksum': checksum, 1728 'members': ', '.join(all_members_names), 1729 'class_name': node.class_name, 1730 'assignments': '; '.join( 1731 '__pyx_result.%s = __pyx_state[%s]' % (v, ix) 1732 for ix, v in enumerate(all_members_names)), 1733 'num_members': len(all_members_names), 1734 }, level='module', pipeline=[NormalizeTree(None)]).substitute({}) 1735 unpickle_func.analyse_declarations(node.entry.scope) 1736 self.visit(unpickle_func) 1737 self.extra_module_declarations.append(unpickle_func) 1738 1739 pickle_func = TreeFragment(u""" 1740 def __reduce_cython__(self): 1741 cdef tuple state 1742 cdef object _dict 1743 cdef bint use_setstate 1744 state = (%(members)s) 1745 _dict = getattr(self, '__dict__', None) 1746 if _dict is not None: 1747 state += (_dict,) 1748 use_setstate = True 1749 else: 1750 use_setstate = %(any_notnone_members)s 1751 if use_setstate: 1752 return %(unpickle_func_name)s, (type(self), %(checksum)s, None), state 1753 else: 1754 return %(unpickle_func_name)s, (type(self), %(checksum)s, state) 1755 1756 def __setstate_cython__(self, __pyx_state): 1757 %(unpickle_func_name)s__set_state(self, __pyx_state) 1758 """ % { 1759 'unpickle_func_name': unpickle_func_name, 1760 'checksum': checksum, 1761 'members': ', '.join('self.%s' % v for v in all_members_names) + (',' if len(all_members_names) == 1 else ''), 1762 # Even better, we could check PyType_IS_GC. 1763 'any_notnone_members' : ' or '.join(['self.%s is not None' % e.name for e in all_members if e.type.is_pyobject] or ['False']), 1764 }, 1765 level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) 1766 pickle_func.analyse_declarations(node.scope) 1767 self.enter_scope(node, node.scope) # functions should be visited in the class scope 1768 self.visit(pickle_func) 1769 self.exit_scope() 1770 node.body.stats.append(pickle_func) 1771 1772 def _handle_fused_def_decorators(self, old_decorators, env, node): 1773 """ 1774 Create function calls to the decorators and reassignments to 1775 the function. 1776 """ 1777 # Delete staticmethod and classmethod decorators, this is 1778 # handled directly by the fused function object. 1779 decorators = [] 1780 for decorator in old_decorators: 1781 func = decorator.decorator 1782 if (not func.is_name or 1783 func.name not in ('staticmethod', 'classmethod') or 1784 env.lookup_here(func.name)): 1785 # not a static or classmethod 1786 decorators.append(decorator) 1787 1788 if decorators: 1789 transform = DecoratorTransform(self.context) 1790 def_node = node.node 1791 _, reassignments = transform.chain_decorators( 1792 def_node, decorators, def_node.name) 1793 reassignments.analyse_declarations(env) 1794 node = [node, reassignments] 1795 1796 return node 1797 1798 def _handle_def(self, decorators, env, node): 1799 "Handle def or cpdef fused functions" 1800 # Create PyCFunction nodes for each specialization 1801 node.stats.insert(0, node.py_func) 1802 node.py_func = self.visit(node.py_func) 1803 node.update_fused_defnode_entry(env) 1804 pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True) 1805 pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env)) 1806 node.resulting_fused_function = pycfunc 1807 # Create assignment node for our def function 1808 node.fused_func_assignment = self._create_assignment( 1809 node.py_func, ExprNodes.CloneNode(pycfunc), env) 1810 1811 if decorators: 1812 node = self._handle_fused_def_decorators(decorators, env, node) 1813 1814 return node 1815 1816 def _create_fused_function(self, env, node): 1817 "Create a fused function for a DefNode with fused arguments" 1818 from . import FusedNode 1819 1820 if self.fused_function or self.in_lambda: 1821 if self.fused_function not in self.fused_error_funcs: 1822 if self.in_lambda: 1823 error(node.pos, "Fused lambdas not allowed") 1824 else: 1825 error(node.pos, "Cannot nest fused functions") 1826 1827 self.fused_error_funcs.add(self.fused_function) 1828 1829 node.body = Nodes.PassStatNode(node.pos) 1830 for arg in node.args: 1831 if arg.type.is_fused: 1832 arg.type = arg.type.get_fused_types()[0] 1833 1834 return node 1835 1836 decorators = getattr(node, 'decorators', None) 1837 node = FusedNode.FusedCFuncDefNode(node, env) 1838 self.fused_function = node 1839 self.visitchildren(node) 1840 self.fused_function = None 1841 if node.py_func: 1842 node = self._handle_def(decorators, env, node) 1843 1844 return node 1845 1846 def _handle_nogil_cleanup(self, lenv, node): 1847 "Handle cleanup for 'with gil' blocks in nogil functions." 1848 if lenv.nogil and lenv.has_with_gil_block: 1849 # Acquire the GIL for cleanup in 'nogil' functions, by wrapping 1850 # the entire function body in try/finally. 1851 # The corresponding release will be taken care of by 1852 # Nodes.FuncDefNode.generate_function_definitions() 1853 node.body = Nodes.NogilTryFinallyStatNode( 1854 node.body.pos, 1855 body=node.body, 1856 finally_clause=Nodes.EnsureGILNode(node.body.pos), 1857 finally_except_clause=Nodes.EnsureGILNode(node.body.pos)) 1858 1859 def _handle_fused(self, node): 1860 if node.is_generator and node.has_fused_arguments: 1861 node.has_fused_arguments = False 1862 error(node.pos, "Fused generators not supported") 1863 node.gbody = Nodes.StatListNode(node.pos, 1864 stats=[], 1865 body=Nodes.PassStatNode(node.pos)) 1866 1867 return node.has_fused_arguments 1868 1869 def visit_FuncDefNode(self, node): 1870 """ 1871 Analyse a function and its body, as that hasn't happened yet. Also 1872 analyse the directive_locals set by @cython.locals(). 1873 1874 Then, if we are a function with fused arguments, replace the function 1875 (after it has declared itself in the symbol table!) with a 1876 FusedCFuncDefNode, and analyse its children (which are in turn normal 1877 functions). If we're a normal function, just analyse the body of the 1878 function. 1879 """ 1880 env = self.current_env() 1881 1882 self.seen_vars_stack.append(set()) 1883 lenv = node.local_scope 1884 node.declare_arguments(lenv) 1885 1886 # @cython.locals(...) 1887 for var, type_node in node.directive_locals.items(): 1888 if not lenv.lookup_here(var): # don't redeclare args 1889 type = type_node.analyse_as_type(lenv) 1890 if type: 1891 lenv.declare_var(var, type, type_node.pos) 1892 else: 1893 error(type_node.pos, "Not a type") 1894 1895 if self._handle_fused(node): 1896 node = self._create_fused_function(env, node) 1897 else: 1898 node.body.analyse_declarations(lenv) 1899 self._handle_nogil_cleanup(lenv, node) 1900 self._super_visit_FuncDefNode(node) 1901 1902 self.seen_vars_stack.pop() 1903 return node 1904 1905 def visit_DefNode(self, node): 1906 node = self.visit_FuncDefNode(node) 1907 env = self.current_env() 1908 if isinstance(node, Nodes.DefNode) and node.is_wrapper: 1909 env = env.parent_scope 1910 if (not isinstance(node, Nodes.DefNode) or 1911 node.fused_py_func or node.is_generator_body or 1912 not node.needs_assignment_synthesis(env)): 1913 return node 1914 return [node, self._synthesize_assignment(node, env)] 1915 1916 def visit_GeneratorBodyDefNode(self, node): 1917 return self.visit_FuncDefNode(node) 1918 1919 def _synthesize_assignment(self, node, env): 1920 # Synthesize assignment node and put it right after defnode 1921 genv = env 1922 while genv.is_py_class_scope or genv.is_c_class_scope: 1923 genv = genv.outer_scope 1924 1925 if genv.is_closure_scope: 1926 rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode( 1927 node.pos, def_node=node, 1928 pymethdef_cname=node.entry.pymethdef_cname, 1929 code_object=ExprNodes.CodeObjectNode(node)) 1930 else: 1931 binding = self.current_directives.get('binding') 1932 rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding) 1933 node.code_object = rhs.code_object 1934 if node.is_generator: 1935 node.gbody.code_object = node.code_object 1936 1937 if env.is_py_class_scope: 1938 rhs.binding = True 1939 1940 node.is_cyfunction = rhs.binding 1941 return self._create_assignment(node, rhs, env) 1942 1943 def _create_assignment(self, def_node, rhs, env): 1944 if def_node.decorators: 1945 for decorator in def_node.decorators[::-1]: 1946 rhs = ExprNodes.SimpleCallNode( 1947 decorator.pos, 1948 function = decorator.decorator, 1949 args = [rhs]) 1950 def_node.decorators = None 1951 1952 assmt = Nodes.SingleAssignmentNode( 1953 def_node.pos, 1954 lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name), 1955 rhs=rhs) 1956 assmt.analyse_declarations(env) 1957 return assmt 1958 1959 def visit_ScopedExprNode(self, node): 1960 env = self.current_env() 1961 node.analyse_declarations(env) 1962 # the node may or may not have a local scope 1963 if node.has_local_scope: 1964 self.seen_vars_stack.append(set(self.seen_vars_stack[-1])) 1965 self.enter_scope(node, node.expr_scope) 1966 node.analyse_scoped_declarations(node.expr_scope) 1967 self.visitchildren(node) 1968 self.exit_scope() 1969 self.seen_vars_stack.pop() 1970 else: 1971 node.analyse_scoped_declarations(env) 1972 self.visitchildren(node) 1973 return node 1974 1975 def visit_TempResultFromStatNode(self, node): 1976 self.visitchildren(node) 1977 node.analyse_declarations(self.current_env()) 1978 return node 1979 1980 def visit_CppClassNode(self, node): 1981 if node.visibility == 'extern': 1982 return None 1983 else: 1984 return self.visit_ClassDefNode(node) 1985 1986 def visit_CStructOrUnionDefNode(self, node): 1987 # Create a wrapper node if needed. 1988 # We want to use the struct type information (so it can't happen 1989 # before this phase) but also create new objects to be declared 1990 # (so it can't happen later). 1991 # Note that we don't return the original node, as it is 1992 # never used after this phase. 1993 if True: # private (default) 1994 return None 1995 1996 self_value = ExprNodes.AttributeNode( 1997 pos = node.pos, 1998 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"), 1999 attribute = EncodedString(u"value")) 2000 var_entries = node.entry.type.scope.var_entries 2001 attributes = [] 2002 for entry in var_entries: 2003 attributes.append(ExprNodes.AttributeNode(pos = entry.pos, 2004 obj = self_value, 2005 attribute = entry.name)) 2006 # __init__ assignments 2007 init_assignments = [] 2008 for entry, attr in zip(var_entries, attributes): 2009 # TODO: branch on visibility 2010 init_assignments.append(self.init_assignment.substitute({ 2011 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name), 2012 u"ATTR": attr, 2013 }, pos = entry.pos)) 2014 2015 # create the class 2016 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2]) 2017 wrapper_class = self.struct_or_union_wrapper.substitute({ 2018 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments), 2019 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct), 2020 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes), 2021 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)), 2022 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))), 2023 }, pos = node.pos).stats[0] 2024 wrapper_class.class_name = node.name 2025 wrapper_class.shadow = True 2026 class_body = wrapper_class.body.stats 2027 2028 # fix value type 2029 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode) 2030 class_body[0].base_type.name = node.name 2031 2032 # fix __init__ arguments 2033 init_method = class_body[1] 2034 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__' 2035 arg_template = init_method.args[1] 2036 if not node.entry.type.is_struct: 2037 arg_template.kw_only = True 2038 del init_method.args[1] 2039 for entry, attr in zip(var_entries, attributes): 2040 arg = copy.deepcopy(arg_template) 2041 arg.declarator.name = entry.name 2042 init_method.args.append(arg) 2043 2044 # setters/getters 2045 for entry, attr in zip(var_entries, attributes): 2046 # TODO: branch on visibility 2047 if entry.type.is_pyobject: 2048 template = self.basic_pyobject_property 2049 else: 2050 template = self.basic_property 2051 property = template.substitute({ 2052 u"ATTR": attr, 2053 }, pos = entry.pos).stats[0] 2054 property.name = entry.name 2055 wrapper_class.body.stats.append(property) 2056 2057 wrapper_class.analyse_declarations(self.current_env()) 2058 return self.visit_CClassDefNode(wrapper_class) 2059 2060 # Some nodes are no longer needed after declaration 2061 # analysis and can be dropped. The analysis was performed 2062 # on these nodes in a separate recursive process from the 2063 # enclosing function or module, so we can simply drop them. 2064 def visit_CDeclaratorNode(self, node): 2065 # necessary to ensure that all CNameDeclaratorNodes are visited. 2066 self.visitchildren(node) 2067 return node 2068 2069 def visit_CTypeDefNode(self, node): 2070 return node 2071 2072 def visit_CBaseTypeNode(self, node): 2073 return None 2074 2075 def visit_CEnumDefNode(self, node): 2076 if node.visibility == 'public': 2077 return node 2078 else: 2079 return None 2080 2081 def visit_CNameDeclaratorNode(self, node): 2082 if node.name in self.seen_vars_stack[-1]: 2083 entry = self.current_env().lookup(node.name) 2084 if (entry is None or entry.visibility != 'extern' 2085 and not entry.scope.is_c_class_scope): 2086 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2) 2087 self.visitchildren(node) 2088 return node 2089 2090 def visit_CVarDefNode(self, node): 2091 # to ensure all CNameDeclaratorNodes are visited. 2092 self.visitchildren(node) 2093 return None 2094 2095 def visit_CnameDecoratorNode(self, node): 2096 child_node = self.visit(node.node) 2097 if not child_node: 2098 return None 2099 if type(child_node) is list: # Assignment synthesized 2100 node.child_node = child_node[0] 2101 return [node] + child_node[1:] 2102 node.node = child_node 2103 return node 2104 2105 def create_Property(self, entry): 2106 if entry.visibility == 'public': 2107 if entry.type.is_pyobject: 2108 template = self.basic_pyobject_property 2109 else: 2110 template = self.basic_property 2111 elif entry.visibility == 'readonly': 2112 template = self.basic_property_ro 2113 property = template.substitute({ 2114 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos, 2115 obj=ExprNodes.NameNode(pos=entry.pos, name="self"), 2116 attribute=entry.name), 2117 }, pos=entry.pos).stats[0] 2118 property.name = entry.name 2119 property.doc = entry.doc 2120 return property 2121 2122 2123class CalculateQualifiedNamesTransform(EnvTransform): 2124 """ 2125 Calculate and store the '__qualname__' and the global 2126 module name on some nodes. 2127 """ 2128 def visit_ModuleNode(self, node): 2129 self.module_name = self.global_scope().qualified_name 2130 self.qualified_name = [] 2131 _super = super(CalculateQualifiedNamesTransform, self) 2132 self._super_visit_FuncDefNode = _super.visit_FuncDefNode 2133 self._super_visit_ClassDefNode = _super.visit_ClassDefNode 2134 self.visitchildren(node) 2135 return node 2136 2137 def _set_qualname(self, node, name=None): 2138 if name: 2139 qualname = self.qualified_name[:] 2140 qualname.append(name) 2141 else: 2142 qualname = self.qualified_name 2143 node.qualname = EncodedString('.'.join(qualname)) 2144 node.module_name = self.module_name 2145 2146 def _append_entry(self, entry): 2147 if entry.is_pyglobal and not entry.is_pyclass_attr: 2148 self.qualified_name = [entry.name] 2149 else: 2150 self.qualified_name.append(entry.name) 2151 2152 def visit_ClassNode(self, node): 2153 self._set_qualname(node, node.name) 2154 self.visitchildren(node) 2155 return node 2156 2157 def visit_PyClassNamespaceNode(self, node): 2158 # class name was already added by parent node 2159 self._set_qualname(node) 2160 self.visitchildren(node) 2161 return node 2162 2163 def visit_PyCFunctionNode(self, node): 2164 orig_qualified_name = self.qualified_name[:] 2165 if node.def_node.is_wrapper and self.qualified_name and self.qualified_name[-1] == '<locals>': 2166 self.qualified_name.pop() 2167 self._set_qualname(node) 2168 else: 2169 self._set_qualname(node, node.def_node.name) 2170 self.visitchildren(node) 2171 self.qualified_name = orig_qualified_name 2172 return node 2173 2174 def visit_DefNode(self, node): 2175 if node.is_wrapper and self.qualified_name: 2176 assert self.qualified_name[-1] == '<locals>', self.qualified_name 2177 orig_qualified_name = self.qualified_name[:] 2178 self.qualified_name.pop() 2179 self._set_qualname(node) 2180 self._super_visit_FuncDefNode(node) 2181 self.qualified_name = orig_qualified_name 2182 else: 2183 self._set_qualname(node, node.name) 2184 self.visit_FuncDefNode(node) 2185 return node 2186 2187 def visit_FuncDefNode(self, node): 2188 orig_qualified_name = self.qualified_name[:] 2189 if getattr(node, 'name', None) == '<lambda>': 2190 self.qualified_name.append('<lambda>') 2191 else: 2192 self._append_entry(node.entry) 2193 self.qualified_name.append('<locals>') 2194 self._super_visit_FuncDefNode(node) 2195 self.qualified_name = orig_qualified_name 2196 return node 2197 2198 def visit_ClassDefNode(self, node): 2199 orig_qualified_name = self.qualified_name[:] 2200 entry = (getattr(node, 'entry', None) or # PyClass 2201 self.current_env().lookup_here(node.name)) # CClass 2202 self._append_entry(entry) 2203 self._super_visit_ClassDefNode(node) 2204 self.qualified_name = orig_qualified_name 2205 return node 2206 2207 2208class AnalyseExpressionsTransform(CythonTransform): 2209 2210 def visit_ModuleNode(self, node): 2211 node.scope.infer_types() 2212 node.body = node.body.analyse_expressions(node.scope) 2213 self.visitchildren(node) 2214 return node 2215 2216 def visit_FuncDefNode(self, node): 2217 node.local_scope.infer_types() 2218 node.body = node.body.analyse_expressions(node.local_scope) 2219 self.visitchildren(node) 2220 return node 2221 2222 def visit_ScopedExprNode(self, node): 2223 if node.has_local_scope: 2224 node.expr_scope.infer_types() 2225 node = node.analyse_scoped_expressions(node.expr_scope) 2226 self.visitchildren(node) 2227 return node 2228 2229 def visit_IndexNode(self, node): 2230 """ 2231 Replace index nodes used to specialize cdef functions with fused 2232 argument types with the Attribute- or NameNode referring to the 2233 function. We then need to copy over the specialization properties to 2234 the attribute or name node. 2235 2236 Because the indexing might be a Python indexing operation on a fused 2237 function, or (usually) a Cython indexing operation, we need to 2238 re-analyse the types. 2239 """ 2240 self.visit_Node(node) 2241 if node.is_fused_index and not node.type.is_error: 2242 node = node.base 2243 return node 2244 2245 2246class FindInvalidUseOfFusedTypes(CythonTransform): 2247 2248 def visit_FuncDefNode(self, node): 2249 # Errors related to use in functions with fused args will already 2250 # have been detected 2251 if not node.has_fused_arguments: 2252 if not node.is_generator_body and node.return_type.is_fused: 2253 error(node.pos, "Return type is not specified as argument type") 2254 else: 2255 self.visitchildren(node) 2256 2257 return node 2258 2259 def visit_ExprNode(self, node): 2260 if node.type and node.type.is_fused: 2261 error(node.pos, "Invalid use of fused types, type cannot be specialized") 2262 else: 2263 self.visitchildren(node) 2264 2265 return node 2266 2267 2268class ExpandInplaceOperators(EnvTransform): 2269 2270 def visit_InPlaceAssignmentNode(self, node): 2271 lhs = node.lhs 2272 rhs = node.rhs 2273 if lhs.type.is_cpp_class: 2274 # No getting around this exact operator here. 2275 return node 2276 if isinstance(lhs, ExprNodes.BufferIndexNode): 2277 # There is code to handle this case in InPlaceAssignmentNode 2278 return node 2279 2280 env = self.current_env() 2281 def side_effect_free_reference(node, setting=False): 2282 if node.is_name: 2283 return node, [] 2284 elif node.type.is_pyobject and not setting: 2285 node = LetRefNode(node) 2286 return node, [node] 2287 elif node.is_subscript: 2288 base, temps = side_effect_free_reference(node.base) 2289 index = LetRefNode(node.index) 2290 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index] 2291 elif node.is_attribute: 2292 obj, temps = side_effect_free_reference(node.obj) 2293 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps 2294 elif isinstance(node, ExprNodes.BufferIndexNode): 2295 raise ValueError("Don't allow things like attributes of buffer indexing operations") 2296 else: 2297 node = LetRefNode(node) 2298 return node, [node] 2299 try: 2300 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True) 2301 except ValueError: 2302 return node 2303 dup = lhs.__class__(**lhs.__dict__) 2304 binop = ExprNodes.binop_node(node.pos, 2305 operator = node.operator, 2306 operand1 = dup, 2307 operand2 = rhs, 2308 inplace=True) 2309 # Manually analyse types for new node. 2310 lhs.analyse_target_types(env) 2311 dup.analyse_types(env) 2312 binop.analyse_operation(env) 2313 node = Nodes.SingleAssignmentNode( 2314 node.pos, 2315 lhs = lhs, 2316 rhs=binop.coerce_to(lhs.type, env)) 2317 # Use LetRefNode to avoid side effects. 2318 let_ref_nodes.reverse() 2319 for t in let_ref_nodes: 2320 node = LetNode(t, node) 2321 return node 2322 2323 def visit_ExprNode(self, node): 2324 # In-place assignments can't happen within an expression. 2325 return node 2326 2327 2328class AdjustDefByDirectives(CythonTransform, SkipDeclarations): 2329 """ 2330 Adjust function and class definitions by the decorator directives: 2331 2332 @cython.cfunc 2333 @cython.cclass 2334 @cython.ccall 2335 @cython.inline 2336 @cython.nogil 2337 """ 2338 2339 def visit_ModuleNode(self, node): 2340 self.directives = node.directives 2341 self.in_py_class = False 2342 self.visitchildren(node) 2343 return node 2344 2345 def visit_CompilerDirectivesNode(self, node): 2346 old_directives = self.directives 2347 self.directives = node.directives 2348 self.visitchildren(node) 2349 self.directives = old_directives 2350 return node 2351 2352 def visit_DefNode(self, node): 2353 modifiers = [] 2354 if 'inline' in self.directives: 2355 modifiers.append('inline') 2356 nogil = self.directives.get('nogil') 2357 except_val = self.directives.get('exceptval') 2358 return_type_node = self.directives.get('returns') 2359 if return_type_node is None and self.directives['annotation_typing']: 2360 return_type_node = node.return_type_annotation 2361 # for Python anntations, prefer safe exception handling by default 2362 if return_type_node is not None and except_val is None: 2363 except_val = (None, True) # except * 2364 elif except_val is None: 2365 # backward compatible default: no exception check 2366 except_val = (None, False) 2367 if 'ccall' in self.directives: 2368 node = node.as_cfunction( 2369 overridable=True, modifiers=modifiers, nogil=nogil, 2370 returns=return_type_node, except_val=except_val) 2371 return self.visit(node) 2372 if 'cfunc' in self.directives: 2373 if self.in_py_class: 2374 error(node.pos, "cfunc directive is not allowed here") 2375 else: 2376 node = node.as_cfunction( 2377 overridable=False, modifiers=modifiers, nogil=nogil, 2378 returns=return_type_node, except_val=except_val) 2379 return self.visit(node) 2380 if 'inline' in modifiers: 2381 error(node.pos, "Python functions cannot be declared 'inline'") 2382 if nogil: 2383 # TODO: turn this into a "with gil" declaration. 2384 error(node.pos, "Python functions cannot be declared 'nogil'") 2385 self.visitchildren(node) 2386 return node 2387 2388 def visit_LambdaNode(self, node): 2389 # No directives should modify lambdas or generator expressions (and also nothing in them). 2390 return node 2391 2392 def visit_PyClassDefNode(self, node): 2393 if 'cclass' in self.directives: 2394 node = node.as_cclass() 2395 return self.visit(node) 2396 else: 2397 old_in_pyclass = self.in_py_class 2398 self.in_py_class = True 2399 self.visitchildren(node) 2400 self.in_py_class = old_in_pyclass 2401 return node 2402 2403 def visit_CClassDefNode(self, node): 2404 old_in_pyclass = self.in_py_class 2405 self.in_py_class = False 2406 self.visitchildren(node) 2407 self.in_py_class = old_in_pyclass 2408 return node 2409 2410 2411class AlignFunctionDefinitions(CythonTransform): 2412 """ 2413 This class takes the signatures from a .pxd file and applies them to 2414 the def methods in a .py file. 2415 """ 2416 2417 def visit_ModuleNode(self, node): 2418 self.scope = node.scope 2419 self.directives = node.directives 2420 self.imported_names = set() # hack, see visit_FromImportStatNode() 2421 self.visitchildren(node) 2422 return node 2423 2424 def visit_PyClassDefNode(self, node): 2425 pxd_def = self.scope.lookup(node.name) 2426 if pxd_def: 2427 if pxd_def.is_cclass: 2428 return self.visit_CClassDefNode(node.as_cclass(), pxd_def) 2429 elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope: 2430 error(node.pos, "'%s' redeclared" % node.name) 2431 if pxd_def.pos: 2432 error(pxd_def.pos, "previous declaration here") 2433 return None 2434 return node 2435 2436 def visit_CClassDefNode(self, node, pxd_def=None): 2437 if pxd_def is None: 2438 pxd_def = self.scope.lookup(node.class_name) 2439 if pxd_def: 2440 if not pxd_def.defined_in_pxd: 2441 return node 2442 outer_scope = self.scope 2443 self.scope = pxd_def.type.scope 2444 self.visitchildren(node) 2445 if pxd_def: 2446 self.scope = outer_scope 2447 return node 2448 2449 def visit_DefNode(self, node): 2450 pxd_def = self.scope.lookup(node.name) 2451 if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope): 2452 if not pxd_def.is_cfunction: 2453 error(node.pos, "'%s' redeclared" % node.name) 2454 if pxd_def.pos: 2455 error(pxd_def.pos, "previous declaration here") 2456 return None 2457 node = node.as_cfunction(pxd_def) 2458 elif (self.scope.is_module_scope and self.directives['auto_cpdef'] 2459 and not node.name in self.imported_names 2460 and node.is_cdef_func_compatible()): 2461 # FIXME: cpdef-ing should be done in analyse_declarations() 2462 node = node.as_cfunction(scope=self.scope) 2463 # Enable this when nested cdef functions are allowed. 2464 # self.visitchildren(node) 2465 return node 2466 2467 def visit_FromImportStatNode(self, node): 2468 # hack to prevent conditional import fallback functions from 2469 # being cdpef-ed (global Python variables currently conflict 2470 # with imports) 2471 if self.scope.is_module_scope: 2472 for name, _ in node.items: 2473 self.imported_names.add(name) 2474 return node 2475 2476 def visit_ExprNode(self, node): 2477 # ignore lambdas and everything else that appears in expressions 2478 return node 2479 2480 2481class RemoveUnreachableCode(CythonTransform): 2482 def visit_StatListNode(self, node): 2483 if not self.current_directives['remove_unreachable']: 2484 return node 2485 self.visitchildren(node) 2486 for idx, stat in enumerate(node.stats): 2487 idx += 1 2488 if stat.is_terminator: 2489 if idx < len(node.stats): 2490 if self.current_directives['warn.unreachable']: 2491 warning(node.stats[idx].pos, "Unreachable code", 2) 2492 node.stats = node.stats[:idx] 2493 node.is_terminator = True 2494 break 2495 return node 2496 2497 def visit_IfClauseNode(self, node): 2498 self.visitchildren(node) 2499 if node.body.is_terminator: 2500 node.is_terminator = True 2501 return node 2502 2503 def visit_IfStatNode(self, node): 2504 self.visitchildren(node) 2505 if node.else_clause and node.else_clause.is_terminator: 2506 for clause in node.if_clauses: 2507 if not clause.is_terminator: 2508 break 2509 else: 2510 node.is_terminator = True 2511 return node 2512 2513 def visit_TryExceptStatNode(self, node): 2514 self.visitchildren(node) 2515 if node.body.is_terminator and node.else_clause: 2516 if self.current_directives['warn.unreachable']: 2517 warning(node.else_clause.pos, "Unreachable code", 2) 2518 node.else_clause = None 2519 return node 2520 2521 def visit_TryFinallyStatNode(self, node): 2522 self.visitchildren(node) 2523 if node.finally_clause.is_terminator: 2524 node.is_terminator = True 2525 return node 2526 2527 2528class YieldNodeCollector(TreeVisitor): 2529 2530 def __init__(self): 2531 super(YieldNodeCollector, self).__init__() 2532 self.yields = [] 2533 self.returns = [] 2534 self.finallys = [] 2535 self.excepts = [] 2536 self.has_return_value = False 2537 self.has_yield = False 2538 self.has_await = False 2539 2540 def visit_Node(self, node): 2541 self.visitchildren(node) 2542 2543 def visit_YieldExprNode(self, node): 2544 self.yields.append(node) 2545 self.has_yield = True 2546 self.visitchildren(node) 2547 2548 def visit_AwaitExprNode(self, node): 2549 self.yields.append(node) 2550 self.has_await = True 2551 self.visitchildren(node) 2552 2553 def visit_ReturnStatNode(self, node): 2554 self.visitchildren(node) 2555 if node.value: 2556 self.has_return_value = True 2557 self.returns.append(node) 2558 2559 def visit_TryFinallyStatNode(self, node): 2560 self.visitchildren(node) 2561 self.finallys.append(node) 2562 2563 def visit_TryExceptStatNode(self, node): 2564 self.visitchildren(node) 2565 self.excepts.append(node) 2566 2567 def visit_ClassDefNode(self, node): 2568 pass 2569 2570 def visit_FuncDefNode(self, node): 2571 pass 2572 2573 def visit_LambdaNode(self, node): 2574 pass 2575 2576 def visit_GeneratorExpressionNode(self, node): 2577 pass 2578 2579 def visit_CArgDeclNode(self, node): 2580 # do not look into annotations 2581 # FIXME: support (yield) in default arguments (currently crashes) 2582 pass 2583 2584 2585class MarkClosureVisitor(CythonTransform): 2586 2587 def visit_ModuleNode(self, node): 2588 self.needs_closure = False 2589 self.visitchildren(node) 2590 return node 2591 2592 def visit_FuncDefNode(self, node): 2593 self.needs_closure = False 2594 self.visitchildren(node) 2595 node.needs_closure = self.needs_closure 2596 self.needs_closure = True 2597 2598 collector = YieldNodeCollector() 2599 collector.visitchildren(node) 2600 2601 if node.is_async_def: 2602 coroutine_type = Nodes.AsyncDefNode 2603 if collector.has_yield: 2604 coroutine_type = Nodes.AsyncGenNode 2605 for yield_expr in collector.yields + collector.returns: 2606 yield_expr.in_async_gen = True 2607 elif self.current_directives['iterable_coroutine']: 2608 coroutine_type = Nodes.IterableAsyncDefNode 2609 elif collector.has_await: 2610 found = next(y for y in collector.yields if y.is_await) 2611 error(found.pos, "'await' not allowed in generators (use 'yield')") 2612 return node 2613 elif collector.has_yield: 2614 coroutine_type = Nodes.GeneratorDefNode 2615 else: 2616 return node 2617 2618 for i, yield_expr in enumerate(collector.yields, 1): 2619 yield_expr.label_num = i 2620 for retnode in collector.returns + collector.finallys + collector.excepts: 2621 retnode.in_generator = True 2622 2623 gbody = Nodes.GeneratorBodyDefNode( 2624 pos=node.pos, name=node.name, body=node.body, 2625 is_async_gen_body=node.is_async_def and collector.has_yield) 2626 coroutine = coroutine_type( 2627 pos=node.pos, name=node.name, args=node.args, 2628 star_arg=node.star_arg, starstar_arg=node.starstar_arg, 2629 doc=node.doc, decorators=node.decorators, 2630 gbody=gbody, lambda_name=node.lambda_name, 2631 return_type_annotation=node.return_type_annotation) 2632 return coroutine 2633 2634 def visit_CFuncDefNode(self, node): 2635 self.needs_closure = False 2636 self.visitchildren(node) 2637 node.needs_closure = self.needs_closure 2638 self.needs_closure = True 2639 if node.needs_closure and node.overridable: 2640 error(node.pos, "closures inside cpdef functions not yet supported") 2641 return node 2642 2643 def visit_LambdaNode(self, node): 2644 self.needs_closure = False 2645 self.visitchildren(node) 2646 node.needs_closure = self.needs_closure 2647 self.needs_closure = True 2648 return node 2649 2650 def visit_ClassDefNode(self, node): 2651 self.visitchildren(node) 2652 self.needs_closure = True 2653 return node 2654 2655 2656class CreateClosureClasses(CythonTransform): 2657 # Output closure classes in module scope for all functions 2658 # that really need it. 2659 2660 def __init__(self, context): 2661 super(CreateClosureClasses, self).__init__(context) 2662 self.path = [] 2663 self.in_lambda = False 2664 2665 def visit_ModuleNode(self, node): 2666 self.module_scope = node.scope 2667 self.visitchildren(node) 2668 return node 2669 2670 def find_entries_used_in_closures(self, node): 2671 from_closure = [] 2672 in_closure = [] 2673 for scope in node.local_scope.iter_local_scopes(): 2674 for name, entry in scope.entries.items(): 2675 if not name: 2676 continue 2677 if entry.from_closure: 2678 from_closure.append((name, entry)) 2679 elif entry.in_closure: 2680 in_closure.append((name, entry)) 2681 return from_closure, in_closure 2682 2683 def create_class_from_scope(self, node, target_module_scope, inner_node=None): 2684 # move local variables into closure 2685 if node.is_generator: 2686 for scope in node.local_scope.iter_local_scopes(): 2687 for entry in scope.entries.values(): 2688 if not (entry.from_closure or entry.is_pyglobal or entry.is_cglobal): 2689 entry.in_closure = True 2690 2691 from_closure, in_closure = self.find_entries_used_in_closures(node) 2692 in_closure.sort() 2693 2694 # Now from the beginning 2695 node.needs_closure = False 2696 node.needs_outer_scope = False 2697 2698 func_scope = node.local_scope 2699 cscope = node.entry.scope 2700 while cscope.is_py_class_scope or cscope.is_c_class_scope: 2701 cscope = cscope.outer_scope 2702 2703 if not from_closure and (self.path or inner_node): 2704 if not inner_node: 2705 if not node.py_cfunc_node: 2706 raise InternalError("DefNode does not have assignment node") 2707 inner_node = node.py_cfunc_node 2708 inner_node.needs_self_code = False 2709 node.needs_outer_scope = False 2710 2711 if node.is_generator: 2712 pass 2713 elif not in_closure and not from_closure: 2714 return 2715 elif not in_closure: 2716 func_scope.is_passthrough = True 2717 func_scope.scope_class = cscope.scope_class 2718 node.needs_outer_scope = True 2719 return 2720 2721 # entry.cname can contain periods (eg. a derived C method of a class). 2722 # We want to use the cname as part of a C struct name, so we replace 2723 # periods with double underscores. 2724 as_name = '%s_%s' % ( 2725 target_module_scope.next_id(Naming.closure_class_prefix), 2726 node.entry.cname.replace('.','__')) 2727 2728 entry = target_module_scope.declare_c_class( 2729 name=as_name, pos=node.pos, defining=True, 2730 implementing=True) 2731 entry.type.is_final_type = True 2732 2733 func_scope.scope_class = entry 2734 class_scope = entry.type.scope 2735 class_scope.is_internal = True 2736 class_scope.is_closure_class_scope = True 2737 if node.is_async_def or node.is_generator: 2738 # Generators need their closure intact during cleanup as they resume to handle GeneratorExit 2739 class_scope.directives['no_gc_clear'] = True 2740 if Options.closure_freelist_size: 2741 class_scope.directives['freelist'] = Options.closure_freelist_size 2742 2743 if from_closure: 2744 assert cscope.is_closure_scope 2745 class_scope.declare_var(pos=node.pos, 2746 name=Naming.outer_scope_cname, 2747 cname=Naming.outer_scope_cname, 2748 type=cscope.scope_class.type, 2749 is_cdef=True) 2750 node.needs_outer_scope = True 2751 for name, entry in in_closure: 2752 closure_entry = class_scope.declare_var( 2753 pos=entry.pos, 2754 name=entry.name if not entry.in_subscope else None, 2755 cname=entry.cname, 2756 type=entry.type, 2757 is_cdef=True) 2758 if entry.is_declared_generic: 2759 closure_entry.is_declared_generic = 1 2760 node.needs_closure = True 2761 # Do it here because other classes are already checked 2762 target_module_scope.check_c_class(func_scope.scope_class) 2763 2764 def visit_LambdaNode(self, node): 2765 if not isinstance(node.def_node, Nodes.DefNode): 2766 # fused function, an error has been previously issued 2767 return node 2768 2769 was_in_lambda = self.in_lambda 2770 self.in_lambda = True 2771 self.create_class_from_scope(node.def_node, self.module_scope, node) 2772 self.visitchildren(node) 2773 self.in_lambda = was_in_lambda 2774 return node 2775 2776 def visit_FuncDefNode(self, node): 2777 if self.in_lambda: 2778 self.visitchildren(node) 2779 return node 2780 if node.needs_closure or self.path: 2781 self.create_class_from_scope(node, self.module_scope) 2782 self.path.append(node) 2783 self.visitchildren(node) 2784 self.path.pop() 2785 return node 2786 2787 def visit_GeneratorBodyDefNode(self, node): 2788 self.visitchildren(node) 2789 return node 2790 2791 def visit_CFuncDefNode(self, node): 2792 if not node.overridable: 2793 return self.visit_FuncDefNode(node) 2794 else: 2795 self.visitchildren(node) 2796 return node 2797 2798 2799class InjectGilHandling(VisitorTransform, SkipDeclarations): 2800 """ 2801 Allow certain Python operations inside of nogil blocks by implicitly acquiring the GIL. 2802 2803 Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get 2804 set up, parallel sections know that the GIL is acquired inside of them, etc. 2805 """ 2806 def __call__(self, root): 2807 self.nogil = False 2808 return super(InjectGilHandling, self).__call__(root) 2809 2810 # special node handling 2811 2812 def visit_RaiseStatNode(self, node): 2813 """Allow raising exceptions in nogil sections by wrapping them in a 'with gil' block.""" 2814 if self.nogil: 2815 node = Nodes.GILStatNode(node.pos, state='gil', body=node) 2816 return node 2817 2818 # further candidates: 2819 # def visit_AssertStatNode(self, node): 2820 # def visit_ReraiseStatNode(self, node): 2821 2822 # nogil tracking 2823 2824 def visit_GILStatNode(self, node): 2825 was_nogil = self.nogil 2826 self.nogil = (node.state == 'nogil') 2827 self.visitchildren(node) 2828 self.nogil = was_nogil 2829 return node 2830 2831 def visit_CFuncDefNode(self, node): 2832 was_nogil = self.nogil 2833 if isinstance(node.declarator, Nodes.CFuncDeclaratorNode): 2834 self.nogil = node.declarator.nogil and not node.declarator.with_gil 2835 self.visitchildren(node) 2836 self.nogil = was_nogil 2837 return node 2838 2839 def visit_ParallelRangeNode(self, node): 2840 was_nogil = self.nogil 2841 self.nogil = node.nogil 2842 self.visitchildren(node) 2843 self.nogil = was_nogil 2844 return node 2845 2846 def visit_ExprNode(self, node): 2847 # No special GIL handling inside of expressions for now. 2848 return node 2849 2850 visit_Node = VisitorTransform.recurse_to_children 2851 2852 2853class GilCheck(VisitorTransform): 2854 """ 2855 Call `node.gil_check(env)` on each node to make sure we hold the 2856 GIL when we need it. Raise an error when on Python operations 2857 inside a `nogil` environment. 2858 2859 Additionally, raise exceptions for closely nested with gil or with nogil 2860 statements. The latter would abort Python. 2861 """ 2862 2863 def __call__(self, root): 2864 self.env_stack = [root.scope] 2865 self.nogil = False 2866 2867 # True for 'cdef func() nogil:' functions, as the GIL may be held while 2868 # calling this function (thus contained 'nogil' blocks may be valid). 2869 self.nogil_declarator_only = False 2870 return super(GilCheck, self).__call__(root) 2871 2872 def _visit_scoped_children(self, node, gil_state): 2873 was_nogil = self.nogil 2874 outer_attrs = node.outer_attrs 2875 if outer_attrs and len(self.env_stack) > 1: 2876 self.nogil = self.env_stack[-2].nogil 2877 self.visitchildren(node, outer_attrs) 2878 2879 self.nogil = gil_state 2880 self.visitchildren(node, exclude=outer_attrs) 2881 self.nogil = was_nogil 2882 2883 def visit_FuncDefNode(self, node): 2884 self.env_stack.append(node.local_scope) 2885 inner_nogil = node.local_scope.nogil 2886 2887 if inner_nogil: 2888 self.nogil_declarator_only = True 2889 2890 if inner_nogil and node.nogil_check: 2891 node.nogil_check(node.local_scope) 2892 2893 self._visit_scoped_children(node, inner_nogil) 2894 2895 # This cannot be nested, so it doesn't need backup/restore 2896 self.nogil_declarator_only = False 2897 2898 self.env_stack.pop() 2899 return node 2900 2901 def visit_GILStatNode(self, node): 2902 if self.nogil and node.nogil_check: 2903 node.nogil_check() 2904 2905 was_nogil = self.nogil 2906 is_nogil = (node.state == 'nogil') 2907 2908 if was_nogil == is_nogil and not self.nogil_declarator_only: 2909 if not was_nogil: 2910 error(node.pos, "Trying to acquire the GIL while it is " 2911 "already held.") 2912 else: 2913 error(node.pos, "Trying to release the GIL while it was " 2914 "previously released.") 2915 2916 if isinstance(node.finally_clause, Nodes.StatListNode): 2917 # The finally clause of the GILStatNode is a GILExitNode, 2918 # which is wrapped in a StatListNode. Just unpack that. 2919 node.finally_clause, = node.finally_clause.stats 2920 2921 self._visit_scoped_children(node, is_nogil) 2922 return node 2923 2924 def visit_ParallelRangeNode(self, node): 2925 if node.nogil: 2926 node.nogil = False 2927 node = Nodes.GILStatNode(node.pos, state='nogil', body=node) 2928 return self.visit_GILStatNode(node) 2929 2930 if not self.nogil: 2931 error(node.pos, "prange() can only be used without the GIL") 2932 # Forget about any GIL-related errors that may occur in the body 2933 return None 2934 2935 node.nogil_check(self.env_stack[-1]) 2936 self.visitchildren(node) 2937 return node 2938 2939 def visit_ParallelWithBlockNode(self, node): 2940 if not self.nogil: 2941 error(node.pos, "The parallel section may only be used without " 2942 "the GIL") 2943 return None 2944 2945 if node.nogil_check: 2946 # It does not currently implement this, but test for it anyway to 2947 # avoid potential future surprises 2948 node.nogil_check(self.env_stack[-1]) 2949 2950 self.visitchildren(node) 2951 return node 2952 2953 def visit_TryFinallyStatNode(self, node): 2954 """ 2955 Take care of try/finally statements in nogil code sections. 2956 """ 2957 if not self.nogil or isinstance(node, Nodes.GILStatNode): 2958 return self.visit_Node(node) 2959 2960 node.nogil_check = None 2961 node.is_try_finally_in_nogil = True 2962 self.visitchildren(node) 2963 return node 2964 2965 def visit_Node(self, node): 2966 if self.env_stack and self.nogil and node.nogil_check: 2967 node.nogil_check(self.env_stack[-1]) 2968 if node.outer_attrs: 2969 self._visit_scoped_children(node, self.nogil) 2970 else: 2971 self.visitchildren(node) 2972 if self.nogil: 2973 node.in_nogil_context = True 2974 return node 2975 2976 2977class TransformBuiltinMethods(EnvTransform): 2978 """ 2979 Replace Cython's own cython.* builtins by the corresponding tree nodes. 2980 """ 2981 2982 def visit_SingleAssignmentNode(self, node): 2983 if node.declaration_only: 2984 return None 2985 else: 2986 self.visitchildren(node) 2987 return node 2988 2989 def visit_AttributeNode(self, node): 2990 self.visitchildren(node) 2991 return self.visit_cython_attribute(node) 2992 2993 def visit_NameNode(self, node): 2994 return self.visit_cython_attribute(node) 2995 2996 def visit_cython_attribute(self, node): 2997 attribute = node.as_cython_attribute() 2998 if attribute: 2999 if attribute == u'compiled': 3000 node = ExprNodes.BoolNode(node.pos, value=True) 3001 elif attribute == u'__version__': 3002 from .. import __version__ as version 3003 node = ExprNodes.StringNode(node.pos, value=EncodedString(version)) 3004 elif attribute == u'NULL': 3005 node = ExprNodes.NullNode(node.pos) 3006 elif attribute in (u'set', u'frozenset', u'staticmethod'): 3007 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute), 3008 entry=self.current_env().builtin_scope().lookup_here(attribute)) 3009 elif PyrexTypes.parse_basic_type(attribute): 3010 pass 3011 elif self.context.cython_scope.lookup_qualified_name(attribute): 3012 pass 3013 else: 3014 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute) 3015 return node 3016 3017 def visit_ExecStatNode(self, node): 3018 lenv = self.current_env() 3019 self.visitchildren(node) 3020 if len(node.args) == 1: 3021 node.args.append(ExprNodes.GlobalsExprNode(node.pos)) 3022 if not lenv.is_module_scope: 3023 node.args.append( 3024 ExprNodes.LocalsExprNode( 3025 node.pos, self.current_scope_node(), lenv)) 3026 return node 3027 3028 def _inject_locals(self, node, func_name): 3029 # locals()/dir()/vars() builtins 3030 lenv = self.current_env() 3031 entry = lenv.lookup_here(func_name) 3032 if entry: 3033 # not the builtin 3034 return node 3035 pos = node.pos 3036 if func_name in ('locals', 'vars'): 3037 if func_name == 'locals' and len(node.args) > 0: 3038 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" 3039 % len(node.args)) 3040 return node 3041 elif func_name == 'vars': 3042 if len(node.args) > 1: 3043 error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d" 3044 % len(node.args)) 3045 if len(node.args) > 0: 3046 return node # nothing to do 3047 return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv) 3048 else: # dir() 3049 if len(node.args) > 1: 3050 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d" 3051 % len(node.args)) 3052 if len(node.args) > 0: 3053 # optimised in Builtin.py 3054 return node 3055 if lenv.is_py_class_scope or lenv.is_module_scope: 3056 if lenv.is_py_class_scope: 3057 pyclass = self.current_scope_node() 3058 locals_dict = ExprNodes.CloneNode(pyclass.dict) 3059 else: 3060 locals_dict = ExprNodes.GlobalsExprNode(pos) 3061 return ExprNodes.SortedDictKeysNode(locals_dict) 3062 local_names = sorted(var.name for var in lenv.entries.values() if var.name) 3063 items = [ExprNodes.IdentifierStringNode(pos, value=var) 3064 for var in local_names] 3065 return ExprNodes.ListNode(pos, args=items) 3066 3067 def visit_PrimaryCmpNode(self, node): 3068 # special case: for in/not-in test, we do not need to sort locals() 3069 self.visitchildren(node) 3070 if node.operator in 'not_in': # in/not_in 3071 if isinstance(node.operand2, ExprNodes.SortedDictKeysNode): 3072 arg = node.operand2.arg 3073 if isinstance(arg, ExprNodes.NoneCheckNode): 3074 arg = arg.arg 3075 node.operand2 = arg 3076 return node 3077 3078 def visit_CascadedCmpNode(self, node): 3079 return self.visit_PrimaryCmpNode(node) 3080 3081 def _inject_eval(self, node, func_name): 3082 lenv = self.current_env() 3083 entry = lenv.lookup_here(func_name) 3084 if entry or len(node.args) != 1: 3085 return node 3086 # Inject globals and locals 3087 node.args.append(ExprNodes.GlobalsExprNode(node.pos)) 3088 if not lenv.is_module_scope: 3089 node.args.append( 3090 ExprNodes.LocalsExprNode( 3091 node.pos, self.current_scope_node(), lenv)) 3092 return node 3093 3094 def _inject_super(self, node, func_name): 3095 lenv = self.current_env() 3096 entry = lenv.lookup_here(func_name) 3097 if entry or node.args: 3098 return node 3099 # Inject no-args super 3100 def_node = self.current_scope_node() 3101 if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or 3102 len(self.env_stack) < 2): 3103 return node 3104 class_node, class_scope = self.env_stack[-2] 3105 if class_scope.is_py_class_scope: 3106 def_node.requires_classobj = True 3107 class_node.class_cell.is_active = True 3108 node.args = [ 3109 ExprNodes.ClassCellNode( 3110 node.pos, is_generator=def_node.is_generator), 3111 ExprNodes.NameNode(node.pos, name=def_node.args[0].name) 3112 ] 3113 elif class_scope.is_c_class_scope: 3114 node.args = [ 3115 ExprNodes.NameNode( 3116 node.pos, name=class_node.scope.name, 3117 entry=class_node.entry), 3118 ExprNodes.NameNode(node.pos, name=def_node.args[0].name) 3119 ] 3120 return node 3121 3122 def visit_SimpleCallNode(self, node): 3123 # cython.foo 3124 function = node.function.as_cython_attribute() 3125 if function: 3126 if function in InterpretCompilerDirectives.unop_method_nodes: 3127 if len(node.args) != 1: 3128 error(node.function.pos, u"%s() takes exactly one argument" % function) 3129 else: 3130 node = InterpretCompilerDirectives.unop_method_nodes[function]( 3131 node.function.pos, operand=node.args[0]) 3132 elif function in InterpretCompilerDirectives.binop_method_nodes: 3133 if len(node.args) != 2: 3134 error(node.function.pos, u"%s() takes exactly two arguments" % function) 3135 else: 3136 node = InterpretCompilerDirectives.binop_method_nodes[function]( 3137 node.function.pos, operand1=node.args[0], operand2=node.args[1]) 3138 elif function == u'cast': 3139 if len(node.args) != 2: 3140 error(node.function.pos, 3141 u"cast() takes exactly two arguments and an optional typecheck keyword") 3142 else: 3143 type = node.args[0].analyse_as_type(self.current_env()) 3144 if type: 3145 node = ExprNodes.TypecastNode( 3146 node.function.pos, type=type, operand=node.args[1], typecheck=False) 3147 else: 3148 error(node.args[0].pos, "Not a type") 3149 elif function == u'sizeof': 3150 if len(node.args) != 1: 3151 error(node.function.pos, u"sizeof() takes exactly one argument") 3152 else: 3153 type = node.args[0].analyse_as_type(self.current_env()) 3154 if type: 3155 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type) 3156 else: 3157 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0]) 3158 elif function == 'cmod': 3159 if len(node.args) != 2: 3160 error(node.function.pos, u"cmod() takes exactly two arguments") 3161 else: 3162 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1]) 3163 node.cdivision = True 3164 elif function == 'cdiv': 3165 if len(node.args) != 2: 3166 error(node.function.pos, u"cdiv() takes exactly two arguments") 3167 else: 3168 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1]) 3169 node.cdivision = True 3170 elif function == u'set': 3171 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set')) 3172 elif function == u'staticmethod': 3173 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod')) 3174 elif self.context.cython_scope.lookup_qualified_name(function): 3175 pass 3176 else: 3177 error(node.function.pos, 3178 u"'%s' not a valid cython language construct" % function) 3179 3180 self.visitchildren(node) 3181 3182 if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name: 3183 func_name = node.function.name 3184 if func_name in ('dir', 'locals', 'vars'): 3185 return self._inject_locals(node, func_name) 3186 if func_name == 'eval': 3187 return self._inject_eval(node, func_name) 3188 if func_name == 'super': 3189 return self._inject_super(node, func_name) 3190 return node 3191 3192 def visit_GeneralCallNode(self, node): 3193 function = node.function.as_cython_attribute() 3194 if function == u'cast': 3195 # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args 3196 args = node.positional_args.args 3197 kwargs = node.keyword_args.compile_time_value(None) 3198 if (len(args) != 2 or len(kwargs) > 1 or 3199 (len(kwargs) == 1 and 'typecheck' not in kwargs)): 3200 error(node.function.pos, 3201 u"cast() takes exactly two arguments and an optional typecheck keyword") 3202 else: 3203 type = args[0].analyse_as_type(self.current_env()) 3204 if type: 3205 typecheck = kwargs.get('typecheck', False) 3206 node = ExprNodes.TypecastNode( 3207 node.function.pos, type=type, operand=args[1], typecheck=typecheck) 3208 else: 3209 error(args[0].pos, "Not a type") 3210 3211 self.visitchildren(node) 3212 return node 3213 3214 3215class ReplaceFusedTypeChecks(VisitorTransform): 3216 """ 3217 This is not a transform in the pipeline. It is invoked on the specific 3218 versions of a cdef function with fused argument types. It filters out any 3219 type branches that don't match. e.g. 3220 3221 if fused_t is mytype: 3222 ... 3223 elif fused_t in other_fused_type: 3224 ... 3225 """ 3226 def __init__(self, local_scope): 3227 super(ReplaceFusedTypeChecks, self).__init__() 3228 self.local_scope = local_scope 3229 # defer the import until now to avoid circular import time dependencies 3230 from .Optimize import ConstantFolding 3231 self.transform = ConstantFolding(reevaluate=True) 3232 3233 def visit_IfStatNode(self, node): 3234 """ 3235 Filters out any if clauses with false compile time type check 3236 expression. 3237 """ 3238 self.visitchildren(node) 3239 return self.transform(node) 3240 3241 def visit_PrimaryCmpNode(self, node): 3242 with Errors.local_errors(ignore=True): 3243 type1 = node.operand1.analyse_as_type(self.local_scope) 3244 type2 = node.operand2.analyse_as_type(self.local_scope) 3245 3246 if type1 and type2: 3247 false_node = ExprNodes.BoolNode(node.pos, value=False) 3248 true_node = ExprNodes.BoolNode(node.pos, value=True) 3249 3250 type1 = self.specialize_type(type1, node.operand1.pos) 3251 op = node.operator 3252 3253 if op in ('is', 'is_not', '==', '!='): 3254 type2 = self.specialize_type(type2, node.operand2.pos) 3255 3256 is_same = type1.same_as(type2) 3257 eq = op in ('is', '==') 3258 3259 if (is_same and eq) or (not is_same and not eq): 3260 return true_node 3261 3262 elif op in ('in', 'not_in'): 3263 # We have to do an instance check directly, as operand2 3264 # needs to be a fused type and not a type with a subtype 3265 # that is fused. First unpack the typedef 3266 if isinstance(type2, PyrexTypes.CTypedefType): 3267 type2 = type2.typedef_base_type 3268 3269 if type1.is_fused: 3270 error(node.operand1.pos, "Type is fused") 3271 elif not type2.is_fused: 3272 error(node.operand2.pos, 3273 "Can only use 'in' or 'not in' on a fused type") 3274 else: 3275 types = PyrexTypes.get_specialized_types(type2) 3276 3277 for specialized_type in types: 3278 if type1.same_as(specialized_type): 3279 if op == 'in': 3280 return true_node 3281 else: 3282 return false_node 3283 3284 if op == 'not_in': 3285 return true_node 3286 3287 return false_node 3288 3289 return node 3290 3291 def specialize_type(self, type, pos): 3292 try: 3293 return type.specialize(self.local_scope.fused_to_specific) 3294 except KeyError: 3295 error(pos, "Type is not specific") 3296 return type 3297 3298 def visit_Node(self, node): 3299 self.visitchildren(node) 3300 return node 3301 3302 3303class DebugTransform(CythonTransform): 3304 """ 3305 Write debug information for this Cython module. 3306 """ 3307 3308 def __init__(self, context, options, result): 3309 super(DebugTransform, self).__init__(context) 3310 self.visited = set() 3311 # our treebuilder and debug output writer 3312 # (see Cython.Debugger.debug_output.CythonDebugWriter) 3313 self.tb = self.context.gdb_debug_outputwriter 3314 #self.c_output_file = options.output_file 3315 self.c_output_file = result.c_file 3316 3317 # Closure support, basically treat nested functions as if the AST were 3318 # never nested 3319 self.nested_funcdefs = [] 3320 3321 # tells visit_NameNode whether it should register step-into functions 3322 self.register_stepinto = False 3323 3324 def visit_ModuleNode(self, node): 3325 self.tb.module_name = node.full_module_name 3326 attrs = dict( 3327 module_name=node.full_module_name, 3328 filename=node.pos[0].filename, 3329 c_filename=self.c_output_file) 3330 3331 self.tb.start('Module', attrs) 3332 3333 # serialize functions 3334 self.tb.start('Functions') 3335 # First, serialize functions normally... 3336 self.visitchildren(node) 3337 3338 # ... then, serialize nested functions 3339 for nested_funcdef in self.nested_funcdefs: 3340 self.visit_FuncDefNode(nested_funcdef) 3341 3342 self.register_stepinto = True 3343 self.serialize_modulenode_as_function(node) 3344 self.register_stepinto = False 3345 self.tb.end('Functions') 3346 3347 # 2.3 compatibility. Serialize global variables 3348 self.tb.start('Globals') 3349 entries = {} 3350 3351 for k, v in node.scope.entries.items(): 3352 if (v.qualified_name not in self.visited and not 3353 v.name.startswith('__pyx_') and not 3354 v.type.is_cfunction and not 3355 v.type.is_extension_type): 3356 entries[k]= v 3357 3358 self.serialize_local_variables(entries) 3359 self.tb.end('Globals') 3360 # self.tb.end('Module') # end Module after the line number mapping in 3361 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map 3362 return node 3363 3364 def visit_FuncDefNode(self, node): 3365 self.visited.add(node.local_scope.qualified_name) 3366 3367 if getattr(node, 'is_wrapper', False): 3368 return node 3369 3370 if self.register_stepinto: 3371 self.nested_funcdefs.append(node) 3372 return node 3373 3374 # node.entry.visibility = 'extern' 3375 if node.py_func is None: 3376 pf_cname = '' 3377 else: 3378 pf_cname = node.py_func.entry.func_cname 3379 3380 attrs = dict( 3381 name=node.entry.name or getattr(node, 'name', '<unknown>'), 3382 cname=node.entry.func_cname, 3383 pf_cname=pf_cname, 3384 qualified_name=node.local_scope.qualified_name, 3385 lineno=str(node.pos[1])) 3386 3387 self.tb.start('Function', attrs=attrs) 3388 3389 self.tb.start('Locals') 3390 self.serialize_local_variables(node.local_scope.entries) 3391 self.tb.end('Locals') 3392 3393 self.tb.start('Arguments') 3394 for arg in node.local_scope.arg_entries: 3395 self.tb.start(arg.name) 3396 self.tb.end(arg.name) 3397 self.tb.end('Arguments') 3398 3399 self.tb.start('StepIntoFunctions') 3400 self.register_stepinto = True 3401 self.visitchildren(node) 3402 self.register_stepinto = False 3403 self.tb.end('StepIntoFunctions') 3404 self.tb.end('Function') 3405 3406 return node 3407 3408 def visit_NameNode(self, node): 3409 if (self.register_stepinto and 3410 node.type is not None and 3411 node.type.is_cfunction and 3412 getattr(node, 'is_called', False) and 3413 node.entry.func_cname is not None): 3414 # don't check node.entry.in_cinclude, as 'cdef extern: ...' 3415 # declared functions are not 'in_cinclude'. 3416 # This means we will list called 'cdef' functions as 3417 # "step into functions", but this is not an issue as they will be 3418 # recognized as Cython functions anyway. 3419 attrs = dict(name=node.entry.func_cname) 3420 self.tb.start('StepIntoFunction', attrs=attrs) 3421 self.tb.end('StepIntoFunction') 3422 3423 self.visitchildren(node) 3424 return node 3425 3426 def serialize_modulenode_as_function(self, node): 3427 """ 3428 Serialize the module-level code as a function so the debugger will know 3429 it's a "relevant frame" and it will know where to set the breakpoint 3430 for 'break modulename'. 3431 """ 3432 name = node.full_module_name.rpartition('.')[-1] 3433 3434 cname_py2 = 'init' + name 3435 cname_py3 = 'PyInit_' + name 3436 3437 py2_attrs = dict( 3438 name=name, 3439 cname=cname_py2, 3440 pf_cname='', 3441 # Ignore the qualified_name, breakpoints should be set using 3442 # `cy break modulename:lineno` for module-level breakpoints. 3443 qualified_name='', 3444 lineno='1', 3445 is_initmodule_function="True", 3446 ) 3447 3448 py3_attrs = dict(py2_attrs, cname=cname_py3) 3449 3450 self._serialize_modulenode_as_function(node, py2_attrs) 3451 self._serialize_modulenode_as_function(node, py3_attrs) 3452 3453 def _serialize_modulenode_as_function(self, node, attrs): 3454 self.tb.start('Function', attrs=attrs) 3455 3456 self.tb.start('Locals') 3457 self.serialize_local_variables(node.scope.entries) 3458 self.tb.end('Locals') 3459 3460 self.tb.start('Arguments') 3461 self.tb.end('Arguments') 3462 3463 self.tb.start('StepIntoFunctions') 3464 self.register_stepinto = True 3465 self.visitchildren(node) 3466 self.register_stepinto = False 3467 self.tb.end('StepIntoFunctions') 3468 3469 self.tb.end('Function') 3470 3471 def serialize_local_variables(self, entries): 3472 for entry in entries.values(): 3473 if not entry.cname: 3474 # not a local variable 3475 continue 3476 if entry.type.is_pyobject: 3477 vartype = 'PythonObject' 3478 else: 3479 vartype = 'CObject' 3480 3481 if entry.from_closure: 3482 # We're dealing with a closure where a variable from an outer 3483 # scope is accessed, get it from the scope object. 3484 cname = '%s->%s' % (Naming.cur_scope_cname, 3485 entry.outer_entry.cname) 3486 3487 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name, 3488 entry.scope.name, 3489 entry.name) 3490 elif entry.in_closure: 3491 cname = '%s->%s' % (Naming.cur_scope_cname, 3492 entry.cname) 3493 qname = entry.qualified_name 3494 else: 3495 cname = entry.cname 3496 qname = entry.qualified_name 3497 3498 if not entry.pos: 3499 # this happens for variables that are not in the user's code, 3500 # e.g. for the global __builtins__, __doc__, etc. We can just 3501 # set the lineno to 0 for those. 3502 lineno = '0' 3503 else: 3504 lineno = str(entry.pos[1]) 3505 3506 attrs = dict( 3507 name=entry.name, 3508 cname=cname, 3509 qualified_name=qname, 3510 type=vartype, 3511 lineno=lineno) 3512 3513 self.tb.start('LocalVar', attrs) 3514 self.tb.end('LocalVar') 3515