1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Assignment related nodes. 19 20The most simple assignment statement ``a = b`` is what we have here. All others 21are either re-formulated using temporary variables, e.g. ``a, b = c`` or are 22attribute, slice, subscript assignments. 23 24The deletion is a separate node unlike in CPython where assigning to ``NULL`` is 25internally what deletion is. But deleting is something entirely different to us 26during code generation, which is why we keep them separate. 27 28Tracing assignments in SSA form is the core of optimization for which we use 29the traces. 30 31""" 32 33from nuitka.ModuleRegistry import getOwnerFromCodeName 34from nuitka.Options import isExperimental 35 36from .NodeBases import StatementBase, StatementChildHavingBase 37from .NodeMakingHelpers import ( 38 makeRaiseExceptionReplacementStatement, 39 makeStatementExpressionOnlyReplacementNode, 40 makeStatementsSequenceReplacementNode, 41) 42from .shapes.StandardShapes import tshape_unknown 43 44 45class StatementAssignmentVariableName(StatementChildHavingBase): 46 """Precursor of StatementAssignmentVariable used during tree building phase""" 47 48 kind = "STATEMENT_ASSIGNMENT_VARIABLE_NAME" 49 50 named_child = "source" 51 nice_child = "assignment source" 52 53 __slots__ = ("variable_name", "provider") 54 55 def __init__(self, provider, variable_name, source, source_ref): 56 assert source is not None, source_ref 57 58 StatementChildHavingBase.__init__(self, value=source, source_ref=source_ref) 59 60 self.variable_name = variable_name 61 self.provider = provider 62 63 assert not provider.isExpressionOutlineBody(), source_ref 64 65 def getDetails(self): 66 return {"variable_name": self.variable_name, "provider": self.provider} 67 68 def getVariableName(self): 69 return self.variable_name 70 71 def computeStatement(self, trace_collection): 72 # Only for abc, pylint: disable=no-self-use 73 74 # These must not enter real optimization, they only live during the 75 # tree building. 76 assert False 77 78 @staticmethod 79 def getStatementNiceName(): 80 return "variable assignment statement" 81 82 83class StatementDelVariableName(StatementBase): 84 """Precursor of StatementDelVariable used during tree building phase""" 85 86 kind = "STATEMENT_DEL_VARIABLE_NAME" 87 88 __slots__ = "variable_name", "provider", "tolerant" 89 90 def __init__(self, provider, variable_name, tolerant, source_ref): 91 StatementBase.__init__(self, source_ref=source_ref) 92 93 self.variable_name = variable_name 94 self.provider = provider 95 96 self.tolerant = tolerant 97 98 def finalize(self): 99 del self.parent 100 del self.provider 101 102 def getDetails(self): 103 return { 104 "variable_name": self.variable_name, 105 "provider": self.provider, 106 "tolerant": self.tolerant, 107 } 108 109 def getVariableName(self): 110 return self.variable_name 111 112 def computeStatement(self, trace_collection): 113 # Only for abc, pylint: disable=no-self-use 114 115 # These must not enter real optimization, they only live during the 116 # tree building. 117 assert False 118 119 120class StatementAssignmentVariable(StatementChildHavingBase): 121 """Assignment to a variable from an expression. 122 123 All assignment forms that are not to attributes, slices, subscripts 124 use this. 125 126 The source might be a complex expression. The target can be any kind 127 of variable, temporary, local, global, etc. 128 129 Assigning a variable is something we trace in a new version, this is 130 hidden behind target variable reference, which has this version once 131 it can be determined. 132 """ 133 134 kind = "STATEMENT_ASSIGNMENT_VARIABLE" 135 136 named_child = "source" 137 nice_child = "assignment source" 138 139 __slots__ = ( 140 "subnode_source", 141 "variable", 142 "variable_version", 143 "variable_trace", 144 "inplace_suspect", 145 ) 146 147 def __init__(self, source, variable, source_ref, version=None): 148 assert source is not None, source_ref 149 150 if variable is not None: 151 if version is None: 152 version = variable.allocateTargetNumber() 153 154 self.variable = variable 155 self.variable_version = version 156 157 StatementChildHavingBase.__init__(self, value=source, source_ref=source_ref) 158 159 self.variable_trace = None 160 self.inplace_suspect = None 161 162 def finalize(self): 163 StatementChildHavingBase.finalize(self) 164 165 del self.variable 166 del self.variable_trace 167 168 def getDetails(self): 169 return {"variable": self.variable} 170 171 def getDetailsForDisplay(self): 172 return { 173 "variable_name": self.getVariableName(), 174 "is_temp": self.variable.isTempVariable(), 175 "var_type": self.variable.getVariableType(), 176 "owner": self.variable.getOwner().getCodeName(), 177 } 178 179 @classmethod 180 def fromXML(cls, provider, source_ref, **args): 181 assert cls is StatementAssignmentVariable, cls 182 183 owner = getOwnerFromCodeName(args["owner"]) 184 185 if args["is_temp"] == "True": 186 variable = owner.createTempVariable( 187 args["variable_name"], temp_type=["var_type"] 188 ) 189 else: 190 variable = owner.getProvidedVariable(args["variable_name"]) 191 192 del args["is_temp"] 193 del args["var_type"] 194 del args["owner"] 195 196 version = variable.allocateTargetNumber() 197 198 return cls(variable=variable, version=version, source_ref=source_ref, **args) 199 200 def makeClone(self): 201 if self.variable is not None: 202 version = self.variable.allocateTargetNumber() 203 else: 204 version = None 205 206 return StatementAssignmentVariable( 207 source=self.subnode_source.makeClone(), 208 variable=self.variable, 209 version=version, 210 source_ref=self.source_ref, 211 ) 212 213 def getVariableName(self): 214 return self.variable.getName() 215 216 def getVariable(self): 217 return self.variable 218 219 def setVariable(self, variable): 220 self.variable = variable 221 self.variable_version = variable.allocateTargetNumber() 222 223 def getVariableTrace(self): 224 return self.variable_trace 225 226 def markAsInplaceSuspect(self): 227 self.inplace_suspect = True 228 229 def isInplaceSuspect(self): 230 return self.inplace_suspect 231 232 def unmarkAsInplaceSuspect(self): 233 self.inplace_suspect = False 234 235 def mayRaiseException(self, exception_type): 236 return self.subnode_source.mayRaiseException(exception_type) 237 238 def computeStatement(self, trace_collection): 239 # This is very complex stuff, pylint: disable=too-many-branches,too-many-return-statements 240 241 # TODO: Way too ugly to have global trace kinds just here, and needs to 242 # be abstracted somehow. But for now we let it live here. 243 source = self.subnode_source 244 245 if source.isExpressionSideEffects(): 246 # If the assignment source has side effects, we can put them into a 247 # sequence and compute that instead. 248 statements = [ 249 makeStatementExpressionOnlyReplacementNode(side_effect, self) 250 for side_effect in source.subnode_side_effects 251 ] 252 253 statements.append(self) 254 255 # Remember out parent, we will assign it for the sequence to use. 256 parent = self.parent 257 258 # Need to update ourselves to no longer reference the side effects, 259 # but go to the wrapped thing. 260 self.setChild("source", source.subnode_expression) 261 262 result = makeStatementsSequenceReplacementNode( 263 statements=statements, node=self 264 ) 265 result.parent = parent 266 267 return ( 268 result.computeStatementsSequence(trace_collection), 269 "new_statements", 270 """\ 271Side effects of assignments promoted to statements.""", 272 ) 273 274 # Let assignment source may re-compute first. 275 source = trace_collection.onExpression(self.subnode_source) 276 277 # No assignment will occur, if the assignment source raises, so give up 278 # on this, and return it as the only side effect. 279 if source.willRaiseException(BaseException): 280 result = makeStatementExpressionOnlyReplacementNode( 281 expression=source, node=self 282 ) 283 284 del self.parent 285 286 return ( 287 result, 288 "new_raise", 289 """\ 290Assignment raises exception in assigned value, removed assignment.""", 291 ) 292 293 variable = self.variable 294 295 # Not allowed anymore at this point. 296 assert variable is not None 297 298 # Assigning from and to the same variable, can be optimized away 299 # immediately, there is no point in doing it. Exceptions are of course 300 # module variables that collide with built-in names. 301 302 # TODO: The variable type checks ought to become unnecessary, as they 303 # are to be a feature of the trace. Assigning from known assigned is 304 # supposed to be possible to eliminate. If we get that wrong, we are 305 # doing it wrong. 306 if ( 307 not variable.isModuleVariable() 308 and source.isExpressionVariableRef() 309 and source.getVariable() is variable 310 ): 311 312 # A variable access that has a side effect, must be preserved, 313 # so it can e.g. raise an exception, otherwise we can be fully 314 # removed. 315 if source.mayHaveSideEffects(): 316 result = makeStatementExpressionOnlyReplacementNode( 317 expression=source, node=self 318 ) 319 320 return ( 321 result, 322 "new_statements", 323 """\ 324Lowered assignment of %s from itself to mere access of it.""" 325 % variable.getDescription(), 326 ) 327 else: 328 return ( 329 None, 330 "new_statements", 331 """\ 332Removed assignment of %s from itself which is known to be defined.""" 333 % variable.getDescription(), 334 ) 335 336 # Set-up the trace to the trace collection, so future references will 337 # find this assignment. 338 self.variable_trace = trace_collection.onVariableSet( 339 variable=variable, version=self.variable_version, assign_node=self 340 ) 341 342 provider = trace_collection.getOwner() 343 344 if variable.hasAccessesOutsideOf(provider) is False: 345 last_trace = variable.getMatchingAssignTrace(self) 346 347 if last_trace is not None and not last_trace.getMergeOrNameUsageCount(): 348 if source.isCompileTimeConstant(): 349 if ( 350 variable.isModuleVariable() 351 or variable.owner.locals_scope.isUnoptimizedFunctionScope() 352 ): 353 # TODO: We do not trust these yet a lot, but more might be 354 pass 355 else: 356 # Unused constants can be eliminated in any case. 357 if not last_trace.getUsageCount(): 358 if not last_trace.getPrevious().isUnassignedTrace(): 359 result = StatementDelVariable( 360 variable=self.variable, 361 version=self.variable_version, 362 tolerant=True, 363 source_ref=self.source_ref, 364 ) 365 else: 366 result = None 367 368 return ( 369 result, 370 "new_statements", 371 "Dropped dead assignment statement to '%s'." 372 % (self.getVariableName()), 373 ) 374 375 # Can safely forward propagate only non-mutable constants. 376 if not source.isMutable(): 377 self.variable_trace.setReplacementNode( 378 lambda _usage: source.makeClone() 379 ) 380 381 if not last_trace.getPrevious().isUnassignedTrace(): 382 result = StatementDelVariable( 383 variable=self.variable, 384 version=self.variable_version, 385 tolerant=True, 386 source_ref=self.source_ref, 387 ) 388 else: 389 result = None 390 391 return ( 392 result, 393 "new_statements", 394 "Dropped propagated assignment statement to '%s'." 395 % self.getVariableName(), 396 ) 397 elif source.isExpressionFunctionCreation(): 398 # TODO: Prepare for inlining. 399 pass 400 else: 401 # More cases thinkable. 402 pass 403 404 return self, None, None 405 406 def needsReleasePreviousValue(self): 407 previous = self.variable_trace.getPrevious() 408 409 if previous.mustNotHaveValue(): 410 return False 411 elif previous.mustHaveValue(): 412 return True 413 else: 414 return None 415 416 @staticmethod 417 def getStatementNiceName(): 418 return "variable assignment statement" 419 420 def getTypeShape(self): 421 # Might be finalized, e.g. due to being dead code. 422 try: 423 source = self.subnode_source 424 except AttributeError: 425 return tshape_unknown 426 427 return source.getTypeShape() 428 429 430class StatementDelVariable(StatementBase): 431 """Deleting a variable. 432 433 All del forms that are not to attributes, slices, subscripts 434 use this. 435 436 The target can be any kind of variable, temporary, local, global, etc. 437 438 Deleting a variable is something we trace in a new version, this is 439 hidden behind target variable reference, which has this version once 440 it can be determined. 441 442 Tolerance means that the value might be unset. That can happen with 443 re-formulation of ours, and Python3 exception variables. 444 """ 445 446 kind = "STATEMENT_DEL_VARIABLE" 447 448 __slots__ = ( 449 "variable", 450 "variable_version", 451 "variable_trace", 452 "previous_trace", 453 "tolerant", 454 ) 455 456 def __init__(self, tolerant, source_ref, variable, version=None): 457 if type(tolerant) is str: 458 tolerant = tolerant == "True" 459 assert tolerant is True or tolerant is False, repr(tolerant) 460 461 if variable is not None: 462 if version is None: 463 version = variable.allocateTargetNumber() 464 465 StatementBase.__init__(self, source_ref=source_ref) 466 467 self.variable = variable 468 self.variable_version = version 469 470 self.variable_trace = None 471 self.previous_trace = None 472 473 self.tolerant = tolerant 474 475 def finalize(self): 476 del self.parent 477 del self.variable 478 del self.variable_trace 479 del self.previous_trace 480 481 def getDetails(self): 482 return { 483 "variable": self.variable, 484 "version": self.variable_version, 485 "tolerant": self.tolerant, 486 } 487 488 def getDetailsForDisplay(self): 489 return { 490 "variable_name": self.getVariableName(), 491 "is_temp": self.variable.isTempVariable(), 492 "var_type": self.variable.getVariableType(), 493 "owner": self.variable.getOwner().getCodeName(), 494 "tolerant": self.tolerant, 495 } 496 497 @classmethod 498 def fromXML(cls, provider, source_ref, **args): 499 assert cls is StatementDelVariable, cls 500 501 owner = getOwnerFromCodeName(args["owner"]) 502 503 if args["is_temp"] == "True": 504 variable = owner.createTempVariable( 505 args["variable_name"], temp_type=args["var_type"] 506 ) 507 else: 508 variable = owner.getProvidedVariable(args["variable_name"]) 509 510 del args["is_temp"] 511 del args["var_type"] 512 del args["owner"] 513 514 version = variable.allocateTargetNumber() 515 variable.version_number = max(variable.version_number, version) 516 517 return cls(variable=variable, source_ref=source_ref, **args) 518 519 def makeClone(self): 520 if self.variable is not None: 521 version = self.variable.allocateTargetNumber() 522 else: 523 version = None 524 525 return StatementDelVariable( 526 variable=self.variable, 527 version=version, 528 tolerant=self.tolerant, 529 source_ref=self.source_ref, 530 ) 531 532 # TODO: Value propagation needs to make a difference based on this. 533 def isTolerant(self): 534 return self.tolerant 535 536 def getVariableName(self): 537 return self.variable.getName() 538 539 def getVariableTrace(self): 540 return self.variable_trace 541 542 def getPreviousVariableTrace(self): 543 return self.previous_trace 544 545 def getVariable(self): 546 return self.variable 547 548 def setVariable(self, variable): 549 self.variable = variable 550 self.variable_version = variable.allocateTargetNumber() 551 552 def computeStatement(self, trace_collection): 553 variable = self.variable 554 555 # Special case, boolean temp variables need no "del". 556 # TODO: Later, these might not exist, if we forward propagate them not as "del" 557 # at all 558 if variable.isTempVariableBool(): 559 return ( 560 None, 561 "new_statements", 562 "Removed 'del' statement of boolean '%s' without effect." 563 % (self.getVariableName(),), 564 ) 565 566 self.previous_trace = trace_collection.getVariableCurrentTrace(variable) 567 568 # First eliminate us entirely if we can. 569 if self.previous_trace.mustNotHaveValue(): 570 if self.tolerant: 571 return ( 572 None, 573 "new_statements", 574 "Removed tolerant 'del' statement of '%s' without effect." 575 % (self.getVariableName(),), 576 ) 577 else: 578 if self.variable.isLocalVariable(): 579 result = makeRaiseExceptionReplacementStatement( 580 statement=self, 581 exception_type="UnboundLocalError", 582 exception_value="""local variable '%s' referenced before assignment""" 583 % variable.getName(), 584 ) 585 else: 586 result = makeRaiseExceptionReplacementStatement( 587 statement=self, 588 exception_type="NameError", 589 exception_value="""name '%s' is not defined""" 590 % variable.getName(), 591 ) 592 593 return trace_collection.computedStatementResult( 594 result, 595 "new_raise", 596 "Variable del of not initialized variable '%s'" 597 % variable.getName(), 598 ) 599 600 if not self.tolerant: 601 self.previous_trace.addNameUsage() 602 603 # TODO: Why doesn't this module variable check not follow from other checks done here, e.g. name usages. 604 # TODO: This currently cannot be done as releases do not create successor traces yet, although they 605 # probably should. 606 if isExperimental("del_optimization") and not variable.isModuleVariable(): 607 provider = trace_collection.getOwner() 608 609 if variable.hasAccessesOutsideOf(provider) is False: 610 last_trace = variable.getMatchingDelTrace(self) 611 612 if last_trace is not None and not last_trace.getMergeOrNameUsageCount(): 613 if not last_trace.getUsageCount(): 614 result = StatementReleaseVariable( 615 variable=variable, source_ref=self.source_ref 616 ) 617 618 return trace_collection.computedStatementResult( 619 result, 620 "new_statements", 621 "Changed del to release for variable '%s' not used afterwards." 622 % variable.getName(), 623 ) 624 625 # If not tolerant, we may exception exit now during the __del__ 626 if not self.tolerant and not self.previous_trace.mustHaveValue(): 627 trace_collection.onExceptionRaiseExit(BaseException) 628 629 # Record the deletion, needs to start a new version then. 630 self.variable_trace = trace_collection.onVariableDel( 631 variable=variable, version=self.variable_version, del_node=self 632 ) 633 634 # Any code could be run, note that. 635 trace_collection.onControlFlowEscape(self) 636 637 return self, None, None 638 639 def mayHaveSideEffects(self): 640 return True 641 642 def mayRaiseException(self, exception_type): 643 if self.tolerant: 644 return False 645 else: 646 if self.variable_trace is not None: 647 # Temporary variables deletions won't raise, just because we 648 # don't create them that way. We can avoid going through SSA in 649 # these cases. 650 if self.variable.isTempVariable(): 651 return False 652 653 # If SSA knows, that's fine. 654 if ( 655 self.previous_trace is not None 656 and self.previous_trace.mustHaveValue() 657 ): 658 return False 659 660 return True 661 662 663class StatementReleaseVariable(StatementBase): 664 """Releasing a variable. 665 666 Just release the value, which of course is not to be used afterwards. 667 668 Typical code: Function exit, try/finally release of temporary 669 variables. 670 """ 671 672 kind = "STATEMENT_RELEASE_VARIABLE" 673 674 __slots__ = "variable", "variable_trace" 675 676 def __init__(self, variable, source_ref): 677 assert variable is not None, source_ref 678 679 StatementBase.__init__(self, source_ref=source_ref) 680 681 self.variable = variable 682 683 self.variable_trace = None 684 685 def finalize(self): 686 del self.variable 687 del self.variable_trace 688 del self.parent 689 690 def getDetails(self): 691 return {"variable": self.variable} 692 693 def getDetailsForDisplay(self): 694 return { 695 "variable_name": self.variable.getName(), 696 "owner": self.variable.getOwner().getCodeName(), 697 } 698 699 @classmethod 700 def fromXML(cls, provider, source_ref, **args): 701 assert cls is StatementReleaseVariable, cls 702 703 owner = getOwnerFromCodeName(args["owner"]) 704 assert owner is not None, args["owner"] 705 706 variable = owner.getProvidedVariable(args["variable_name"]) 707 708 return cls(variable=variable, source_ref=source_ref) 709 710 def getVariable(self): 711 return self.variable 712 713 def getVariableTrace(self): 714 return self.variable_trace 715 716 def setVariable(self, variable): 717 self.variable = variable 718 719 def computeStatement(self, trace_collection): 720 if self.variable.isParameterVariable(): 721 if self.variable.getOwner().isAutoReleaseVariable(self.variable): 722 return ( 723 None, 724 "new_statements", 725 "Original parameter variable value %s is not released." 726 % (self.variable.getDescription()), 727 ) 728 729 self.variable_trace = trace_collection.getVariableCurrentTrace(self.variable) 730 731 if self.variable_trace.mustNotHaveValue(): 732 return ( 733 None, 734 "new_statements", 735 "Uninitialized %s is not released." % (self.variable.getDescription()), 736 ) 737 738 # TODO: Annotate value content as escaped, as destruction might run. 739 740 # Any code could be run, note that. 741 trace_collection.onControlFlowEscape(self) 742 743 # TODO: We might be able to remove ourselves based on the trace 744 # we belong to. 745 746 return self, None, None 747 748 def mayHaveSideEffects(self): 749 # May execute __del__ code, it would be sweet to be able to predict 750 # that another reference will still be active for a value though. 751 return True 752 753 def mayRaiseException(self, exception_type): 754 # By default, __del__ is not allowed to raise an exception. 755 return False 756