1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Node for variable references. 19 20These represent all variable references in the node tree. Can be in assignments 21and its expressions, changing the meaning of course dramatically. 22 23""" 24 25from nuitka import Builtins, Variables 26from nuitka.ModuleRegistry import getOwnerFromCodeName 27from nuitka.PythonVersions import python_version 28 29from .DictionaryNodes import ( 30 ExpressionDictOperationGet, 31 ExpressionDictOperationIn, 32 ExpressionDictOperationNotIn, 33 StatementDictOperationRemove, 34 StatementDictOperationSet, 35) 36from .ExpressionBases import ExpressionBase 37from .ModuleAttributeNodes import ( 38 ExpressionModuleAttributeLoaderRef, 39 ExpressionModuleAttributeNameRef, 40 ExpressionModuleAttributePackageRef, 41 ExpressionModuleAttributeSpecRef, 42) 43from .NodeMakingHelpers import ( 44 makeRaiseExceptionReplacementExpression, 45 makeRaiseTypeErrorExceptionReplacementFromTemplateAndValue, 46) 47from .shapes.StandardShapes import tshape_unknown 48 49 50class ExpressionVariableNameRef(ExpressionBase): 51 """These are used before the actual variable object is known from VariableClosure.""" 52 53 kind = "EXPRESSION_VARIABLE_NAME_REF" 54 55 __slots__ = "variable_name", "provider" 56 57 def __init__(self, provider, variable_name, source_ref): 58 assert not provider.isExpressionOutlineBody(), source_ref 59 60 ExpressionBase.__init__(self, source_ref=source_ref) 61 62 self.variable_name = variable_name 63 64 self.provider = provider 65 66 def finalize(self): 67 del self.parent 68 del self.provider 69 70 @staticmethod 71 def isExpressionVariableNameRef(): 72 return True 73 74 def getDetails(self): 75 return {"variable_name": self.variable_name, "provider": self.provider} 76 77 def getVariableName(self): 78 return self.variable_name 79 80 def computeExpressionRaw(self, trace_collection): 81 return self, None, None 82 83 @staticmethod 84 def needsFallback(): 85 return True 86 87 88class ExpressionVariableLocalNameRef(ExpressionVariableNameRef): 89 """These are used before the actual variable object is known from VariableClosure. 90 91 The special thing about this as opposed to ExpressionVariableNameRef is that 92 these must remain local names and cannot fallback to outside scopes. This is 93 used for "__annotations__". 94 95 """ 96 97 kind = "EXPRESSION_VARIABLE_LOCAL_NAME_REF" 98 99 @staticmethod 100 def needsFallback(): 101 return False 102 103 104class ExpressionVariableRefBase(ExpressionBase): 105 # Base classes can be abstract, pylint: disable=abstract-method 106 107 __slots__ = "variable", "variable_trace" 108 109 def __init__(self, variable, source_ref): 110 ExpressionBase.__init__(self, source_ref=source_ref) 111 112 self.variable = variable 113 self.variable_trace = None 114 115 def finalize(self): 116 del self.parent 117 del self.variable 118 del self.variable_trace 119 120 def getVariableName(self): 121 return self.variable.getName() 122 123 def getVariable(self): 124 return self.variable 125 126 def getVariableTrace(self): 127 return self.variable_trace 128 129 def getTypeShape(self): 130 if self.variable_trace is None: 131 return tshape_unknown 132 else: 133 return self.variable_trace.getTypeShape() 134 135 def onContentEscapes(self, trace_collection): 136 trace_collection.onVariableContentEscapes(self.variable) 137 138 def computeExpressionLen(self, len_node, trace_collection): 139 if self.variable_trace is not None and self.variable_trace.isAssignTrace(): 140 value = self.variable_trace.getAssignNode().subnode_source 141 142 shape = value.getValueShape() 143 144 has_len = shape.hasShapeSlotLen() 145 146 if has_len is False: 147 return makeRaiseTypeErrorExceptionReplacementFromTemplateAndValue( 148 template="object of type '%s' has no len()", 149 operation="len", 150 original_node=len_node, 151 value_node=self, 152 ) 153 elif has_len is True: 154 iter_length = value.getIterationLength() 155 156 if iter_length is not None: 157 from .ConstantRefNodes import makeConstantRefNode 158 159 result = makeConstantRefNode( 160 constant=int(iter_length), # make sure to downcast long 161 source_ref=len_node.getSourceReference(), 162 ) 163 164 return ( 165 result, 166 "new_constant", 167 "Predicted 'len' result of variable.", 168 ) 169 170 # The variable itself is to be considered escaped. 171 trace_collection.markActiveVariableAsEscaped(self.variable) 172 173 # Any code could be run, note that. 174 trace_collection.onControlFlowEscape(self) 175 176 # Any exception may be raised. 177 trace_collection.onExceptionRaiseExit(BaseException) 178 179 return len_node, None, None 180 181 def computeExpressionAttribute(self, lookup_node, attribute_name, trace_collection): 182 # Any code could be run, note that. 183 trace_collection.onControlFlowEscape(self) 184 185 # The variable itself is to be considered escaped. 186 trace_collection.markActiveVariableAsEscaped(self.variable) 187 188 if not self.isKnownToHaveAttribute(attribute_name): 189 trace_collection.onExceptionRaiseExit(BaseException) 190 191 return lookup_node, None, None 192 193 def computeExpressionComparisonIn(self, in_node, value_node, trace_collection): 194 tags = None 195 message = None 196 197 # Any code could be run, note that. 198 trace_collection.onControlFlowEscape(in_node) 199 200 if self.variable_trace.hasShapeDictionaryExact(): 201 tags = "new_expression" 202 message = """\ 203Check '%s' on dictionary lowered to dictionary '%s'.""" % ( 204 in_node.comparator, 205 in_node.comparator, 206 ) 207 208 if in_node.comparator == "In": 209 in_node = ExpressionDictOperationIn( 210 key=value_node, 211 dict_arg=self, 212 source_ref=in_node.getSourceReference(), 213 ) 214 else: 215 in_node = ExpressionDictOperationNotIn( 216 key=value_node, 217 dict_arg=self, 218 source_ref=in_node.getSourceReference(), 219 ) 220 221 # Any exception may be raised. 222 if in_node.mayRaiseException(BaseException): 223 trace_collection.onExceptionRaiseExit(BaseException) 224 225 return in_node, tags, message 226 227 def computeExpressionSetSubscript( 228 self, set_node, subscript, value_node, trace_collection 229 ): 230 tags = None 231 message = None 232 233 # By default, an subscript may change everything about the lookup 234 # source. 235 if self.variable_trace.hasShapeDictionaryExact(): 236 result = StatementDictOperationSet( 237 dict_arg=self, 238 key=subscript, 239 value=value_node, 240 source_ref=set_node.getSourceReference(), 241 ) 242 change_tags = "new_statements" 243 change_desc = """\ 244Subscript assignment to dictionary lowered to dictionary assignment.""" 245 246 trace_collection.removeKnowledge(self) 247 248 result2, change_tags2, change_desc2 = result.computeStatementOperation( 249 trace_collection 250 ) 251 252 if result2 is not result: 253 trace_collection.signalChange( 254 tags=change_tags, 255 source_ref=self.source_ref, 256 message=change_desc, 257 ) 258 259 return result2, change_tags2, change_desc2 260 else: 261 return result, change_tags, change_desc 262 263 trace_collection.removeKnowledge(self) 264 265 # Any code could be run, note that. 266 trace_collection.onControlFlowEscape(self) 267 268 # Any exception might be raised. 269 if set_node.mayRaiseException(BaseException): 270 trace_collection.onExceptionRaiseExit(BaseException) 271 272 return set_node, tags, message 273 274 def computeExpressionDelSubscript(self, del_node, subscript, trace_collection): 275 tags = None 276 message = None 277 278 if self.variable_trace.hasShapeDictionaryExact(): 279 result = StatementDictOperationRemove( 280 dict_arg=self, 281 key=subscript, 282 source_ref=del_node.getSourceReference(), 283 ) 284 change_tags = "new_statements" 285 change_desc = """\ 286Subscript del to dictionary lowered to dictionary del.""" 287 288 trace_collection.removeKnowledge(self) 289 290 result2, change_tags2, change_desc2 = result.computeStatementOperation( 291 trace_collection 292 ) 293 294 if result2 is not result: 295 trace_collection.signalChange( 296 tags=change_tags, 297 source_ref=self.source_ref, 298 message=change_desc, 299 ) 300 301 return result2, change_tags2, change_desc2 302 else: 303 return result, change_tags, change_desc 304 305 # By default, an subscript may change everything about the lookup 306 # source. 307 # Any code could be run, note that. 308 trace_collection.onControlFlowEscape(self) 309 310 # Any exception might be raised. 311 if del_node.mayRaiseException(BaseException): 312 trace_collection.onExceptionRaiseExit(BaseException) 313 314 return del_node, tags, message 315 316 def computeExpressionSubscript(self, lookup_node, subscript, trace_collection): 317 tags = None 318 message = None 319 320 if self.variable_trace.hasShapeDictionaryExact(): 321 return trace_collection.computedExpressionResult( 322 expression=ExpressionDictOperationGet( 323 dict_arg=self, 324 key=subscript, 325 source_ref=lookup_node.getSourceReference(), 326 ), 327 change_tags="new_expression", 328 change_desc="""\ 329Subscript look-up to dictionary lowered to dictionary look-up.""", 330 ) 331 332 # Any code could be run, note that. 333 trace_collection.onControlFlowEscape(self) 334 335 # Any exception might be raised. 336 if lookup_node.mayRaiseException(BaseException): 337 trace_collection.onExceptionRaiseExit(BaseException) 338 339 return lookup_node, tags, message 340 341 def _applyReplacement(self, trace_collection, replacement): 342 trace_collection.signalChange( 343 "new_expression", 344 self.source_ref, 345 "Value propagated for '%s' from '%s'." 346 % (self.variable.getName(), replacement.getSourceReference().getAsString()), 347 ) 348 349 # Special case for in-place assignments. 350 if self.parent.isExpressionOperationInplace(): 351 statement = self.parent.parent 352 353 if statement.isStatementAssignmentVariable(): 354 statement.unmarkAsInplaceSuspect() 355 356 # Need to compute the replacement still. 357 return replacement.computeExpressionRaw(trace_collection) 358 359 360_hard_names = ("dir", "eval", "exec", "execfile", "locals", "vars", "super") 361 362 363class ExpressionVariableRef(ExpressionVariableRefBase): 364 kind = "EXPRESSION_VARIABLE_REF" 365 366 __slots__ = () 367 368 def __init__(self, variable, source_ref): 369 assert variable is not None 370 371 ExpressionVariableRefBase.__init__( 372 self, variable=variable, source_ref=source_ref 373 ) 374 375 @staticmethod 376 def isExpressionVariableRef(): 377 return True 378 379 def getDetails(self): 380 return {"variable": self.variable} 381 382 def getDetailsForDisplay(self): 383 return { 384 "variable_name": self.variable.getName(), 385 "owner": self.variable.getOwner().getCodeName(), 386 } 387 388 @classmethod 389 def fromXML(cls, provider, source_ref, **args): 390 assert cls is ExpressionVariableRef, cls 391 392 owner = getOwnerFromCodeName(args["owner"]) 393 variable = owner.getProvidedVariable(args["variable_name"]) 394 395 return cls(variable=variable, source_ref=source_ref) 396 397 @staticmethod 398 def isTargetVariableRef(): 399 return False 400 401 def getVariable(self): 402 return self.variable 403 404 def setVariable(self, variable): 405 assert isinstance(variable, Variables.Variable), repr(variable) 406 407 self.variable = variable 408 409 def computeExpressionRaw(self, trace_collection): 410 # Terribly detailed, pylint: disable=too-many-branches,too-many-statements 411 412 variable = self.variable 413 assert variable is not None 414 415 self.variable_trace = trace_collection.getVariableCurrentTrace( 416 variable=variable 417 ) 418 419 replacement = self.variable_trace.getReplacementNode(self) 420 if replacement is not None: 421 return self._applyReplacement(trace_collection, replacement) 422 423 if not self.variable_trace.mustHaveValue(): 424 # TODO: This could be way more specific surely, either NameError or UnboundLocalError 425 # could be decided from context. 426 trace_collection.onExceptionRaiseExit(BaseException) 427 428 if variable.isModuleVariable() and variable.hasDefiniteWrites() is False: 429 variable_name = self.variable.getName() 430 431 if variable_name in Builtins.builtin_exception_names: 432 if not self.variable.getOwner().getLocalsScope().isEscaped(): 433 from .BuiltinRefNodes import ExpressionBuiltinExceptionRef 434 435 new_node = ExpressionBuiltinExceptionRef( 436 exception_name=self.variable.getName(), 437 source_ref=self.source_ref, 438 ) 439 440 change_tags = "new_builtin_ref" 441 change_desc = """\ 442Module variable '%s' found to be built-in exception reference.""" % ( 443 variable_name 444 ) 445 else: 446 self.variable_trace.addUsage() 447 448 new_node = self 449 change_tags = None 450 change_desc = None 451 452 elif variable_name in Builtins.builtin_names: 453 if ( 454 variable_name in _hard_names 455 or not self.variable.getOwner().getLocalsScope().isEscaped() 456 ): 457 from .BuiltinRefNodes import makeExpressionBuiltinRef 458 459 new_node = makeExpressionBuiltinRef( 460 builtin_name=variable_name, 461 locals_scope=self.getFunctionsLocalsScope(), 462 source_ref=self.source_ref, 463 ) 464 465 change_tags = "new_builtin_ref" 466 change_desc = """\ 467Module variable '%s' found to be built-in reference.""" % ( 468 variable_name 469 ) 470 else: 471 self.variable_trace.addUsage() 472 473 new_node = self 474 change_tags = None 475 change_desc = None 476 elif variable_name == "__name__": 477 new_node = ExpressionModuleAttributeNameRef( 478 variable=variable, source_ref=self.source_ref 479 ) 480 481 change_tags = "new_expression" 482 change_desc = """\ 483Replaced read-only module attribute '__name__' with module attribute reference.""" 484 elif variable_name == "__package__": 485 new_node = ExpressionModuleAttributePackageRef( 486 variable=variable, source_ref=self.source_ref 487 ) 488 489 change_tags = "new_expression" 490 change_desc = """\ 491Replaced read-only module attribute '__package__' with module attribute reference.""" 492 elif variable_name == "__loader__" and python_version >= 0x300: 493 new_node = ExpressionModuleAttributeLoaderRef( 494 variable=variable, source_ref=self.source_ref 495 ) 496 497 change_tags = "new_expression" 498 change_desc = """\ 499Replaced read-only module attribute '__loader__' with module attribute reference.""" 500 elif variable_name == "__spec__" and python_version >= 0x340: 501 new_node = ExpressionModuleAttributeSpecRef( 502 variable=variable, source_ref=self.source_ref 503 ) 504 505 change_tags = "new_expression" 506 change_desc = """\ 507Replaced read-only module attribute '__spec__' with module attribute reference.""" 508 else: 509 self.variable_trace.addUsage() 510 511 # Probably should give a warning once about it. 512 new_node = self 513 change_tags = None 514 change_desc = None 515 516 return new_node, change_tags, change_desc 517 518 self.variable_trace.addUsage() 519 520 if self.variable_trace.mustNotHaveValue(): 521 assert self.variable.isLocalVariable(), self.variable 522 523 variable_name = self.variable.getName() 524 525 result = makeRaiseExceptionReplacementExpression( 526 expression=self, 527 exception_type="UnboundLocalError", 528 exception_value="""local variable '%s' referenced before assignment""" 529 % variable_name, 530 ) 531 532 return ( 533 result, 534 "new_raise", 535 "Variable access of not initialized variable '%s'" % variable_name, 536 ) 537 538 return self, None, None 539 540 def computeExpressionCall(self, call_node, call_args, call_kw, trace_collection): 541 # The called and the arguments escape for good. 542 self.onContentEscapes(trace_collection) 543 if call_args is not None: 544 call_args.onContentEscapes(trace_collection) 545 if call_kw is not None: 546 call_kw.onContentEscapes(trace_collection) 547 548 # Any code could be run, note that. 549 trace_collection.onControlFlowEscape(self) 550 551 # Any exception may be raised. 552 trace_collection.onExceptionRaiseExit(BaseException) 553 554 if ( 555 self.variable.getName() in _hard_names 556 and self.variable.isIncompleteModuleVariable() 557 ): 558 # Just inform the collection that all escaped. 559 trace_collection.onLocalsUsage(locals_scope=self.getFunctionsLocalsScope()) 560 561 return call_node, None, None 562 563 def hasShapeDictionaryExact(self): 564 return self.variable_trace.hasShapeDictionaryExact() 565 566 def getTruthValue(self): 567 return self.variable_trace.getTruthValue() 568 569 @staticmethod 570 def isKnownToBeIterable(count): 571 return None 572 573 def mayHaveSideEffects(self): 574 return not self.variable_trace.mustHaveValue() 575 576 def mayRaiseException(self, exception_type): 577 return self.variable_trace is None or not self.variable_trace.mustHaveValue() 578 579 def mayRaiseExceptionBool(self, exception_type): 580 return ( 581 self.variable_trace is None 582 or not self.variable_trace.mustHaveValue() 583 or not self.variable_trace.getTypeShape().hasShapeSlotBool() 584 ) 585 586 def getFunctionsLocalsScope(self): 587 return self.getParentVariableProvider().getLocalsScope() 588 589 590class ExpressionVariableOrBuiltinRef(ExpressionVariableRef): 591 kind = "EXPRESSION_VARIABLE_OR_BUILTIN_REF" 592 593 __slots__ = ("locals_scope",) 594 595 def __init__(self, variable, locals_scope, source_ref): 596 ExpressionVariableRef.__init__(self, variable=variable, source_ref=source_ref) 597 598 self.locals_scope = locals_scope 599 600 def getDetails(self): 601 return {"variable": self.variable, "locals_scope": self.locals_scope} 602 603 def getFunctionsLocalsScope(self): 604 return self.locals_scope 605 606 607def makeExpressionVariableRef(variable, locals_scope, source_ref): 608 if variable.getName() in _hard_names: 609 return ExpressionVariableOrBuiltinRef( 610 variable=variable, locals_scope=locals_scope, source_ref=source_ref 611 ) 612 else: 613 return ExpressionVariableRef(variable=variable, source_ref=source_ref) 614 615 616class ExpressionTempVariableRef(ExpressionVariableRefBase): 617 kind = "EXPRESSION_TEMP_VARIABLE_REF" 618 619 def __init__(self, variable, source_ref): 620 assert variable.isTempVariable() 621 622 ExpressionVariableRefBase.__init__( 623 self, variable=variable, source_ref=source_ref 624 ) 625 626 def getDetailsForDisplay(self): 627 return { 628 "temp_name": self.variable.getName(), 629 "owner": self.variable.getOwner().getCodeName(), 630 } 631 632 def getDetails(self): 633 return {"variable": self.variable} 634 635 @classmethod 636 def fromXML(cls, provider, source_ref, **args): 637 assert cls is ExpressionTempVariableRef, cls 638 639 owner = getOwnerFromCodeName(args["owner"]) 640 641 variable = owner.getTempVariable(None, args["temp_name"]) 642 643 return cls(variable=variable, source_ref=source_ref) 644 645 @staticmethod 646 def isTargetVariableRef(): 647 return False 648 649 def computeExpressionRaw(self, trace_collection): 650 self.variable_trace = trace_collection.getVariableCurrentTrace( 651 variable=self.variable 652 ) 653 654 replacement = self.variable_trace.getReplacementNode(self) 655 if replacement is not None: 656 return self._applyReplacement(trace_collection, replacement) 657 658 self.variable_trace.addUsage() 659 660 # Nothing to do here. 661 return self, None, None 662 663 def computeExpressionNext1(self, next_node, trace_collection): 664 may_not_raise = False 665 666 if self.variable_trace.isAssignTrace(): 667 value = self.variable_trace.getAssignNode().subnode_source 668 669 # TODO: Add iteration handles to trace collections instead. 670 current_index = trace_collection.getIteratorNextCount(value) 671 trace_collection.onIteratorNext(value) 672 673 if value.hasShapeSlotNext(): 674 if ( 675 current_index is not None 676 # TODO: Change to iteration handles. 677 and value.isKnownToBeIterableAtMin(current_index + 1) 678 ): 679 may_not_raise = True 680 681 # TODO: Make use of this 682 # candidate = value.getIterationValue(current_index) 683 684 # if False: 685 # and value.canPredictIterationValues() 686 # return ( 687 # candidate, 688 # "new_expression", 689 # "Predicted 'next' value from iteration.", 690 # ) 691 else: 692 # TODO: Could ask it about exception predictability for that case 693 # or warn about it at least. 694 pass 695 # assert False, value 696 697 self.onContentEscapes(trace_collection) 698 699 # Any code could be run, note that. 700 trace_collection.onControlFlowEscape(self) 701 702 # Any exception may be raised. 703 trace_collection.onExceptionRaiseExit(BaseException) 704 705 return may_not_raise, (next_node, None, None) 706 707 @staticmethod 708 def mayHaveSideEffects(): 709 # Can't happen with temporary variables, unless we used them wrongly. 710 return False 711 712 @staticmethod 713 def mayRaiseException(exception_type): 714 # Can't happen with temporary variables, unless we used them wrongly. 715 return False 716 717 def mayRaiseExceptionImportName(self, exception_type, import_name): 718 if self.variable_trace is not None and self.variable_trace.isAssignTrace(): 719 return self.variable_trace.getAssignNode().subnode_source.mayRaiseExceptionImportName( 720 exception_type, import_name 721 ) 722 723 else: 724 return True 725 726 @staticmethod 727 def isKnownToBeIterableAtMin(count): 728 # TODO: See through the variable current trace. 729 return None 730