1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Variable closure taking. 19 20This is the completion of variable object completion. The variables were not 21immediately resolved to be bound to actual scopes, but are only now. 22 23Only after this is executed, variable reference nodes can be considered 24complete. 25""" 26 27from nuitka.nodes.AssignNodes import ( 28 StatementAssignmentVariable, 29 StatementDelVariable, 30 StatementReleaseVariable, 31) 32from nuitka.nodes.FunctionNodes import MaybeLocalVariableUsage 33from nuitka.nodes.LocalsDictNodes import ( 34 ExpressionLocalsVariableRef, 35 ExpressionLocalsVariableRefOrFallback, 36 StatementLocalsDictOperationDel, 37 StatementLocalsDictOperationSet, 38) 39from nuitka.nodes.NodeMakingHelpers import ( 40 makeConstantReplacementNode, 41 mergeStatements, 42) 43from nuitka.nodes.OperatorNodes import makeExpressionOperationBinaryInplace 44from nuitka.nodes.VariableRefNodes import ( 45 ExpressionTempVariableRef, 46 makeExpressionVariableRef, 47) 48from nuitka.PythonVersions import ( 49 getErrorMessageExecWithNestedFunction, 50 python_version, 51) 52from nuitka.Variables import isSharedAmongScopes, releaseSharedScopeInformation 53 54from .Operations import VisitorNoopMixin, visitTree 55from .ReformulationFunctionStatements import addFunctionVariableReleases 56from .ReformulationTryFinallyStatements import makeTryFinallyStatement 57from .SyntaxErrors import raiseSyntaxError 58 59# Note: We do the variable scope assignment, as an extra step from tree 60# building, because tree building creates the tree without any consideration of 61# evaluation order. And the ordered way these visitors are entered, will ensure 62# this order. 63 64# The main complexity is that there are two ways of visiting. One where variable 65# lookups are to be done immediately, and one where it is delayed. This is 66# basically class vs. function scope handling. 67 68 69class VariableClosureLookupVisitorPhase1(VisitorNoopMixin): 70 """Variable closure phase 1: Find assignments and early closure references. 71 72 In class context, a reference to a variable must be obeyed immediately, 73 so that "variable = variable" takes first "variable" as a closure and 74 then adds a new local "variable" to override it from there on. For the 75 not early closure case of a function, this will not be done and only 76 assignments shall add local variables, and references will be ignored 77 until phase 2. 78 """ 79 80 @staticmethod 81 def _handleNonLocal(node): 82 # Take closure variables for non-local declarations. 83 84 for ( 85 non_local_names, 86 user_provided, 87 source_ref, 88 ) in node.consumeNonlocalDeclarations(): 89 for non_local_name in non_local_names: 90 91 variable = node.takeVariableForClosure(variable_name=non_local_name) 92 93 node.getLocalsScope().registerClosureVariable(variable) 94 95 if variable.isModuleVariable() and user_provided: 96 raiseSyntaxError( 97 "no binding for nonlocal '%s' found" % (non_local_name), 98 source_ref, 99 ) 100 101 variable.addVariableUser(node) 102 103 @staticmethod 104 def _handleQualnameSetup(node): 105 if node.qualname_setup is not None: 106 provider = node.getParentVariableProvider() 107 108 if node.isExpressionClassBody(): 109 class_variable_name, qualname_assign = node.qualname_setup 110 111 if provider.hasProvidedVariable(class_variable_name): 112 class_variable = provider.getVariableForReference( 113 class_variable_name 114 ) 115 116 if class_variable.isModuleVariable(): 117 qualname_node = qualname_assign.subnode_source 118 119 new_node = makeConstantReplacementNode( 120 constant=class_variable.getName(), 121 node=qualname_node, 122 user_provided=True, 123 ) 124 125 parent = qualname_node.parent 126 qualname_node.finalize() 127 parent.replaceChild(qualname_node, new_node) 128 129 node.qualname_provider = node.getParentModule() 130 else: 131 if provider.hasProvidedVariable(node.qualname_setup): 132 function_variable = provider.getVariableForReference( 133 node.qualname_setup 134 ) 135 136 if function_variable.isModuleVariable(): 137 node.qualname_provider = node.getParentModule() 138 139 # TODO: Actually for nested global classes, this approach 140 # may not work, as their "qualname" will be wrong. In that 141 # case a dedicated node for "qualname" references might be 142 # needed. 143 144 node.qualname_setup = None 145 146 @staticmethod 147 def _shouldUseLocalsDict(provider, variable_name): 148 return provider.isExpressionClassBody() and ( 149 not provider.hasProvidedVariable(variable_name) 150 or provider.getProvidedVariable(variable_name).getOwner() is provider 151 ) 152 153 def onLeaveNode(self, node): 154 if node.isStatementAssignmentVariableName(): 155 variable_name = node.getVariableName() 156 provider = node.provider 157 158 # Classes always assign to locals dictionary except for closure 159 # variables taken. 160 if self._shouldUseLocalsDict(provider, variable_name): 161 if node.subnode_source.isExpressionOperationInplace(): 162 temp_scope = provider.allocateTempScope("class_inplace") 163 164 tmp_variable = provider.allocateTempVariable( 165 temp_scope=temp_scope, name="value" 166 ) 167 168 statements = mergeStatements( 169 statements=( 170 StatementAssignmentVariable( 171 variable=tmp_variable, 172 source=node.subnode_source.subnode_left, 173 source_ref=node.source_ref, 174 ), 175 makeTryFinallyStatement( 176 provider=provider, 177 tried=( 178 StatementAssignmentVariable( 179 variable=tmp_variable, 180 source=makeExpressionOperationBinaryInplace( 181 left=ExpressionTempVariableRef( 182 variable=tmp_variable, 183 source_ref=node.source_ref, 184 ), 185 right=node.subnode_source.subnode_right, 186 operator=node.subnode_source.getOperator(), 187 source_ref=node.source_ref, 188 ), 189 source_ref=node.source_ref, 190 ), 191 StatementLocalsDictOperationSet( 192 locals_scope=provider.getLocalsScope(), 193 variable_name=variable_name, 194 value=ExpressionTempVariableRef( 195 variable=tmp_variable, 196 source_ref=node.source_ref, 197 ), 198 source_ref=node.source_ref, 199 ), 200 ), 201 final=StatementReleaseVariable( 202 variable=tmp_variable, source_ref=node.source_ref 203 ), 204 source_ref=node.source_ref, 205 ), 206 ) 207 ) 208 209 node.parent.replaceStatement(node, statements) 210 211 else: 212 new_node = StatementLocalsDictOperationSet( 213 locals_scope=provider.getLocalsScope(), 214 variable_name=variable_name, 215 value=node.subnode_source, 216 source_ref=node.source_ref, 217 ) 218 219 node.parent.replaceChild(node, new_node) 220 else: 221 variable = provider.getVariableForAssignment( 222 variable_name=variable_name 223 ) 224 225 new_node = StatementAssignmentVariable( 226 variable=variable, 227 source=node.subnode_source, 228 source_ref=node.source_ref, 229 ) 230 231 variable.addVariableUser(provider) 232 233 node.parent.replaceChild(node, new_node) 234 235 del node.parent 236 del node.provider 237 elif node.isStatementDelVariableName(): 238 variable_name = node.getVariableName() 239 240 provider = node.provider 241 242 if self._shouldUseLocalsDict(provider, variable_name): 243 # Classes always assign to locals dictionary except for closure 244 # variables taken. 245 new_node = StatementLocalsDictOperationDel( 246 locals_scope=provider.getLocalsScope(), 247 variable_name=variable_name, 248 tolerant=node.tolerant, 249 source_ref=node.source_ref, 250 ) 251 else: 252 variable = provider.getVariableForAssignment( 253 variable_name=variable_name 254 ) 255 256 new_node = StatementDelVariable( 257 variable=variable, 258 tolerant=node.tolerant, 259 source_ref=node.source_ref, 260 ) 261 262 variable.addVariableUser(provider) 263 264 parent = node.parent 265 node.finalize() 266 267 parent.replaceChild(node, new_node) 268 269 def onEnterNode(self, node): 270 # Mighty complex code with lots of branches, but we aim to get rid of it. 271 # pylint: disable=too-many-branches 272 273 if node.isExpressionVariableNameRef(): 274 provider = node.provider 275 276 if provider.isExpressionClassBody(): 277 if node.needsFallback(): 278 variable = provider.getVariableForReference( 279 variable_name=node.getVariableName() 280 ) 281 282 new_node = ExpressionLocalsVariableRefOrFallback( 283 locals_scope=provider.getLocalsScope(), 284 variable_name=node.getVariableName(), 285 fallback=makeExpressionVariableRef( 286 variable=variable, 287 locals_scope=provider.getLocalsScope(), 288 source_ref=node.source_ref, 289 ), 290 source_ref=node.source_ref, 291 ) 292 293 variable.addVariableUser(provider) 294 else: 295 new_node = ExpressionLocalsVariableRef( 296 locals_scope=provider.getLocalsScope(), 297 variable_name=node.getVariableName(), 298 source_ref=node.source_ref, 299 ) 300 301 parent = node.parent 302 node.finalize() 303 304 parent.replaceChild(node, new_node) 305 elif node.isExpressionTempVariableRef(): 306 if node.getVariable().getOwner() != node.getParentVariableProvider(): 307 node.getParentVariableProvider().addClosureVariable(node.getVariable()) 308 elif node.isExpressionGeneratorObjectBody(): 309 if python_version >= 0x300: 310 self._handleNonLocal(node) 311 312 # Only Python3.4 or later allows for generators to have qualname. 313 if python_version >= 0x340: 314 self._handleQualnameSetup(node) 315 elif node.isExpressionCoroutineObjectBody(): 316 self._handleNonLocal(node) 317 318 self._handleQualnameSetup(node) 319 elif node.isExpressionAsyncgenObjectBody(): 320 self._handleNonLocal(node) 321 322 self._handleQualnameSetup(node) 323 elif node.isExpressionClassBody(): 324 if python_version >= 0x300: 325 self._handleNonLocal(node) 326 327 # Python3.4 allows for class declarations to be made global, even 328 # after they were declared, so we need to fix this up. 329 if python_version >= 0x340: 330 self._handleQualnameSetup(node) 331 elif node.isExpressionFunctionBody(): 332 if python_version >= 0x300: 333 self._handleNonLocal(node) 334 335 # Python 3.4 allows for class declarations to be made global, even 336 # after they were declared, so we need to fix this up. 337 if python_version >= 0x340: 338 self._handleQualnameSetup(node) 339 # Check if continue and break are properly in loops. If not, raise a 340 # syntax error. 341 elif node.isStatementLoopBreak() or node.isStatementLoopContinue(): 342 current = node 343 344 while True: 345 current = current.getParent() 346 347 if current.isStatementLoop(): 348 break 349 350 if current.isParentVariableProvider(): 351 if node.isStatementLoopContinue(): 352 message = "'continue' not properly in loop" 353 else: 354 message = "'break' outside loop" 355 356 raiseSyntaxError(message, node.getSourceReference()) 357 358 359class VariableClosureLookupVisitorPhase2(VisitorNoopMixin): 360 """Variable closure phase 2: Find assignments and references. 361 362 In class context, a reference to a variable must be obeyed immediately, 363 so that "variable = variable" takes first "variable" as a closure and 364 then adds a new local "variable" to override it from there on. 365 366 So, assignments for early closure, accesses will already have a 367 variable set now, the others, only in this phase. 368 """ 369 370 @staticmethod 371 def _attachVariable(node, provider): 372 # print "Late reference", node.getVariableName(), "for", provider, "caused at", node, "of", node.getParent() 373 374 variable_name = node.getVariableName() 375 376 variable = provider.getVariableForReference(variable_name=variable_name) 377 378 # Need to catch functions with "exec" and closure variables not allowed. 379 if python_version < 0x300 and provider.isExpressionFunctionBodyBase(): 380 was_taken = provider.hasTakenVariable(variable_name) 381 382 if not was_taken and variable.getOwner() is not provider: 383 parent_provider = provider.getParentVariableProvider() 384 385 while parent_provider.isExpressionClassBody(): 386 parent_provider = parent_provider.getParentVariableProvider() 387 388 if ( 389 parent_provider.isExpressionFunctionBody() 390 and parent_provider.isUnqualifiedExec() 391 ): 392 raiseSyntaxError( 393 getErrorMessageExecWithNestedFunction() 394 % parent_provider.getName(), 395 node.getSourceReference(), 396 display_line=False, # Wrong line anyway 397 ) 398 399 return variable 400 401 def onEnterNode(self, node): 402 if node.isExpressionVariableNameRef(): 403 provider = node.provider 404 405 try: 406 variable = self._attachVariable(node, provider) 407 except MaybeLocalVariableUsage: 408 variable_name = node.getVariableName() 409 410 new_node = ExpressionLocalsVariableRefOrFallback( 411 locals_scope=provider.getLocalsScope(), 412 variable_name=variable_name, 413 fallback=makeExpressionVariableRef( 414 variable=node.getParentModule().getVariableForReference( 415 variable_name 416 ), 417 locals_scope=provider.getLocalsScope(), 418 source_ref=node.source_ref, 419 ), 420 source_ref=node.source_ref, 421 ) 422 else: 423 new_node = makeExpressionVariableRef( 424 variable=variable, 425 locals_scope=provider.getLocalsScope(), 426 source_ref=node.source_ref, 427 ) 428 429 variable.addVariableUser(provider) 430 431 parent = node.parent 432 node.finalize() 433 434 parent.replaceChild(node, new_node) 435 436 437class VariableClosureLookupVisitorPhase3(VisitorNoopMixin): 438 """Variable closure phase 3: Find errors and complete frame variables. 439 440 In this phase, we can do some fix-ups and find errors. We might e.g. 441 detect that a "del" was executed on a shared variable, which is not 442 allowed for Python 2.x, so it must be caught. The parsing wouldn't do 443 that. 444 445 Also, frame objects for functions should learn their variable names. 446 """ 447 448 def onEnterNode(self, node): 449 if python_version < 0x300 and node.isStatementDelVariable(): 450 variable = node.getVariable() 451 452 if not variable.isModuleVariable() and isSharedAmongScopes(variable): 453 raiseSyntaxError( 454 """\ 455can not delete variable '%s' referenced in nested scope""" 456 % (variable.getName()), 457 node.getSourceReference(), 458 ) 459 elif node.isStatementsFrame(): 460 node.updateLocalNames() 461 elif node.isExpressionFunctionBodyBase(): 462 addFunctionVariableReleases(node) 463 464 # Python3 is influenced by the mere use of a variable named as 465 # "super". So we need to prepare ability to take closure. 466 if node.hasFlag("has_super"): 467 if not node.hasVariableName("__class__"): 468 class_var = node.takeVariableForClosure("__class__") 469 class_var.addVariableUser(node) 470 471 node.getLocalsScope().registerClosureVariable(class_var) 472 while node != class_var.getOwner(): 473 node = node.getParentVariableProvider() 474 node.getLocalsScope().registerClosureVariable(class_var) 475 476 477def completeVariableClosures(tree): 478 visitors = ( 479 VariableClosureLookupVisitorPhase1(), 480 VariableClosureLookupVisitorPhase2(), 481 VariableClosureLookupVisitorPhase3(), 482 ) 483 484 for visitor in visitors: 485 visitTree(tree, visitor) 486 487 # Only used to detect syntax errors. 488 releaseSharedScopeInformation(tree) 489