1#     Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com
2#
3#     Part of "Nuitka", an optimizing Python compiler that is compatible and
4#     integrates with CPython, but also works on its own.
5#
6#     Licensed under the Apache License, Version 2.0 (the "License");
7#     you may not use this file except in compliance with the License.
8#     You may obtain a copy of the License at
9#
10#        http://www.apache.org/licenses/LICENSE-2.0
11#
12#     Unless required by applicable law or agreed to in writing, software
13#     distributed under the License is distributed on an "AS IS" BASIS,
14#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15#     See the License for the specific language governing permissions and
16#     limitations under the License.
17#
18""" Variable closure taking.
19
20This is the completion of variable object completion. The variables were not
21immediately resolved to be bound to actual scopes, but are only now.
22
23Only after this is executed, variable reference nodes can be considered
24complete.
25"""
26
27from nuitka.nodes.AssignNodes import (
28    StatementAssignmentVariable,
29    StatementDelVariable,
30    StatementReleaseVariable,
31)
32from nuitka.nodes.FunctionNodes import MaybeLocalVariableUsage
33from nuitka.nodes.LocalsDictNodes import (
34    ExpressionLocalsVariableRef,
35    ExpressionLocalsVariableRefOrFallback,
36    StatementLocalsDictOperationDel,
37    StatementLocalsDictOperationSet,
38)
39from nuitka.nodes.NodeMakingHelpers import (
40    makeConstantReplacementNode,
41    mergeStatements,
42)
43from nuitka.nodes.OperatorNodes import makeExpressionOperationBinaryInplace
44from nuitka.nodes.VariableRefNodes import (
45    ExpressionTempVariableRef,
46    makeExpressionVariableRef,
47)
48from nuitka.PythonVersions import (
49    getErrorMessageExecWithNestedFunction,
50    python_version,
51)
52from nuitka.Variables import isSharedAmongScopes, releaseSharedScopeInformation
53
54from .Operations import VisitorNoopMixin, visitTree
55from .ReformulationFunctionStatements import addFunctionVariableReleases
56from .ReformulationTryFinallyStatements import makeTryFinallyStatement
57from .SyntaxErrors import raiseSyntaxError
58
59# Note: We do the variable scope assignment, as an extra step from tree
60# building, because tree building creates the tree without any consideration of
61# evaluation order. And the ordered way these visitors are entered, will ensure
62# this order.
63
64# The main complexity is that there are two ways of visiting. One where variable
65# lookups are to be done immediately, and one where it is delayed. This is
66# basically class vs. function scope handling.
67
68
69class VariableClosureLookupVisitorPhase1(VisitorNoopMixin):
70    """Variable closure phase 1: Find assignments and early closure references.
71
72    In class context, a reference to a variable must be obeyed immediately,
73    so that "variable = variable" takes first "variable" as a closure and
74    then adds a new local "variable" to override it from there on. For the
75    not early closure case of a function, this will not be done and only
76    assignments shall add local variables, and references will be ignored
77    until phase 2.
78    """
79
80    @staticmethod
81    def _handleNonLocal(node):
82        # Take closure variables for non-local declarations.
83
84        for (
85            non_local_names,
86            user_provided,
87            source_ref,
88        ) in node.consumeNonlocalDeclarations():
89            for non_local_name in non_local_names:
90
91                variable = node.takeVariableForClosure(variable_name=non_local_name)
92
93                node.getLocalsScope().registerClosureVariable(variable)
94
95                if variable.isModuleVariable() and user_provided:
96                    raiseSyntaxError(
97                        "no binding for nonlocal '%s' found" % (non_local_name),
98                        source_ref,
99                    )
100
101                variable.addVariableUser(node)
102
103    @staticmethod
104    def _handleQualnameSetup(node):
105        if node.qualname_setup is not None:
106            provider = node.getParentVariableProvider()
107
108            if node.isExpressionClassBody():
109                class_variable_name, qualname_assign = node.qualname_setup
110
111                if provider.hasProvidedVariable(class_variable_name):
112                    class_variable = provider.getVariableForReference(
113                        class_variable_name
114                    )
115
116                    if class_variable.isModuleVariable():
117                        qualname_node = qualname_assign.subnode_source
118
119                        new_node = makeConstantReplacementNode(
120                            constant=class_variable.getName(),
121                            node=qualname_node,
122                            user_provided=True,
123                        )
124
125                        parent = qualname_node.parent
126                        qualname_node.finalize()
127                        parent.replaceChild(qualname_node, new_node)
128
129                        node.qualname_provider = node.getParentModule()
130            else:
131                if provider.hasProvidedVariable(node.qualname_setup):
132                    function_variable = provider.getVariableForReference(
133                        node.qualname_setup
134                    )
135
136                    if function_variable.isModuleVariable():
137                        node.qualname_provider = node.getParentModule()
138
139            # TODO: Actually for nested global classes, this approach
140            # may not work, as their "qualname" will be wrong. In that
141            # case a dedicated node for "qualname" references might be
142            # needed.
143
144            node.qualname_setup = None
145
146    @staticmethod
147    def _shouldUseLocalsDict(provider, variable_name):
148        return provider.isExpressionClassBody() and (
149            not provider.hasProvidedVariable(variable_name)
150            or provider.getProvidedVariable(variable_name).getOwner() is provider
151        )
152
153    def onLeaveNode(self, node):
154        if node.isStatementAssignmentVariableName():
155            variable_name = node.getVariableName()
156            provider = node.provider
157
158            # Classes always assign to locals dictionary except for closure
159            # variables taken.
160            if self._shouldUseLocalsDict(provider, variable_name):
161                if node.subnode_source.isExpressionOperationInplace():
162                    temp_scope = provider.allocateTempScope("class_inplace")
163
164                    tmp_variable = provider.allocateTempVariable(
165                        temp_scope=temp_scope, name="value"
166                    )
167
168                    statements = mergeStatements(
169                        statements=(
170                            StatementAssignmentVariable(
171                                variable=tmp_variable,
172                                source=node.subnode_source.subnode_left,
173                                source_ref=node.source_ref,
174                            ),
175                            makeTryFinallyStatement(
176                                provider=provider,
177                                tried=(
178                                    StatementAssignmentVariable(
179                                        variable=tmp_variable,
180                                        source=makeExpressionOperationBinaryInplace(
181                                            left=ExpressionTempVariableRef(
182                                                variable=tmp_variable,
183                                                source_ref=node.source_ref,
184                                            ),
185                                            right=node.subnode_source.subnode_right,
186                                            operator=node.subnode_source.getOperator(),
187                                            source_ref=node.source_ref,
188                                        ),
189                                        source_ref=node.source_ref,
190                                    ),
191                                    StatementLocalsDictOperationSet(
192                                        locals_scope=provider.getLocalsScope(),
193                                        variable_name=variable_name,
194                                        value=ExpressionTempVariableRef(
195                                            variable=tmp_variable,
196                                            source_ref=node.source_ref,
197                                        ),
198                                        source_ref=node.source_ref,
199                                    ),
200                                ),
201                                final=StatementReleaseVariable(
202                                    variable=tmp_variable, source_ref=node.source_ref
203                                ),
204                                source_ref=node.source_ref,
205                            ),
206                        )
207                    )
208
209                    node.parent.replaceStatement(node, statements)
210
211                else:
212                    new_node = StatementLocalsDictOperationSet(
213                        locals_scope=provider.getLocalsScope(),
214                        variable_name=variable_name,
215                        value=node.subnode_source,
216                        source_ref=node.source_ref,
217                    )
218
219                    node.parent.replaceChild(node, new_node)
220            else:
221                variable = provider.getVariableForAssignment(
222                    variable_name=variable_name
223                )
224
225                new_node = StatementAssignmentVariable(
226                    variable=variable,
227                    source=node.subnode_source,
228                    source_ref=node.source_ref,
229                )
230
231                variable.addVariableUser(provider)
232
233                node.parent.replaceChild(node, new_node)
234
235            del node.parent
236            del node.provider
237        elif node.isStatementDelVariableName():
238            variable_name = node.getVariableName()
239
240            provider = node.provider
241
242            if self._shouldUseLocalsDict(provider, variable_name):
243                # Classes always assign to locals dictionary except for closure
244                # variables taken.
245                new_node = StatementLocalsDictOperationDel(
246                    locals_scope=provider.getLocalsScope(),
247                    variable_name=variable_name,
248                    tolerant=node.tolerant,
249                    source_ref=node.source_ref,
250                )
251            else:
252                variable = provider.getVariableForAssignment(
253                    variable_name=variable_name
254                )
255
256                new_node = StatementDelVariable(
257                    variable=variable,
258                    tolerant=node.tolerant,
259                    source_ref=node.source_ref,
260                )
261
262                variable.addVariableUser(provider)
263
264            parent = node.parent
265            node.finalize()
266
267            parent.replaceChild(node, new_node)
268
269    def onEnterNode(self, node):
270        # Mighty complex code with lots of branches, but we aim to get rid of it.
271        # pylint: disable=too-many-branches
272
273        if node.isExpressionVariableNameRef():
274            provider = node.provider
275
276            if provider.isExpressionClassBody():
277                if node.needsFallback():
278                    variable = provider.getVariableForReference(
279                        variable_name=node.getVariableName()
280                    )
281
282                    new_node = ExpressionLocalsVariableRefOrFallback(
283                        locals_scope=provider.getLocalsScope(),
284                        variable_name=node.getVariableName(),
285                        fallback=makeExpressionVariableRef(
286                            variable=variable,
287                            locals_scope=provider.getLocalsScope(),
288                            source_ref=node.source_ref,
289                        ),
290                        source_ref=node.source_ref,
291                    )
292
293                    variable.addVariableUser(provider)
294                else:
295                    new_node = ExpressionLocalsVariableRef(
296                        locals_scope=provider.getLocalsScope(),
297                        variable_name=node.getVariableName(),
298                        source_ref=node.source_ref,
299                    )
300
301                parent = node.parent
302                node.finalize()
303
304                parent.replaceChild(node, new_node)
305        elif node.isExpressionTempVariableRef():
306            if node.getVariable().getOwner() != node.getParentVariableProvider():
307                node.getParentVariableProvider().addClosureVariable(node.getVariable())
308        elif node.isExpressionGeneratorObjectBody():
309            if python_version >= 0x300:
310                self._handleNonLocal(node)
311
312            # Only Python3.4 or later allows for generators to have qualname.
313            if python_version >= 0x340:
314                self._handleQualnameSetup(node)
315        elif node.isExpressionCoroutineObjectBody():
316            self._handleNonLocal(node)
317
318            self._handleQualnameSetup(node)
319        elif node.isExpressionAsyncgenObjectBody():
320            self._handleNonLocal(node)
321
322            self._handleQualnameSetup(node)
323        elif node.isExpressionClassBody():
324            if python_version >= 0x300:
325                self._handleNonLocal(node)
326
327            # Python3.4 allows for class declarations to be made global, even
328            # after they were declared, so we need to fix this up.
329            if python_version >= 0x340:
330                self._handleQualnameSetup(node)
331        elif node.isExpressionFunctionBody():
332            if python_version >= 0x300:
333                self._handleNonLocal(node)
334
335            # Python 3.4 allows for class declarations to be made global, even
336            # after they were declared, so we need to fix this up.
337            if python_version >= 0x340:
338                self._handleQualnameSetup(node)
339        # Check if continue and break are properly in loops. If not, raise a
340        # syntax error.
341        elif node.isStatementLoopBreak() or node.isStatementLoopContinue():
342            current = node
343
344            while True:
345                current = current.getParent()
346
347                if current.isStatementLoop():
348                    break
349
350                if current.isParentVariableProvider():
351                    if node.isStatementLoopContinue():
352                        message = "'continue' not properly in loop"
353                    else:
354                        message = "'break' outside loop"
355
356                    raiseSyntaxError(message, node.getSourceReference())
357
358
359class VariableClosureLookupVisitorPhase2(VisitorNoopMixin):
360    """Variable closure phase 2: Find assignments and references.
361
362    In class context, a reference to a variable must be obeyed immediately,
363    so that "variable = variable" takes first "variable" as a closure and
364    then adds a new local "variable" to override it from there on.
365
366    So, assignments for early closure, accesses will already have a
367    variable set now, the others, only in this phase.
368    """
369
370    @staticmethod
371    def _attachVariable(node, provider):
372        # print "Late reference", node.getVariableName(), "for", provider, "caused at", node, "of", node.getParent()
373
374        variable_name = node.getVariableName()
375
376        variable = provider.getVariableForReference(variable_name=variable_name)
377
378        # Need to catch functions with "exec" and closure variables not allowed.
379        if python_version < 0x300 and provider.isExpressionFunctionBodyBase():
380            was_taken = provider.hasTakenVariable(variable_name)
381
382            if not was_taken and variable.getOwner() is not provider:
383                parent_provider = provider.getParentVariableProvider()
384
385                while parent_provider.isExpressionClassBody():
386                    parent_provider = parent_provider.getParentVariableProvider()
387
388                if (
389                    parent_provider.isExpressionFunctionBody()
390                    and parent_provider.isUnqualifiedExec()
391                ):
392                    raiseSyntaxError(
393                        getErrorMessageExecWithNestedFunction()
394                        % parent_provider.getName(),
395                        node.getSourceReference(),
396                        display_line=False,  # Wrong line anyway
397                    )
398
399        return variable
400
401    def onEnterNode(self, node):
402        if node.isExpressionVariableNameRef():
403            provider = node.provider
404
405            try:
406                variable = self._attachVariable(node, provider)
407            except MaybeLocalVariableUsage:
408                variable_name = node.getVariableName()
409
410                new_node = ExpressionLocalsVariableRefOrFallback(
411                    locals_scope=provider.getLocalsScope(),
412                    variable_name=variable_name,
413                    fallback=makeExpressionVariableRef(
414                        variable=node.getParentModule().getVariableForReference(
415                            variable_name
416                        ),
417                        locals_scope=provider.getLocalsScope(),
418                        source_ref=node.source_ref,
419                    ),
420                    source_ref=node.source_ref,
421                )
422            else:
423                new_node = makeExpressionVariableRef(
424                    variable=variable,
425                    locals_scope=provider.getLocalsScope(),
426                    source_ref=node.source_ref,
427                )
428
429                variable.addVariableUser(provider)
430
431            parent = node.parent
432            node.finalize()
433
434            parent.replaceChild(node, new_node)
435
436
437class VariableClosureLookupVisitorPhase3(VisitorNoopMixin):
438    """Variable closure phase 3: Find errors and complete frame variables.
439
440    In this phase, we can do some fix-ups and find errors. We might e.g.
441    detect that a "del" was executed on a shared variable, which is not
442    allowed for Python 2.x, so it must be caught. The parsing wouldn't do
443    that.
444
445    Also, frame objects for functions should learn their variable names.
446    """
447
448    def onEnterNode(self, node):
449        if python_version < 0x300 and node.isStatementDelVariable():
450            variable = node.getVariable()
451
452            if not variable.isModuleVariable() and isSharedAmongScopes(variable):
453                raiseSyntaxError(
454                    """\
455can not delete variable '%s' referenced in nested scope"""
456                    % (variable.getName()),
457                    node.getSourceReference(),
458                )
459        elif node.isStatementsFrame():
460            node.updateLocalNames()
461        elif node.isExpressionFunctionBodyBase():
462            addFunctionVariableReleases(node)
463
464            # Python3 is influenced by the mere use of a variable named as
465            # "super". So we need to prepare ability to take closure.
466            if node.hasFlag("has_super"):
467                if not node.hasVariableName("__class__"):
468                    class_var = node.takeVariableForClosure("__class__")
469                    class_var.addVariableUser(node)
470
471                    node.getLocalsScope().registerClosureVariable(class_var)
472                    while node != class_var.getOwner():
473                        node = node.getParentVariableProvider()
474                        node.getLocalsScope().registerClosureVariable(class_var)
475
476
477def completeVariableClosures(tree):
478    visitors = (
479        VariableClosureLookupVisitorPhase1(),
480        VariableClosureLookupVisitorPhase2(),
481        VariableClosureLookupVisitorPhase3(),
482    )
483
484    for visitor in visitors:
485        visitTree(tree, visitor)
486
487    # Only used to detect syntax errors.
488    releaseSharedScopeInformation(tree)
489