1#     Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com
2#
3#     Part of "Nuitka", an optimizing Python compiler that is compatible and
4#     integrates with CPython, but also works on its own.
5#
6#     Licensed under the Apache License, Version 2.0 (the "License");
7#     you may not use this file except in compliance with the License.
8#     You may obtain a copy of the License at
9#
10#        http://www.apache.org/licenses/LICENSE-2.0
11#
12#     Unless required by applicable law or agreed to in writing, software
13#     distributed under the License is distributed on an "AS IS" BASIS,
14#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15#     See the License for the specific language governing permissions and
16#     limitations under the License.
17#
18""" Node for variable references.
19
20These represent all variable references in the node tree. Can be in assignments
21and its expressions, changing the meaning of course dramatically.
22
23"""
24
25from nuitka import Builtins, Variables
26from nuitka.ModuleRegistry import getOwnerFromCodeName
27from nuitka.PythonVersions import python_version
28
29from .DictionaryNodes import (
30    ExpressionDictOperationGet,
31    ExpressionDictOperationIn,
32    ExpressionDictOperationNotIn,
33    StatementDictOperationRemove,
34    StatementDictOperationSet,
35)
36from .ExpressionBases import ExpressionBase
37from .ModuleAttributeNodes import (
38    ExpressionModuleAttributeLoaderRef,
39    ExpressionModuleAttributeNameRef,
40    ExpressionModuleAttributePackageRef,
41    ExpressionModuleAttributeSpecRef,
42)
43from .NodeMakingHelpers import (
44    makeRaiseExceptionReplacementExpression,
45    makeRaiseTypeErrorExceptionReplacementFromTemplateAndValue,
46)
47from .shapes.StandardShapes import tshape_unknown
48
49
50class ExpressionVariableNameRef(ExpressionBase):
51    """These are used before the actual variable object is known from VariableClosure."""
52
53    kind = "EXPRESSION_VARIABLE_NAME_REF"
54
55    __slots__ = "variable_name", "provider"
56
57    def __init__(self, provider, variable_name, source_ref):
58        assert not provider.isExpressionOutlineBody(), source_ref
59
60        ExpressionBase.__init__(self, source_ref=source_ref)
61
62        self.variable_name = variable_name
63
64        self.provider = provider
65
66    def finalize(self):
67        del self.parent
68        del self.provider
69
70    @staticmethod
71    def isExpressionVariableNameRef():
72        return True
73
74    def getDetails(self):
75        return {"variable_name": self.variable_name, "provider": self.provider}
76
77    def getVariableName(self):
78        return self.variable_name
79
80    def computeExpressionRaw(self, trace_collection):
81        return self, None, None
82
83    @staticmethod
84    def needsFallback():
85        return True
86
87
88class ExpressionVariableLocalNameRef(ExpressionVariableNameRef):
89    """These are used before the actual variable object is known from VariableClosure.
90
91    The special thing about this as opposed to ExpressionVariableNameRef is that
92    these must remain local names and cannot fallback to outside scopes. This is
93    used for "__annotations__".
94
95    """
96
97    kind = "EXPRESSION_VARIABLE_LOCAL_NAME_REF"
98
99    @staticmethod
100    def needsFallback():
101        return False
102
103
104class ExpressionVariableRefBase(ExpressionBase):
105    # Base classes can be abstract, pylint: disable=abstract-method
106
107    __slots__ = "variable", "variable_trace"
108
109    def __init__(self, variable, source_ref):
110        ExpressionBase.__init__(self, source_ref=source_ref)
111
112        self.variable = variable
113        self.variable_trace = None
114
115    def finalize(self):
116        del self.parent
117        del self.variable
118        del self.variable_trace
119
120    def getVariableName(self):
121        return self.variable.getName()
122
123    def getVariable(self):
124        return self.variable
125
126    def getVariableTrace(self):
127        return self.variable_trace
128
129    def getTypeShape(self):
130        if self.variable_trace is None:
131            return tshape_unknown
132        else:
133            return self.variable_trace.getTypeShape()
134
135    def onContentEscapes(self, trace_collection):
136        trace_collection.onVariableContentEscapes(self.variable)
137
138    def computeExpressionLen(self, len_node, trace_collection):
139        if self.variable_trace is not None and self.variable_trace.isAssignTrace():
140            value = self.variable_trace.getAssignNode().subnode_source
141
142            shape = value.getValueShape()
143
144            has_len = shape.hasShapeSlotLen()
145
146            if has_len is False:
147                return makeRaiseTypeErrorExceptionReplacementFromTemplateAndValue(
148                    template="object of type '%s' has no len()",
149                    operation="len",
150                    original_node=len_node,
151                    value_node=self,
152                )
153            elif has_len is True:
154                iter_length = value.getIterationLength()
155
156                if iter_length is not None:
157                    from .ConstantRefNodes import makeConstantRefNode
158
159                    result = makeConstantRefNode(
160                        constant=int(iter_length),  # make sure to downcast long
161                        source_ref=len_node.getSourceReference(),
162                    )
163
164                    return (
165                        result,
166                        "new_constant",
167                        "Predicted 'len' result of variable.",
168                    )
169
170        # The variable itself is to be considered escaped.
171        trace_collection.markActiveVariableAsEscaped(self.variable)
172
173        # Any code could be run, note that.
174        trace_collection.onControlFlowEscape(self)
175
176        # Any exception may be raised.
177        trace_collection.onExceptionRaiseExit(BaseException)
178
179        return len_node, None, None
180
181    def computeExpressionAttribute(self, lookup_node, attribute_name, trace_collection):
182        # Any code could be run, note that.
183        trace_collection.onControlFlowEscape(self)
184
185        # The variable itself is to be considered escaped.
186        trace_collection.markActiveVariableAsEscaped(self.variable)
187
188        if not self.isKnownToHaveAttribute(attribute_name):
189            trace_collection.onExceptionRaiseExit(BaseException)
190
191        return lookup_node, None, None
192
193    def computeExpressionComparisonIn(self, in_node, value_node, trace_collection):
194        tags = None
195        message = None
196
197        # Any code could be run, note that.
198        trace_collection.onControlFlowEscape(in_node)
199
200        if self.variable_trace.hasShapeDictionaryExact():
201            tags = "new_expression"
202            message = """\
203Check '%s' on dictionary lowered to dictionary '%s'.""" % (
204                in_node.comparator,
205                in_node.comparator,
206            )
207
208            if in_node.comparator == "In":
209                in_node = ExpressionDictOperationIn(
210                    key=value_node,
211                    dict_arg=self,
212                    source_ref=in_node.getSourceReference(),
213                )
214            else:
215                in_node = ExpressionDictOperationNotIn(
216                    key=value_node,
217                    dict_arg=self,
218                    source_ref=in_node.getSourceReference(),
219                )
220
221        # Any exception may be raised.
222        if in_node.mayRaiseException(BaseException):
223            trace_collection.onExceptionRaiseExit(BaseException)
224
225        return in_node, tags, message
226
227    def computeExpressionSetSubscript(
228        self, set_node, subscript, value_node, trace_collection
229    ):
230        tags = None
231        message = None
232
233        # By default, an subscript may change everything about the lookup
234        # source.
235        if self.variable_trace.hasShapeDictionaryExact():
236            result = StatementDictOperationSet(
237                dict_arg=self,
238                key=subscript,
239                value=value_node,
240                source_ref=set_node.getSourceReference(),
241            )
242            change_tags = "new_statements"
243            change_desc = """\
244Subscript assignment to dictionary lowered to dictionary assignment."""
245
246            trace_collection.removeKnowledge(self)
247
248            result2, change_tags2, change_desc2 = result.computeStatementOperation(
249                trace_collection
250            )
251
252            if result2 is not result:
253                trace_collection.signalChange(
254                    tags=change_tags,
255                    source_ref=self.source_ref,
256                    message=change_desc,
257                )
258
259                return result2, change_tags2, change_desc2
260            else:
261                return result, change_tags, change_desc
262
263        trace_collection.removeKnowledge(self)
264
265        # Any code could be run, note that.
266        trace_collection.onControlFlowEscape(self)
267
268        # Any exception might be raised.
269        if set_node.mayRaiseException(BaseException):
270            trace_collection.onExceptionRaiseExit(BaseException)
271
272        return set_node, tags, message
273
274    def computeExpressionDelSubscript(self, del_node, subscript, trace_collection):
275        tags = None
276        message = None
277
278        if self.variable_trace.hasShapeDictionaryExact():
279            result = StatementDictOperationRemove(
280                dict_arg=self,
281                key=subscript,
282                source_ref=del_node.getSourceReference(),
283            )
284            change_tags = "new_statements"
285            change_desc = """\
286Subscript del to dictionary lowered to dictionary del."""
287
288            trace_collection.removeKnowledge(self)
289
290            result2, change_tags2, change_desc2 = result.computeStatementOperation(
291                trace_collection
292            )
293
294            if result2 is not result:
295                trace_collection.signalChange(
296                    tags=change_tags,
297                    source_ref=self.source_ref,
298                    message=change_desc,
299                )
300
301                return result2, change_tags2, change_desc2
302            else:
303                return result, change_tags, change_desc
304
305        # By default, an subscript may change everything about the lookup
306        # source.
307        # Any code could be run, note that.
308        trace_collection.onControlFlowEscape(self)
309
310        # Any exception might be raised.
311        if del_node.mayRaiseException(BaseException):
312            trace_collection.onExceptionRaiseExit(BaseException)
313
314        return del_node, tags, message
315
316    def computeExpressionSubscript(self, lookup_node, subscript, trace_collection):
317        tags = None
318        message = None
319
320        if self.variable_trace.hasShapeDictionaryExact():
321            return trace_collection.computedExpressionResult(
322                expression=ExpressionDictOperationGet(
323                    dict_arg=self,
324                    key=subscript,
325                    source_ref=lookup_node.getSourceReference(),
326                ),
327                change_tags="new_expression",
328                change_desc="""\
329Subscript look-up to dictionary lowered to dictionary look-up.""",
330            )
331
332        # Any code could be run, note that.
333        trace_collection.onControlFlowEscape(self)
334
335        # Any exception might be raised.
336        if lookup_node.mayRaiseException(BaseException):
337            trace_collection.onExceptionRaiseExit(BaseException)
338
339        return lookup_node, tags, message
340
341    def _applyReplacement(self, trace_collection, replacement):
342        trace_collection.signalChange(
343            "new_expression",
344            self.source_ref,
345            "Value propagated for '%s' from '%s'."
346            % (self.variable.getName(), replacement.getSourceReference().getAsString()),
347        )
348
349        # Special case for in-place assignments.
350        if self.parent.isExpressionOperationInplace():
351            statement = self.parent.parent
352
353            if statement.isStatementAssignmentVariable():
354                statement.unmarkAsInplaceSuspect()
355
356        # Need to compute the replacement still.
357        return replacement.computeExpressionRaw(trace_collection)
358
359
360_hard_names = ("dir", "eval", "exec", "execfile", "locals", "vars", "super")
361
362
363class ExpressionVariableRef(ExpressionVariableRefBase):
364    kind = "EXPRESSION_VARIABLE_REF"
365
366    __slots__ = ()
367
368    def __init__(self, variable, source_ref):
369        assert variable is not None
370
371        ExpressionVariableRefBase.__init__(
372            self, variable=variable, source_ref=source_ref
373        )
374
375    @staticmethod
376    def isExpressionVariableRef():
377        return True
378
379    def getDetails(self):
380        return {"variable": self.variable}
381
382    def getDetailsForDisplay(self):
383        return {
384            "variable_name": self.variable.getName(),
385            "owner": self.variable.getOwner().getCodeName(),
386        }
387
388    @classmethod
389    def fromXML(cls, provider, source_ref, **args):
390        assert cls is ExpressionVariableRef, cls
391
392        owner = getOwnerFromCodeName(args["owner"])
393        variable = owner.getProvidedVariable(args["variable_name"])
394
395        return cls(variable=variable, source_ref=source_ref)
396
397    @staticmethod
398    def isTargetVariableRef():
399        return False
400
401    def getVariable(self):
402        return self.variable
403
404    def setVariable(self, variable):
405        assert isinstance(variable, Variables.Variable), repr(variable)
406
407        self.variable = variable
408
409    def computeExpressionRaw(self, trace_collection):
410        # Terribly detailed, pylint: disable=too-many-branches,too-many-statements
411
412        variable = self.variable
413        assert variable is not None
414
415        self.variable_trace = trace_collection.getVariableCurrentTrace(
416            variable=variable
417        )
418
419        replacement = self.variable_trace.getReplacementNode(self)
420        if replacement is not None:
421            return self._applyReplacement(trace_collection, replacement)
422
423        if not self.variable_trace.mustHaveValue():
424            # TODO: This could be way more specific surely, either NameError or UnboundLocalError
425            # could be decided from context.
426            trace_collection.onExceptionRaiseExit(BaseException)
427
428        if variable.isModuleVariable() and variable.hasDefiniteWrites() is False:
429            variable_name = self.variable.getName()
430
431            if variable_name in Builtins.builtin_exception_names:
432                if not self.variable.getOwner().getLocalsScope().isEscaped():
433                    from .BuiltinRefNodes import ExpressionBuiltinExceptionRef
434
435                    new_node = ExpressionBuiltinExceptionRef(
436                        exception_name=self.variable.getName(),
437                        source_ref=self.source_ref,
438                    )
439
440                    change_tags = "new_builtin_ref"
441                    change_desc = """\
442Module variable '%s' found to be built-in exception reference.""" % (
443                        variable_name
444                    )
445                else:
446                    self.variable_trace.addUsage()
447
448                    new_node = self
449                    change_tags = None
450                    change_desc = None
451
452            elif variable_name in Builtins.builtin_names:
453                if (
454                    variable_name in _hard_names
455                    or not self.variable.getOwner().getLocalsScope().isEscaped()
456                ):
457                    from .BuiltinRefNodes import makeExpressionBuiltinRef
458
459                    new_node = makeExpressionBuiltinRef(
460                        builtin_name=variable_name,
461                        locals_scope=self.getFunctionsLocalsScope(),
462                        source_ref=self.source_ref,
463                    )
464
465                    change_tags = "new_builtin_ref"
466                    change_desc = """\
467Module variable '%s' found to be built-in reference.""" % (
468                        variable_name
469                    )
470                else:
471                    self.variable_trace.addUsage()
472
473                    new_node = self
474                    change_tags = None
475                    change_desc = None
476            elif variable_name == "__name__":
477                new_node = ExpressionModuleAttributeNameRef(
478                    variable=variable, source_ref=self.source_ref
479                )
480
481                change_tags = "new_expression"
482                change_desc = """\
483Replaced read-only module attribute '__name__' with module attribute reference."""
484            elif variable_name == "__package__":
485                new_node = ExpressionModuleAttributePackageRef(
486                    variable=variable, source_ref=self.source_ref
487                )
488
489                change_tags = "new_expression"
490                change_desc = """\
491Replaced read-only module attribute '__package__' with module attribute reference."""
492            elif variable_name == "__loader__" and python_version >= 0x300:
493                new_node = ExpressionModuleAttributeLoaderRef(
494                    variable=variable, source_ref=self.source_ref
495                )
496
497                change_tags = "new_expression"
498                change_desc = """\
499Replaced read-only module attribute '__loader__' with module attribute reference."""
500            elif variable_name == "__spec__" and python_version >= 0x340:
501                new_node = ExpressionModuleAttributeSpecRef(
502                    variable=variable, source_ref=self.source_ref
503                )
504
505                change_tags = "new_expression"
506                change_desc = """\
507Replaced read-only module attribute '__spec__' with module attribute reference."""
508            else:
509                self.variable_trace.addUsage()
510
511                # Probably should give a warning once about it.
512                new_node = self
513                change_tags = None
514                change_desc = None
515
516            return new_node, change_tags, change_desc
517
518        self.variable_trace.addUsage()
519
520        if self.variable_trace.mustNotHaveValue():
521            assert self.variable.isLocalVariable(), self.variable
522
523            variable_name = self.variable.getName()
524
525            result = makeRaiseExceptionReplacementExpression(
526                expression=self,
527                exception_type="UnboundLocalError",
528                exception_value="""local variable '%s' referenced before assignment"""
529                % variable_name,
530            )
531
532            return (
533                result,
534                "new_raise",
535                "Variable access of not initialized variable '%s'" % variable_name,
536            )
537
538        return self, None, None
539
540    def computeExpressionCall(self, call_node, call_args, call_kw, trace_collection):
541        # The called and the arguments escape for good.
542        self.onContentEscapes(trace_collection)
543        if call_args is not None:
544            call_args.onContentEscapes(trace_collection)
545        if call_kw is not None:
546            call_kw.onContentEscapes(trace_collection)
547
548        # Any code could be run, note that.
549        trace_collection.onControlFlowEscape(self)
550
551        # Any exception may be raised.
552        trace_collection.onExceptionRaiseExit(BaseException)
553
554        if (
555            self.variable.getName() in _hard_names
556            and self.variable.isIncompleteModuleVariable()
557        ):
558            # Just inform the collection that all escaped.
559            trace_collection.onLocalsUsage(locals_scope=self.getFunctionsLocalsScope())
560
561        return call_node, None, None
562
563    def hasShapeDictionaryExact(self):
564        return self.variable_trace.hasShapeDictionaryExact()
565
566    def getTruthValue(self):
567        return self.variable_trace.getTruthValue()
568
569    @staticmethod
570    def isKnownToBeIterable(count):
571        return None
572
573    def mayHaveSideEffects(self):
574        return not self.variable_trace.mustHaveValue()
575
576    def mayRaiseException(self, exception_type):
577        return self.variable_trace is None or not self.variable_trace.mustHaveValue()
578
579    def mayRaiseExceptionBool(self, exception_type):
580        return (
581            self.variable_trace is None
582            or not self.variable_trace.mustHaveValue()
583            or not self.variable_trace.getTypeShape().hasShapeSlotBool()
584        )
585
586    def getFunctionsLocalsScope(self):
587        return self.getParentVariableProvider().getLocalsScope()
588
589
590class ExpressionVariableOrBuiltinRef(ExpressionVariableRef):
591    kind = "EXPRESSION_VARIABLE_OR_BUILTIN_REF"
592
593    __slots__ = ("locals_scope",)
594
595    def __init__(self, variable, locals_scope, source_ref):
596        ExpressionVariableRef.__init__(self, variable=variable, source_ref=source_ref)
597
598        self.locals_scope = locals_scope
599
600    def getDetails(self):
601        return {"variable": self.variable, "locals_scope": self.locals_scope}
602
603    def getFunctionsLocalsScope(self):
604        return self.locals_scope
605
606
607def makeExpressionVariableRef(variable, locals_scope, source_ref):
608    if variable.getName() in _hard_names:
609        return ExpressionVariableOrBuiltinRef(
610            variable=variable, locals_scope=locals_scope, source_ref=source_ref
611        )
612    else:
613        return ExpressionVariableRef(variable=variable, source_ref=source_ref)
614
615
616class ExpressionTempVariableRef(ExpressionVariableRefBase):
617    kind = "EXPRESSION_TEMP_VARIABLE_REF"
618
619    def __init__(self, variable, source_ref):
620        assert variable.isTempVariable()
621
622        ExpressionVariableRefBase.__init__(
623            self, variable=variable, source_ref=source_ref
624        )
625
626    def getDetailsForDisplay(self):
627        return {
628            "temp_name": self.variable.getName(),
629            "owner": self.variable.getOwner().getCodeName(),
630        }
631
632    def getDetails(self):
633        return {"variable": self.variable}
634
635    @classmethod
636    def fromXML(cls, provider, source_ref, **args):
637        assert cls is ExpressionTempVariableRef, cls
638
639        owner = getOwnerFromCodeName(args["owner"])
640
641        variable = owner.getTempVariable(None, args["temp_name"])
642
643        return cls(variable=variable, source_ref=source_ref)
644
645    @staticmethod
646    def isTargetVariableRef():
647        return False
648
649    def computeExpressionRaw(self, trace_collection):
650        self.variable_trace = trace_collection.getVariableCurrentTrace(
651            variable=self.variable
652        )
653
654        replacement = self.variable_trace.getReplacementNode(self)
655        if replacement is not None:
656            return self._applyReplacement(trace_collection, replacement)
657
658        self.variable_trace.addUsage()
659
660        # Nothing to do here.
661        return self, None, None
662
663    def computeExpressionNext1(self, next_node, trace_collection):
664        may_not_raise = False
665
666        if self.variable_trace.isAssignTrace():
667            value = self.variable_trace.getAssignNode().subnode_source
668
669            # TODO: Add iteration handles to trace collections instead.
670            current_index = trace_collection.getIteratorNextCount(value)
671            trace_collection.onIteratorNext(value)
672
673            if value.hasShapeSlotNext():
674                if (
675                    current_index is not None
676                    # TODO: Change to iteration handles.
677                    and value.isKnownToBeIterableAtMin(current_index + 1)
678                ):
679                    may_not_raise = True
680
681                    # TODO: Make use of this
682                    # candidate = value.getIterationValue(current_index)
683
684                    # if False:
685                    # and value.canPredictIterationValues()
686                    #    return (
687                    #        candidate,
688                    #        "new_expression",
689                    #        "Predicted 'next' value from iteration.",
690                    #    )
691            else:
692                # TODO: Could ask it about exception predictability for that case
693                # or warn about it at least.
694                pass
695                # assert False, value
696
697        self.onContentEscapes(trace_collection)
698
699        # Any code could be run, note that.
700        trace_collection.onControlFlowEscape(self)
701
702        # Any exception may be raised.
703        trace_collection.onExceptionRaiseExit(BaseException)
704
705        return may_not_raise, (next_node, None, None)
706
707    @staticmethod
708    def mayHaveSideEffects():
709        # Can't happen with temporary variables, unless we used them wrongly.
710        return False
711
712    @staticmethod
713    def mayRaiseException(exception_type):
714        # Can't happen with temporary variables, unless we used them wrongly.
715        return False
716
717    def mayRaiseExceptionImportName(self, exception_type, import_name):
718        if self.variable_trace is not None and self.variable_trace.isAssignTrace():
719            return self.variable_trace.getAssignNode().subnode_source.mayRaiseExceptionImportName(
720                exception_type, import_name
721            )
722
723        else:
724            return True
725
726    @staticmethod
727    def isKnownToBeIterableAtMin(count):
728        # TODO: See through the variable current trace.
729        return None
730