1import os
2import sys
3import warnings
4from inspect import isabstract
5from test import support
6from test.support import os_helper
7from test.libregrtest.utils import clear_caches
8
9try:
10    from _abc import _get_dump
11except ImportError:
12    import weakref
13
14    def _get_dump(cls):
15        # Reimplement _get_dump() for pure-Python implementation of
16        # the abc module (Lib/_py_abc.py)
17        registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
18        return (registry_weakrefs, cls._abc_cache,
19                cls._abc_negative_cache, cls._abc_negative_cache_version)
20
21
22def dash_R(ns, test_name, test_func):
23    """Run a test multiple times, looking for reference leaks.
24
25    Returns:
26        False if the test didn't leak references; True if we detected refleaks.
27    """
28    # This code is hackish and inelegant, but it seems to do the job.
29    import copyreg
30    import collections.abc
31
32    if not hasattr(sys, 'gettotalrefcount'):
33        raise Exception("Tracking reference leaks requires a debug build "
34                        "of Python")
35
36    # Avoid false positives due to various caches
37    # filling slowly with random data:
38    warm_caches()
39
40    # Save current values for dash_R_cleanup() to restore.
41    fs = warnings.filters[:]
42    ps = copyreg.dispatch_table.copy()
43    pic = sys.path_importer_cache.copy()
44    try:
45        import zipimport
46    except ImportError:
47        zdc = None # Run unmodified on platforms without zipimport support
48    else:
49        zdc = zipimport._zip_directory_cache.copy()
50    abcs = {}
51    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
52        if not isabstract(abc):
53            continue
54        for obj in abc.__subclasses__() + [abc]:
55            abcs[obj] = _get_dump(obj)[0]
56
57    # bpo-31217: Integer pool to get a single integer object for the same
58    # value. The pool is used to prevent false alarm when checking for memory
59    # block leaks. Fill the pool with values in -1000..1000 which are the most
60    # common (reference, memory block, file descriptor) differences.
61    int_pool = {value: value for value in range(-1000, 1000)}
62    def get_pooled_int(value):
63        return int_pool.setdefault(value, value)
64
65    nwarmup, ntracked, fname = ns.huntrleaks
66    fname = os.path.join(os_helper.SAVEDCWD, fname)
67    repcount = nwarmup + ntracked
68
69    # Pre-allocate to ensure that the loop doesn't allocate anything new
70    rep_range = list(range(repcount))
71    rc_deltas = [0] * repcount
72    alloc_deltas = [0] * repcount
73    fd_deltas = [0] * repcount
74    getallocatedblocks = sys.getallocatedblocks
75    gettotalrefcount = sys.gettotalrefcount
76    _getquickenedcount = sys._getquickenedcount
77    fd_count = os_helper.fd_count
78    # initialize variables to make pyflakes quiet
79    rc_before = alloc_before = fd_before = 0
80
81    if not ns.quiet:
82        print("beginning", repcount, "repetitions", file=sys.stderr)
83        print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
84              flush=True)
85
86    dash_R_cleanup(fs, ps, pic, zdc, abcs)
87    support.gc_collect()
88
89    for i in rep_range:
90        test_func()
91
92        dash_R_cleanup(fs, ps, pic, zdc, abcs)
93        support.gc_collect()
94
95        # Read memory statistics immediately after the garbage collection
96        alloc_after = getallocatedblocks() - _getquickenedcount()
97        rc_after = gettotalrefcount()
98        fd_after = fd_count()
99
100        if not ns.quiet:
101            print('.', end='', file=sys.stderr, flush=True)
102
103        rc_deltas[i] = get_pooled_int(rc_after - rc_before)
104        alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
105        fd_deltas[i] = get_pooled_int(fd_after - fd_before)
106
107        alloc_before = alloc_after
108        rc_before = rc_after
109        fd_before = fd_after
110
111    if not ns.quiet:
112        print(file=sys.stderr)
113
114    # These checkers return False on success, True on failure
115    def check_rc_deltas(deltas):
116        # Checker for reference counters and memory blocks.
117        #
118        # bpo-30776: Try to ignore false positives:
119        #
120        #   [3, 0, 0]
121        #   [0, 1, 0]
122        #   [8, -8, 1]
123        #
124        # Expected leaks:
125        #
126        #   [5, 5, 6]
127        #   [10, 1, 1]
128        return all(delta >= 1 for delta in deltas)
129
130    def check_fd_deltas(deltas):
131        return any(deltas)
132
133    failed = False
134    for deltas, item_name, checker in [
135        (rc_deltas, 'references', check_rc_deltas),
136        (alloc_deltas, 'memory blocks', check_rc_deltas),
137        (fd_deltas, 'file descriptors', check_fd_deltas)
138    ]:
139        # ignore warmup runs
140        deltas = deltas[nwarmup:]
141        if checker(deltas):
142            msg = '%s leaked %s %s, sum=%s' % (
143                test_name, deltas, item_name, sum(deltas))
144            print(msg, file=sys.stderr, flush=True)
145            with open(fname, "a") as refrep:
146                print(msg, file=refrep)
147                refrep.flush()
148            failed = True
149    return failed
150
151
152def dash_R_cleanup(fs, ps, pic, zdc, abcs):
153    import copyreg
154    import collections.abc
155
156    # Restore some original values.
157    warnings.filters[:] = fs
158    copyreg.dispatch_table.clear()
159    copyreg.dispatch_table.update(ps)
160    sys.path_importer_cache.clear()
161    sys.path_importer_cache.update(pic)
162    try:
163        import zipimport
164    except ImportError:
165        pass # Run unmodified on platforms without zipimport support
166    else:
167        zipimport._zip_directory_cache.clear()
168        zipimport._zip_directory_cache.update(zdc)
169
170    # Clear ABC registries, restoring previously saved ABC registries.
171    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
172    abs_classes = filter(isabstract, abs_classes)
173    for abc in abs_classes:
174        for obj in abc.__subclasses__() + [abc]:
175            for ref in abcs.get(obj, set()):
176                if ref() is not None:
177                    obj.register(ref())
178            obj._abc_caches_clear()
179
180    # Clear caches
181    clear_caches()
182
183    # Clear type cache at the end: previous function calls can modify types
184    sys._clear_type_cache()
185
186
187def warm_caches():
188    # char cache
189    s = bytes(range(256))
190    for i in range(256):
191        s[i:i+1]
192    # unicode cache
193    [chr(i) for i in range(256)]
194    # int cache
195    list(range(-5, 257))
196