1from test import support
2import array
3import io
4import marshal
5import sys
6import unittest
7import os
8import types
9
10try:
11    import _testcapi
12except ImportError:
13    _testcapi = None
14
15class HelperMixin:
16    def helper(self, sample, *extra):
17        new = marshal.loads(marshal.dumps(sample, *extra))
18        self.assertEqual(sample, new)
19        try:
20            with open(support.TESTFN, "wb") as f:
21                marshal.dump(sample, f, *extra)
22            with open(support.TESTFN, "rb") as f:
23                new = marshal.load(f)
24            self.assertEqual(sample, new)
25        finally:
26            support.unlink(support.TESTFN)
27
28class IntTestCase(unittest.TestCase, HelperMixin):
29    def test_ints(self):
30        # Test a range of Python ints larger than the machine word size.
31        n = sys.maxsize ** 2
32        while n:
33            for expected in (-n, n):
34                self.helper(expected)
35            n = n >> 1
36
37    def test_int64(self):
38        # Simulate int marshaling with TYPE_INT64.
39        maxint64 = (1 << 63) - 1
40        minint64 = -maxint64-1
41        for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
42            while base:
43                s = b'I' + int.to_bytes(base, 8, 'little', signed=True)
44                got = marshal.loads(s)
45                self.assertEqual(base, got)
46                if base == -1:  # a fixed-point for shifting right 1
47                    base = 0
48                else:
49                    base >>= 1
50
51        got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10')
52        self.assertEqual(got, 0x1032547698badcfe)
53        got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef')
54        self.assertEqual(got, -0x1032547698badcff)
55        got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f')
56        self.assertEqual(got, 0x7f6e5d4c3b2a1908)
57        got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80')
58        self.assertEqual(got, -0x7f6e5d4c3b2a1909)
59
60    def test_bool(self):
61        for b in (True, False):
62            self.helper(b)
63
64class FloatTestCase(unittest.TestCase, HelperMixin):
65    def test_floats(self):
66        # Test a few floats
67        small = 1e-25
68        n = sys.maxsize * 3.7e250
69        while n > small:
70            for expected in (-n, n):
71                self.helper(float(expected))
72            n /= 123.4567
73
74        f = 0.0
75        s = marshal.dumps(f, 2)
76        got = marshal.loads(s)
77        self.assertEqual(f, got)
78        # and with version <= 1 (floats marshalled differently then)
79        s = marshal.dumps(f, 1)
80        got = marshal.loads(s)
81        self.assertEqual(f, got)
82
83        n = sys.maxsize * 3.7e-250
84        while n < small:
85            for expected in (-n, n):
86                f = float(expected)
87                self.helper(f)
88                self.helper(f, 1)
89            n *= 123.4567
90
91class StringTestCase(unittest.TestCase, HelperMixin):
92    def test_unicode(self):
93        for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
94            self.helper(marshal.loads(marshal.dumps(s)))
95
96    def test_string(self):
97        for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
98            self.helper(s)
99
100    def test_bytes(self):
101        for s in [b"", b"Andr\xe8 Previn", b"abc", b" "*10000]:
102            self.helper(s)
103
104class ExceptionTestCase(unittest.TestCase):
105    def test_exceptions(self):
106        new = marshal.loads(marshal.dumps(StopIteration))
107        self.assertEqual(StopIteration, new)
108
109class CodeTestCase(unittest.TestCase):
110    def test_code(self):
111        co = ExceptionTestCase.test_exceptions.__code__
112        new = marshal.loads(marshal.dumps(co))
113        self.assertEqual(co, new)
114
115    def test_many_codeobjects(self):
116        # Issue2957: bad recursion count on code objects
117        count = 5000    # more than MAX_MARSHAL_STACK_DEPTH
118        codes = (ExceptionTestCase.test_exceptions.__code__,) * count
119        marshal.loads(marshal.dumps(codes))
120
121    def test_different_filenames(self):
122        co1 = compile("x", "f1", "exec")
123        co2 = compile("y", "f2", "exec")
124        co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
125        self.assertEqual(co1.co_filename, "f1")
126        self.assertEqual(co2.co_filename, "f2")
127
128    @support.cpython_only
129    def test_same_filename_used(self):
130        s = """def f(): pass\ndef g(): pass"""
131        co = compile(s, "myfile", "exec")
132        co = marshal.loads(marshal.dumps(co))
133        for obj in co.co_consts:
134            if isinstance(obj, types.CodeType):
135                self.assertIs(co.co_filename, obj.co_filename)
136
137class ContainerTestCase(unittest.TestCase, HelperMixin):
138    d = {'astring': 'foo@bar.baz.spam',
139         'afloat': 7283.43,
140         'anint': 2**20,
141         'ashortlong': 2,
142         'alist': ['.zyx.41'],
143         'atuple': ('.zyx.41',)*10,
144         'aboolean': False,
145         'aunicode': "Andr\xe8 Previn"
146         }
147
148    def test_dict(self):
149        self.helper(self.d)
150
151    def test_list(self):
152        self.helper(list(self.d.items()))
153
154    def test_tuple(self):
155        self.helper(tuple(self.d.keys()))
156
157    def test_sets(self):
158        for constructor in (set, frozenset):
159            self.helper(constructor(self.d.keys()))
160
161    @support.cpython_only
162    def test_empty_frozenset_singleton(self):
163        # marshal.loads() must reuse the empty frozenset singleton
164        obj = frozenset()
165        obj2 = marshal.loads(marshal.dumps(obj))
166        self.assertIs(obj2, obj)
167
168
169class BufferTestCase(unittest.TestCase, HelperMixin):
170
171    def test_bytearray(self):
172        b = bytearray(b"abc")
173        self.helper(b)
174        new = marshal.loads(marshal.dumps(b))
175        self.assertEqual(type(new), bytes)
176
177    def test_memoryview(self):
178        b = memoryview(b"abc")
179        self.helper(b)
180        new = marshal.loads(marshal.dumps(b))
181        self.assertEqual(type(new), bytes)
182
183    def test_array(self):
184        a = array.array('B', b"abc")
185        new = marshal.loads(marshal.dumps(a))
186        self.assertEqual(new, b"abc")
187
188
189class BugsTestCase(unittest.TestCase):
190    def test_bug_5888452(self):
191        # Simple-minded check for SF 588452: Debug build crashes
192        marshal.dumps([128] * 1000)
193
194    def test_patch_873224(self):
195        self.assertRaises(Exception, marshal.loads, b'0')
196        self.assertRaises(Exception, marshal.loads, b'f')
197        self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1])
198
199    def test_version_argument(self):
200        # Python 2.4.0 crashes for any call to marshal.dumps(x, y)
201        self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5)
202        self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5)
203
204    def test_fuzz(self):
205        # simple test that it's at least not *totally* trivial to
206        # crash from bad marshal data
207        for i in range(256):
208            c = bytes([i])
209            try:
210                marshal.loads(c)
211            except Exception:
212                pass
213
214    def test_loads_recursion(self):
215        def run_tests(N, check):
216            # (((...None...),),)
217            check(b')\x01' * N + b'N')
218            check(b'(\x01\x00\x00\x00' * N + b'N')
219            # [[[...None...]]]
220            check(b'[\x01\x00\x00\x00' * N + b'N')
221            # {None: {None: {None: ...None...}}}
222            check(b'{N' * N + b'N' + b'0' * N)
223            # frozenset([frozenset([frozenset([...None...])])])
224            check(b'>\x01\x00\x00\x00' * N + b'N')
225        # Check that the generated marshal data is valid and marshal.loads()
226        # works for moderately deep nesting
227        run_tests(100, marshal.loads)
228        # Very deeply nested structure shouldn't blow the stack
229        def check(s):
230            self.assertRaises(ValueError, marshal.loads, s)
231        run_tests(2**20, check)
232
233    def test_recursion_limit(self):
234        # Create a deeply nested structure.
235        head = last = []
236        # The max stack depth should match the value in Python/marshal.c.
237        # BUG: https://bugs.python.org/issue33720
238        # Windows always limits the maximum depth on release and debug builds
239        #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
240        if os.name == 'nt':
241            MAX_MARSHAL_STACK_DEPTH = 1000
242        else:
243            MAX_MARSHAL_STACK_DEPTH = 2000
244        for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
245            last.append([0])
246            last = last[-1]
247
248        # Verify we don't blow out the stack with dumps/load.
249        data = marshal.dumps(head)
250        new_head = marshal.loads(data)
251        # Don't use == to compare objects, it can exceed the recursion limit.
252        self.assertEqual(len(new_head), len(head))
253        self.assertEqual(len(new_head[0]), len(head[0]))
254        self.assertEqual(len(new_head[-1]), len(head[-1]))
255
256        last.append([0])
257        self.assertRaises(ValueError, marshal.dumps, head)
258
259    def test_exact_type_match(self):
260        # Former bug:
261        #   >>> class Int(int): pass
262        #   >>> type(loads(dumps(Int())))
263        #   <type 'int'>
264        for typ in (int, float, complex, tuple, list, dict, set, frozenset):
265            # Note: str subclasses are not tested because they get handled
266            # by marshal's routines for objects supporting the buffer API.
267            subtyp = type('subtyp', (typ,), {})
268            self.assertRaises(ValueError, marshal.dumps, subtyp())
269
270    # Issue #1792 introduced a change in how marshal increases the size of its
271    # internal buffer; this test ensures that the new code is exercised.
272    def test_large_marshal(self):
273        size = int(1e6)
274        testString = 'abc' * size
275        marshal.dumps(testString)
276
277    def test_invalid_longs(self):
278        # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs
279        invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
280        self.assertRaises(ValueError, marshal.loads, invalid_string)
281
282    def test_multiple_dumps_and_loads(self):
283        # Issue 12291: marshal.load() should be callable multiple times
284        # with interleaved data written by non-marshal code
285        # Adapted from a patch by Engelbert Gruber.
286        data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))
287        for interleaved in (b'', b'0123'):
288            ilen = len(interleaved)
289            positions = []
290            try:
291                with open(support.TESTFN, 'wb') as f:
292                    for d in data:
293                        marshal.dump(d, f)
294                        if ilen:
295                            f.write(interleaved)
296                        positions.append(f.tell())
297                with open(support.TESTFN, 'rb') as f:
298                    for i, d in enumerate(data):
299                        self.assertEqual(d, marshal.load(f))
300                        if ilen:
301                            f.read(ilen)
302                        self.assertEqual(positions[i], f.tell())
303            finally:
304                support.unlink(support.TESTFN)
305
306    def test_loads_reject_unicode_strings(self):
307        # Issue #14177: marshal.loads() should not accept unicode strings
308        unicode_string = 'T'
309        self.assertRaises(TypeError, marshal.loads, unicode_string)
310
311    def test_bad_reader(self):
312        class BadReader(io.BytesIO):
313            def readinto(self, buf):
314                n = super().readinto(buf)
315                if n is not None and n > 4:
316                    n += 10**6
317                return n
318        for value in (1.0, 1j, b'0123456789', '0123456789'):
319            self.assertRaises(ValueError, marshal.load,
320                              BadReader(marshal.dumps(value)))
321
322    def test_eof(self):
323        data = marshal.dumps(("hello", "dolly", None))
324        for i in range(len(data)):
325            self.assertRaises(EOFError, marshal.loads, data[0: i])
326
327LARGE_SIZE = 2**31
328pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
329
330class NullWriter:
331    def write(self, s):
332        pass
333
334@unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems")
335class LargeValuesTestCase(unittest.TestCase):
336    def check_unmarshallable(self, data):
337        self.assertRaises(ValueError, marshal.dump, data, NullWriter())
338
339    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
340    def test_bytes(self, size):
341        self.check_unmarshallable(b'x' * size)
342
343    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
344    def test_str(self, size):
345        self.check_unmarshallable('x' * size)
346
347    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
348    def test_tuple(self, size):
349        self.check_unmarshallable((None,) * size)
350
351    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
352    def test_list(self, size):
353        self.check_unmarshallable([None] * size)
354
355    @support.bigmemtest(size=LARGE_SIZE,
356            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
357            dry_run=False)
358    def test_set(self, size):
359        self.check_unmarshallable(set(range(size)))
360
361    @support.bigmemtest(size=LARGE_SIZE,
362            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
363            dry_run=False)
364    def test_frozenset(self, size):
365        self.check_unmarshallable(frozenset(range(size)))
366
367    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
368    def test_bytearray(self, size):
369        self.check_unmarshallable(bytearray(size))
370
371def CollectObjectIDs(ids, obj):
372    """Collect object ids seen in a structure"""
373    if id(obj) in ids:
374        return
375    ids.add(id(obj))
376    if isinstance(obj, (list, tuple, set, frozenset)):
377        for e in obj:
378            CollectObjectIDs(ids, e)
379    elif isinstance(obj, dict):
380        for k, v in obj.items():
381            CollectObjectIDs(ids, k)
382            CollectObjectIDs(ids, v)
383    return len(ids)
384
385class InstancingTestCase(unittest.TestCase, HelperMixin):
386    intobj = 123321
387    floatobj = 1.2345
388    strobj = "abcde"*3
389    dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
390
391    def helper3(self, rsample, recursive=False, simple=False):
392        #we have two instances
393        sample = (rsample, rsample)
394
395        n0 = CollectObjectIDs(set(), sample)
396
397        s3 = marshal.dumps(sample, 3)
398        n3 = CollectObjectIDs(set(), marshal.loads(s3))
399
400        #same number of instances generated
401        self.assertEqual(n3, n0)
402
403        if not recursive:
404            #can compare with version 2
405            s2 = marshal.dumps(sample, 2)
406            n2 = CollectObjectIDs(set(), marshal.loads(s2))
407            #old format generated more instances
408            self.assertGreater(n2, n0)
409
410            #if complex objects are in there, old format is larger
411            if not simple:
412                self.assertGreater(len(s2), len(s3))
413            else:
414                self.assertGreaterEqual(len(s2), len(s3))
415
416    def testInt(self):
417        self.helper(self.intobj)
418        self.helper3(self.intobj, simple=True)
419
420    def testFloat(self):
421        self.helper(self.floatobj)
422        self.helper3(self.floatobj)
423
424    def testStr(self):
425        self.helper(self.strobj)
426        self.helper3(self.strobj)
427
428    def testDict(self):
429        self.helper(self.dictobj)
430        self.helper3(self.dictobj)
431
432    def testModule(self):
433        with open(__file__, "rb") as f:
434            code = f.read()
435        if __file__.endswith(".py"):
436            code = compile(code, __file__, "exec")
437        self.helper(code)
438        self.helper3(code)
439
440    def testRecursion(self):
441        d = dict(self.dictobj)
442        d["self"] = d
443        self.helper3(d, recursive=True)
444        l = [self.dictobj]
445        l.append(l)
446        self.helper3(l, recursive=True)
447
448class CompatibilityTestCase(unittest.TestCase):
449    def _test(self, version):
450        with open(__file__, "rb") as f:
451            code = f.read()
452        if __file__.endswith(".py"):
453            code = compile(code, __file__, "exec")
454        data = marshal.dumps(code, version)
455        marshal.loads(data)
456
457    def test0To3(self):
458        self._test(0)
459
460    def test1To3(self):
461        self._test(1)
462
463    def test2To3(self):
464        self._test(2)
465
466    def test3To3(self):
467        self._test(3)
468
469class InterningTestCase(unittest.TestCase, HelperMixin):
470    strobj = "this is an interned string"
471    strobj = sys.intern(strobj)
472
473    def testIntern(self):
474        s = marshal.loads(marshal.dumps(self.strobj))
475        self.assertEqual(s, self.strobj)
476        self.assertEqual(id(s), id(self.strobj))
477        s2 = sys.intern(s)
478        self.assertEqual(id(s2), id(s))
479
480    def testNoIntern(self):
481        s = marshal.loads(marshal.dumps(self.strobj, 2))
482        self.assertEqual(s, self.strobj)
483        self.assertNotEqual(id(s), id(self.strobj))
484        s2 = sys.intern(s)
485        self.assertNotEqual(id(s2), id(s))
486
487@support.cpython_only
488@unittest.skipUnless(_testcapi, 'requires _testcapi')
489class CAPI_TestCase(unittest.TestCase, HelperMixin):
490
491    def test_write_long_to_file(self):
492        for v in range(marshal.version + 1):
493            _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v)
494            with open(support.TESTFN, 'rb') as f:
495                data = f.read()
496            support.unlink(support.TESTFN)
497            self.assertEqual(data, b'\x78\x56\x34\x12')
498
499    def test_write_object_to_file(self):
500        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000)
501        for v in range(marshal.version + 1):
502            _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v)
503            with open(support.TESTFN, 'rb') as f:
504                data = f.read()
505            support.unlink(support.TESTFN)
506            self.assertEqual(marshal.loads(data), obj)
507
508    def test_read_short_from_file(self):
509        with open(support.TESTFN, 'wb') as f:
510            f.write(b'\x34\x12xxxx')
511        r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN)
512        support.unlink(support.TESTFN)
513        self.assertEqual(r, 0x1234)
514        self.assertEqual(p, 2)
515
516        with open(support.TESTFN, 'wb') as f:
517            f.write(b'\x12')
518        with self.assertRaises(EOFError):
519            _testcapi.pymarshal_read_short_from_file(support.TESTFN)
520        support.unlink(support.TESTFN)
521
522    def test_read_long_from_file(self):
523        with open(support.TESTFN, 'wb') as f:
524            f.write(b'\x78\x56\x34\x12xxxx')
525        r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN)
526        support.unlink(support.TESTFN)
527        self.assertEqual(r, 0x12345678)
528        self.assertEqual(p, 4)
529
530        with open(support.TESTFN, 'wb') as f:
531            f.write(b'\x56\x34\x12')
532        with self.assertRaises(EOFError):
533            _testcapi.pymarshal_read_long_from_file(support.TESTFN)
534        support.unlink(support.TESTFN)
535
536    def test_read_last_object_from_file(self):
537        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
538        for v in range(marshal.version + 1):
539            data = marshal.dumps(obj, v)
540            with open(support.TESTFN, 'wb') as f:
541                f.write(data + b'xxxx')
542            r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN)
543            support.unlink(support.TESTFN)
544            self.assertEqual(r, obj)
545
546            with open(support.TESTFN, 'wb') as f:
547                f.write(data[:1])
548            with self.assertRaises(EOFError):
549                _testcapi.pymarshal_read_last_object_from_file(support.TESTFN)
550            support.unlink(support.TESTFN)
551
552    def test_read_object_from_file(self):
553        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
554        for v in range(marshal.version + 1):
555            data = marshal.dumps(obj, v)
556            with open(support.TESTFN, 'wb') as f:
557                f.write(data + b'xxxx')
558            r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN)
559            support.unlink(support.TESTFN)
560            self.assertEqual(r, obj)
561            self.assertEqual(p, len(data))
562
563            with open(support.TESTFN, 'wb') as f:
564                f.write(data[:1])
565            with self.assertRaises(EOFError):
566                _testcapi.pymarshal_read_object_from_file(support.TESTFN)
567            support.unlink(support.TESTFN)
568
569
570if __name__ == "__main__":
571    unittest.main()
572