1from test import support
2import array
3import io
4import marshal
5import sys
6import unittest
7import os
8import types
9
10try:
11    import _testcapi
12except ImportError:
13    _testcapi = None
14
15class HelperMixin:
16    def helper(self, sample, *extra):
17        new = marshal.loads(marshal.dumps(sample, *extra))
18        self.assertEqual(sample, new)
19        try:
20            with open(support.TESTFN, "wb") as f:
21                marshal.dump(sample, f, *extra)
22            with open(support.TESTFN, "rb") as f:
23                new = marshal.load(f)
24            self.assertEqual(sample, new)
25        finally:
26            support.unlink(support.TESTFN)
27
28class IntTestCase(unittest.TestCase, HelperMixin):
29    def test_ints(self):
30        # Test a range of Python ints larger than the machine word size.
31        n = sys.maxsize ** 2
32        while n:
33            for expected in (-n, n):
34                self.helper(expected)
35            n = n >> 1
36
37    def test_int64(self):
38        # Simulate int marshaling with TYPE_INT64.
39        maxint64 = (1 << 63) - 1
40        minint64 = -maxint64-1
41        for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
42            while base:
43                s = b'I' + int.to_bytes(base, 8, 'little', signed=True)
44                got = marshal.loads(s)
45                self.assertEqual(base, got)
46                if base == -1:  # a fixed-point for shifting right 1
47                    base = 0
48                else:
49                    base >>= 1
50
51        got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10')
52        self.assertEqual(got, 0x1032547698badcfe)
53        got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef')
54        self.assertEqual(got, -0x1032547698badcff)
55        got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f')
56        self.assertEqual(got, 0x7f6e5d4c3b2a1908)
57        got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80')
58        self.assertEqual(got, -0x7f6e5d4c3b2a1909)
59
60    def test_bool(self):
61        for b in (True, False):
62            self.helper(b)
63
64class FloatTestCase(unittest.TestCase, HelperMixin):
65    def test_floats(self):
66        # Test a few floats
67        small = 1e-25
68        n = sys.maxsize * 3.7e250
69        while n > small:
70            for expected in (-n, n):
71                self.helper(float(expected))
72            n /= 123.4567
73
74        f = 0.0
75        s = marshal.dumps(f, 2)
76        got = marshal.loads(s)
77        self.assertEqual(f, got)
78        # and with version <= 1 (floats marshalled differently then)
79        s = marshal.dumps(f, 1)
80        got = marshal.loads(s)
81        self.assertEqual(f, got)
82
83        n = sys.maxsize * 3.7e-250
84        while n < small:
85            for expected in (-n, n):
86                f = float(expected)
87                self.helper(f)
88                self.helper(f, 1)
89            n *= 123.4567
90
91class StringTestCase(unittest.TestCase, HelperMixin):
92    def test_unicode(self):
93        for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
94            self.helper(marshal.loads(marshal.dumps(s)))
95
96    def test_string(self):
97        for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
98            self.helper(s)
99
100    def test_bytes(self):
101        for s in [b"", b"Andr\xe8 Previn", b"abc", b" "*10000]:
102            self.helper(s)
103
104class ExceptionTestCase(unittest.TestCase):
105    def test_exceptions(self):
106        new = marshal.loads(marshal.dumps(StopIteration))
107        self.assertEqual(StopIteration, new)
108
109class CodeTestCase(unittest.TestCase):
110    def test_code(self):
111        co = ExceptionTestCase.test_exceptions.__code__
112        new = marshal.loads(marshal.dumps(co))
113        self.assertEqual(co, new)
114
115    def test_many_codeobjects(self):
116        # Issue2957: bad recursion count on code objects
117        count = 5000    # more than MAX_MARSHAL_STACK_DEPTH
118        codes = (ExceptionTestCase.test_exceptions.__code__,) * count
119        marshal.loads(marshal.dumps(codes))
120
121    def test_different_filenames(self):
122        co1 = compile("x", "f1", "exec")
123        co2 = compile("y", "f2", "exec")
124        co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
125        self.assertEqual(co1.co_filename, "f1")
126        self.assertEqual(co2.co_filename, "f2")
127
128    @support.cpython_only
129    def test_same_filename_used(self):
130        s = """def f(): pass\ndef g(): pass"""
131        co = compile(s, "myfile", "exec")
132        co = marshal.loads(marshal.dumps(co))
133        for obj in co.co_consts:
134            if isinstance(obj, types.CodeType):
135                self.assertIs(co.co_filename, obj.co_filename)
136
137class ContainerTestCase(unittest.TestCase, HelperMixin):
138    d = {'astring': 'foo@bar.baz.spam',
139         'afloat': 7283.43,
140         'anint': 2**20,
141         'ashortlong': 2,
142         'alist': ['.zyx.41'],
143         'atuple': ('.zyx.41',)*10,
144         'aboolean': False,
145         'aunicode': "Andr\xe8 Previn"
146         }
147
148    def test_dict(self):
149        self.helper(self.d)
150
151    def test_list(self):
152        self.helper(list(self.d.items()))
153
154    def test_tuple(self):
155        self.helper(tuple(self.d.keys()))
156
157    def test_sets(self):
158        for constructor in (set, frozenset):
159            self.helper(constructor(self.d.keys()))
160
161    @support.cpython_only
162    def test_empty_frozenset_singleton(self):
163        # marshal.loads() must reuse the empty frozenset singleton
164        obj = frozenset()
165        obj2 = marshal.loads(marshal.dumps(obj))
166        self.assertIs(obj2, obj)
167
168
169class BufferTestCase(unittest.TestCase, HelperMixin):
170
171    def test_bytearray(self):
172        b = bytearray(b"abc")
173        self.helper(b)
174        new = marshal.loads(marshal.dumps(b))
175        self.assertEqual(type(new), bytes)
176
177    def test_memoryview(self):
178        b = memoryview(b"abc")
179        self.helper(b)
180        new = marshal.loads(marshal.dumps(b))
181        self.assertEqual(type(new), bytes)
182
183    def test_array(self):
184        a = array.array('B', b"abc")
185        new = marshal.loads(marshal.dumps(a))
186        self.assertEqual(new, b"abc")
187
188
189class BugsTestCase(unittest.TestCase):
190    def test_bug_5888452(self):
191        # Simple-minded check for SF 588452: Debug build crashes
192        marshal.dumps([128] * 1000)
193
194    def test_patch_873224(self):
195        self.assertRaises(Exception, marshal.loads, b'0')
196        self.assertRaises(Exception, marshal.loads, b'f')
197        self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1])
198
199    def test_version_argument(self):
200        # Python 2.4.0 crashes for any call to marshal.dumps(x, y)
201        self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5)
202        self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5)
203
204    def test_fuzz(self):
205        # simple test that it's at least not *totally* trivial to
206        # crash from bad marshal data
207        for i in range(256):
208            c = bytes([i])
209            try:
210                marshal.loads(c)
211            except Exception:
212                pass
213
214    def test_loads_recursion(self):
215        def run_tests(N, check):
216            # (((...None...),),)
217            check(b')\x01' * N + b'N')
218            check(b'(\x01\x00\x00\x00' * N + b'N')
219            # [[[...None...]]]
220            check(b'[\x01\x00\x00\x00' * N + b'N')
221            # {None: {None: {None: ...None...}}}
222            check(b'{N' * N + b'N' + b'0' * N)
223            # frozenset([frozenset([frozenset([...None...])])])
224            check(b'>\x01\x00\x00\x00' * N + b'N')
225        # Check that the generated marshal data is valid and marshal.loads()
226        # works for moderately deep nesting
227        run_tests(100, marshal.loads)
228        # Very deeply nested structure shouldn't blow the stack
229        def check(s):
230            self.assertRaises(ValueError, marshal.loads, s)
231        run_tests(2**20, check)
232
233    def test_recursion_limit(self):
234        # Create a deeply nested structure.
235        head = last = []
236        # The max stack depth should match the value in Python/marshal.c.
237        # BUG: https://bugs.python.org/issue33720
238        # Windows always limits the maximum depth on release and debug builds
239        #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
240        if os.name == 'nt':
241            MAX_MARSHAL_STACK_DEPTH = 1000
242        else:
243            MAX_MARSHAL_STACK_DEPTH = 2000
244        for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
245            last.append([0])
246            last = last[-1]
247
248        # Verify we don't blow out the stack with dumps/load.
249        data = marshal.dumps(head)
250        new_head = marshal.loads(data)
251        # Don't use == to compare objects, it can exceed the recursion limit.
252        self.assertEqual(len(new_head), len(head))
253        self.assertEqual(len(new_head[0]), len(head[0]))
254        self.assertEqual(len(new_head[-1]), len(head[-1]))
255
256        last.append([0])
257        self.assertRaises(ValueError, marshal.dumps, head)
258
259    def test_exact_type_match(self):
260        # Former bug:
261        #   >>> class Int(int): pass
262        #   >>> type(loads(dumps(Int())))
263        #   <type 'int'>
264        for typ in (int, float, complex, tuple, list, dict, set, frozenset):
265            # Note: str subclasses are not tested because they get handled
266            # by marshal's routines for objects supporting the buffer API.
267            subtyp = type('subtyp', (typ,), {})
268            self.assertRaises(ValueError, marshal.dumps, subtyp())
269
270    # Issue #1792 introduced a change in how marshal increases the size of its
271    # internal buffer; this test ensures that the new code is exercised.
272    def test_large_marshal(self):
273        size = int(1e6)
274        testString = 'abc' * size
275        marshal.dumps(testString)
276
277    def test_invalid_longs(self):
278        # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs
279        invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
280        self.assertRaises(ValueError, marshal.loads, invalid_string)
281
282    def test_multiple_dumps_and_loads(self):
283        # Issue 12291: marshal.load() should be callable multiple times
284        # with interleaved data written by non-marshal code
285        # Adapted from a patch by Engelbert Gruber.
286        data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))
287        for interleaved in (b'', b'0123'):
288            ilen = len(interleaved)
289            positions = []
290            try:
291                with open(support.TESTFN, 'wb') as f:
292                    for d in data:
293                        marshal.dump(d, f)
294                        if ilen:
295                            f.write(interleaved)
296                        positions.append(f.tell())
297                with open(support.TESTFN, 'rb') as f:
298                    for i, d in enumerate(data):
299                        self.assertEqual(d, marshal.load(f))
300                        if ilen:
301                            f.read(ilen)
302                        self.assertEqual(positions[i], f.tell())
303            finally:
304                support.unlink(support.TESTFN)
305
306    def test_loads_reject_unicode_strings(self):
307        # Issue #14177: marshal.loads() should not accept unicode strings
308        unicode_string = 'T'
309        self.assertRaises(TypeError, marshal.loads, unicode_string)
310
311    def test_bad_reader(self):
312        class BadReader(io.BytesIO):
313            def readinto(self, buf):
314                n = super().readinto(buf)
315                if n is not None and n > 4:
316                    n += 10**6
317                return n
318        for value in (1.0, 1j, b'0123456789', '0123456789'):
319            self.assertRaises(ValueError, marshal.load,
320                              BadReader(marshal.dumps(value)))
321
322    def test_eof(self):
323        data = marshal.dumps(("hello", "dolly", None))
324        for i in range(len(data)):
325            self.assertRaises(EOFError, marshal.loads, data[0: i])
326
327LARGE_SIZE = 2**31
328pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
329
330class NullWriter:
331    def write(self, s):
332        pass
333
334@unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems")
335class LargeValuesTestCase(unittest.TestCase):
336    def check_unmarshallable(self, data):
337        self.assertRaises(ValueError, marshal.dump, data, NullWriter())
338
339    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
340    def test_bytes(self, size):
341        self.check_unmarshallable(b'x' * size)
342
343    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
344    def test_str(self, size):
345        self.check_unmarshallable('x' * size)
346
347    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
348    def test_tuple(self, size):
349        self.check_unmarshallable((None,) * size)
350
351    @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
352    def test_list(self, size):
353        self.check_unmarshallable([None] * size)
354
355    @support.bigmemtest(size=LARGE_SIZE,
356            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
357            dry_run=False)
358    def test_set(self, size):
359        self.check_unmarshallable(set(range(size)))
360
361    @support.bigmemtest(size=LARGE_SIZE,
362            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
363            dry_run=False)
364    def test_frozenset(self, size):
365        self.check_unmarshallable(frozenset(range(size)))
366
367    @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
368    def test_bytearray(self, size):
369        self.check_unmarshallable(bytearray(size))
370
371def CollectObjectIDs(ids, obj):
372    """Collect object ids seen in a structure"""
373    if id(obj) in ids:
374        return
375    ids.add(id(obj))
376    if isinstance(obj, (list, tuple, set, frozenset)):
377        for e in obj:
378            CollectObjectIDs(ids, e)
379    elif isinstance(obj, dict):
380        for k, v in obj.items():
381            CollectObjectIDs(ids, k)
382            CollectObjectIDs(ids, v)
383    return len(ids)
384
385class InstancingTestCase(unittest.TestCase, HelperMixin):
386    keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'}))
387
388    def helper3(self, rsample, recursive=False, simple=False):
389        #we have two instances
390        sample = (rsample, rsample)
391
392        n0 = CollectObjectIDs(set(), sample)
393
394        for v in range(3, marshal.version + 1):
395            s3 = marshal.dumps(sample, v)
396            n3 = CollectObjectIDs(set(), marshal.loads(s3))
397
398            #same number of instances generated
399            self.assertEqual(n3, n0)
400
401        if not recursive:
402            #can compare with version 2
403            s2 = marshal.dumps(sample, 2)
404            n2 = CollectObjectIDs(set(), marshal.loads(s2))
405            #old format generated more instances
406            self.assertGreater(n2, n0)
407
408            #if complex objects are in there, old format is larger
409            if not simple:
410                self.assertGreater(len(s2), len(s3))
411            else:
412                self.assertGreaterEqual(len(s2), len(s3))
413
414    def testInt(self):
415        intobj = 123321
416        self.helper(intobj)
417        self.helper3(intobj, simple=True)
418
419    def testFloat(self):
420        floatobj = 1.2345
421        self.helper(floatobj)
422        self.helper3(floatobj)
423
424    def testStr(self):
425        strobj = "abcde"*3
426        self.helper(strobj)
427        self.helper3(strobj)
428
429    def testBytes(self):
430        bytesobj = b"abcde"*3
431        self.helper(bytesobj)
432        self.helper3(bytesobj)
433
434    def testList(self):
435        for obj in self.keys:
436            listobj = [obj, obj]
437            self.helper(listobj)
438            self.helper3(listobj)
439
440    def testTuple(self):
441        for obj in self.keys:
442            tupleobj = (obj, obj)
443            self.helper(tupleobj)
444            self.helper3(tupleobj)
445
446    def testSet(self):
447        for obj in self.keys:
448            setobj = {(obj, 1), (obj, 2)}
449            self.helper(setobj)
450            self.helper3(setobj)
451
452    def testFrozenSet(self):
453        for obj in self.keys:
454            frozensetobj = frozenset({(obj, 1), (obj, 2)})
455            self.helper(frozensetobj)
456            self.helper3(frozensetobj)
457
458    def testDict(self):
459        for obj in self.keys:
460            dictobj = {"hello": obj, "goodbye": obj, obj: "hello"}
461            self.helper(dictobj)
462            self.helper3(dictobj)
463
464    def testModule(self):
465        with open(__file__, "rb") as f:
466            code = f.read()
467        if __file__.endswith(".py"):
468            code = compile(code, __file__, "exec")
469        self.helper(code)
470        self.helper3(code)
471
472    def testRecursion(self):
473        obj = 1.2345
474        d = {"hello": obj, "goodbye": obj, obj: "hello"}
475        d["self"] = d
476        self.helper3(d, recursive=True)
477        l = [obj, obj]
478        l.append(l)
479        self.helper3(l, recursive=True)
480
481class CompatibilityTestCase(unittest.TestCase):
482    def _test(self, version):
483        with open(__file__, "rb") as f:
484            code = f.read()
485        if __file__.endswith(".py"):
486            code = compile(code, __file__, "exec")
487        data = marshal.dumps(code, version)
488        marshal.loads(data)
489
490    def test0To3(self):
491        self._test(0)
492
493    def test1To3(self):
494        self._test(1)
495
496    def test2To3(self):
497        self._test(2)
498
499    def test3To3(self):
500        self._test(3)
501
502class InterningTestCase(unittest.TestCase, HelperMixin):
503    strobj = "this is an interned string"
504    strobj = sys.intern(strobj)
505
506    def testIntern(self):
507        s = marshal.loads(marshal.dumps(self.strobj))
508        self.assertEqual(s, self.strobj)
509        self.assertEqual(id(s), id(self.strobj))
510        s2 = sys.intern(s)
511        self.assertEqual(id(s2), id(s))
512
513    def testNoIntern(self):
514        s = marshal.loads(marshal.dumps(self.strobj, 2))
515        self.assertEqual(s, self.strobj)
516        self.assertNotEqual(id(s), id(self.strobj))
517        s2 = sys.intern(s)
518        self.assertNotEqual(id(s2), id(s))
519
520@support.cpython_only
521@unittest.skipUnless(_testcapi, 'requires _testcapi')
522class CAPI_TestCase(unittest.TestCase, HelperMixin):
523
524    def test_write_long_to_file(self):
525        for v in range(marshal.version + 1):
526            _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v)
527            with open(support.TESTFN, 'rb') as f:
528                data = f.read()
529            support.unlink(support.TESTFN)
530            self.assertEqual(data, b'\x78\x56\x34\x12')
531
532    def test_write_object_to_file(self):
533        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000)
534        for v in range(marshal.version + 1):
535            _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v)
536            with open(support.TESTFN, 'rb') as f:
537                data = f.read()
538            support.unlink(support.TESTFN)
539            self.assertEqual(marshal.loads(data), obj)
540
541    def test_read_short_from_file(self):
542        with open(support.TESTFN, 'wb') as f:
543            f.write(b'\x34\x12xxxx')
544        r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN)
545        support.unlink(support.TESTFN)
546        self.assertEqual(r, 0x1234)
547        self.assertEqual(p, 2)
548
549        with open(support.TESTFN, 'wb') as f:
550            f.write(b'\x12')
551        with self.assertRaises(EOFError):
552            _testcapi.pymarshal_read_short_from_file(support.TESTFN)
553        support.unlink(support.TESTFN)
554
555    def test_read_long_from_file(self):
556        with open(support.TESTFN, 'wb') as f:
557            f.write(b'\x78\x56\x34\x12xxxx')
558        r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN)
559        support.unlink(support.TESTFN)
560        self.assertEqual(r, 0x12345678)
561        self.assertEqual(p, 4)
562
563        with open(support.TESTFN, 'wb') as f:
564            f.write(b'\x56\x34\x12')
565        with self.assertRaises(EOFError):
566            _testcapi.pymarshal_read_long_from_file(support.TESTFN)
567        support.unlink(support.TESTFN)
568
569    def test_read_last_object_from_file(self):
570        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
571        for v in range(marshal.version + 1):
572            data = marshal.dumps(obj, v)
573            with open(support.TESTFN, 'wb') as f:
574                f.write(data + b'xxxx')
575            r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN)
576            support.unlink(support.TESTFN)
577            self.assertEqual(r, obj)
578
579            with open(support.TESTFN, 'wb') as f:
580                f.write(data[:1])
581            with self.assertRaises(EOFError):
582                _testcapi.pymarshal_read_last_object_from_file(support.TESTFN)
583            support.unlink(support.TESTFN)
584
585    def test_read_object_from_file(self):
586        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
587        for v in range(marshal.version + 1):
588            data = marshal.dumps(obj, v)
589            with open(support.TESTFN, 'wb') as f:
590                f.write(data + b'xxxx')
591            r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN)
592            support.unlink(support.TESTFN)
593            self.assertEqual(r, obj)
594            self.assertEqual(p, len(data))
595
596            with open(support.TESTFN, 'wb') as f:
597                f.write(data[:1])
598            with self.assertRaises(EOFError):
599                _testcapi.pymarshal_read_object_from_file(support.TESTFN)
600            support.unlink(support.TESTFN)
601
602
603if __name__ == "__main__":
604    unittest.main()
605