1import collections 2import copyreg 3import dbm 4import io 5import functools 6import os 7import math 8import pickle 9import pickletools 10import shutil 11import struct 12import sys 13import threading 14import unittest 15import weakref 16from textwrap import dedent 17from http.cookies import SimpleCookie 18 19try: 20 import _testbuffer 21except ImportError: 22 _testbuffer = None 23 24from test import support 25from test.support import os_helper 26from test.support import ( 27 TestFailed, run_with_locale, no_tracing, 28 _2G, _4G, bigmemtest 29 ) 30from test.support.import_helper import forget 31from test.support.os_helper import TESTFN 32from test.support import threading_helper 33from test.support.warnings_helper import save_restore_warnings_filters 34 35from pickle import bytes_types 36 37 38# bpo-41003: Save/restore warnings filters to leave them unchanged. 39# Ignore filters installed by numpy. 40try: 41 with save_restore_warnings_filters(): 42 import numpy as np 43except ImportError: 44 np = None 45 46 47requires_32b = unittest.skipUnless(sys.maxsize < 2**32, 48 "test is only meaningful on 32-bit builds") 49 50# Tests that try a number of pickle protocols should have a 51# for proto in protocols: 52# kind of outer loop. 53protocols = range(pickle.HIGHEST_PROTOCOL + 1) 54 55 56# Return True if opcode code appears in the pickle, else False. 57def opcode_in_pickle(code, pickle): 58 for op, dummy, dummy in pickletools.genops(pickle): 59 if op.code == code.decode("latin-1"): 60 return True 61 return False 62 63# Return the number of times opcode code appears in pickle. 64def count_opcode(code, pickle): 65 n = 0 66 for op, dummy, dummy in pickletools.genops(pickle): 67 if op.code == code.decode("latin-1"): 68 n += 1 69 return n 70 71 72def identity(x): 73 return x 74 75 76class UnseekableIO(io.BytesIO): 77 def peek(self, *args): 78 raise NotImplementedError 79 80 def seekable(self): 81 return False 82 83 def seek(self, *args): 84 raise io.UnsupportedOperation 85 86 def tell(self): 87 raise io.UnsupportedOperation 88 89 90class MinimalIO(object): 91 """ 92 A file-like object that doesn't support readinto(). 93 """ 94 def __init__(self, *args): 95 self._bio = io.BytesIO(*args) 96 self.getvalue = self._bio.getvalue 97 self.read = self._bio.read 98 self.readline = self._bio.readline 99 self.write = self._bio.write 100 101 102# We can't very well test the extension registry without putting known stuff 103# in it, but we have to be careful to restore its original state. Code 104# should do this: 105# 106# e = ExtensionSaver(extension_code) 107# try: 108# fiddle w/ the extension registry's stuff for extension_code 109# finally: 110# e.restore() 111 112class ExtensionSaver: 113 # Remember current registration for code (if any), and remove it (if 114 # there is one). 115 def __init__(self, code): 116 self.code = code 117 if code in copyreg._inverted_registry: 118 self.pair = copyreg._inverted_registry[code] 119 copyreg.remove_extension(self.pair[0], self.pair[1], code) 120 else: 121 self.pair = None 122 123 # Restore previous registration for code. 124 def restore(self): 125 code = self.code 126 curpair = copyreg._inverted_registry.get(code) 127 if curpair is not None: 128 copyreg.remove_extension(curpair[0], curpair[1], code) 129 pair = self.pair 130 if pair is not None: 131 copyreg.add_extension(pair[0], pair[1], code) 132 133class C: 134 def __eq__(self, other): 135 return self.__dict__ == other.__dict__ 136 137class D(C): 138 def __init__(self, arg): 139 pass 140 141class E(C): 142 def __getinitargs__(self): 143 return () 144 145# Simple mutable object. 146class Object: 147 pass 148 149# Hashable immutable key object containing unheshable mutable data. 150class K: 151 def __init__(self, value): 152 self.value = value 153 154 def __reduce__(self): 155 # Shouldn't support the recursion itself 156 return K, (self.value,) 157 158import __main__ 159__main__.C = C 160C.__module__ = "__main__" 161__main__.D = D 162D.__module__ = "__main__" 163__main__.E = E 164E.__module__ = "__main__" 165 166class myint(int): 167 def __init__(self, x): 168 self.str = str(x) 169 170class initarg(C): 171 172 def __init__(self, a, b): 173 self.a = a 174 self.b = b 175 176 def __getinitargs__(self): 177 return self.a, self.b 178 179class metaclass(type): 180 pass 181 182class use_metaclass(object, metaclass=metaclass): 183 pass 184 185class pickling_metaclass(type): 186 def __eq__(self, other): 187 return (type(self) == type(other) and 188 self.reduce_args == other.reduce_args) 189 190 def __reduce__(self): 191 return (create_dynamic_class, self.reduce_args) 192 193def create_dynamic_class(name, bases): 194 result = pickling_metaclass(name, bases, dict()) 195 result.reduce_args = (name, bases) 196 return result 197 198 199class ZeroCopyBytes(bytes): 200 readonly = True 201 c_contiguous = True 202 f_contiguous = True 203 zero_copy_reconstruct = True 204 205 def __reduce_ex__(self, protocol): 206 if protocol >= 5: 207 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 208 else: 209 return type(self)._reconstruct, (bytes(self),) 210 211 def __repr__(self): 212 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 213 214 __str__ = __repr__ 215 216 @classmethod 217 def _reconstruct(cls, obj): 218 with memoryview(obj) as m: 219 obj = m.obj 220 if type(obj) is cls: 221 # Zero-copy 222 return obj 223 else: 224 return cls(obj) 225 226 227class ZeroCopyBytearray(bytearray): 228 readonly = False 229 c_contiguous = True 230 f_contiguous = True 231 zero_copy_reconstruct = True 232 233 def __reduce_ex__(self, protocol): 234 if protocol >= 5: 235 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 236 else: 237 return type(self)._reconstruct, (bytes(self),) 238 239 def __repr__(self): 240 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 241 242 __str__ = __repr__ 243 244 @classmethod 245 def _reconstruct(cls, obj): 246 with memoryview(obj) as m: 247 obj = m.obj 248 if type(obj) is cls: 249 # Zero-copy 250 return obj 251 else: 252 return cls(obj) 253 254 255if _testbuffer is not None: 256 257 class PicklableNDArray: 258 # A not-really-zero-copy picklable ndarray, as the ndarray() 259 # constructor doesn't allow for it 260 261 zero_copy_reconstruct = False 262 263 def __init__(self, *args, **kwargs): 264 self.array = _testbuffer.ndarray(*args, **kwargs) 265 266 def __getitem__(self, idx): 267 cls = type(self) 268 new = cls.__new__(cls) 269 new.array = self.array[idx] 270 return new 271 272 @property 273 def readonly(self): 274 return self.array.readonly 275 276 @property 277 def c_contiguous(self): 278 return self.array.c_contiguous 279 280 @property 281 def f_contiguous(self): 282 return self.array.f_contiguous 283 284 def __eq__(self, other): 285 if not isinstance(other, PicklableNDArray): 286 return NotImplemented 287 return (other.array.format == self.array.format and 288 other.array.shape == self.array.shape and 289 other.array.strides == self.array.strides and 290 other.array.readonly == self.array.readonly and 291 other.array.tobytes() == self.array.tobytes()) 292 293 def __ne__(self, other): 294 if not isinstance(other, PicklableNDArray): 295 return NotImplemented 296 return not (self == other) 297 298 def __repr__(self): 299 return (f"{type(self)}(shape={self.array.shape}," 300 f"strides={self.array.strides}, " 301 f"bytes={self.array.tobytes()})") 302 303 def __reduce_ex__(self, protocol): 304 if not self.array.contiguous: 305 raise NotImplementedError("Reconstructing a non-contiguous " 306 "ndarray does not seem possible") 307 ndarray_kwargs = {"shape": self.array.shape, 308 "strides": self.array.strides, 309 "format": self.array.format, 310 "flags": (0 if self.readonly 311 else _testbuffer.ND_WRITABLE)} 312 pb = pickle.PickleBuffer(self.array) 313 if protocol >= 5: 314 return (type(self)._reconstruct, 315 (pb, ndarray_kwargs)) 316 else: 317 # Need to serialize the bytes in physical order 318 with pb.raw() as m: 319 return (type(self)._reconstruct, 320 (m.tobytes(), ndarray_kwargs)) 321 322 @classmethod 323 def _reconstruct(cls, obj, kwargs): 324 with memoryview(obj) as m: 325 # For some reason, ndarray() wants a list of integers... 326 # XXX This only works if format == 'B' 327 items = list(m.tobytes()) 328 return cls(items, **kwargs) 329 330 331# DATA0 .. DATA4 are the pickles we expect under the various protocols, for 332# the object returned by create_data(). 333 334DATA0 = ( 335 b'(lp0\nL0L\naL1L\naF2.0\n' 336 b'ac__builtin__\ncomple' 337 b'x\np1\n(F3.0\nF0.0\ntp2\n' 338 b'Rp3\naL1L\naL-1L\naL255' 339 b'L\naL-255L\naL-256L\naL' 340 b'65535L\naL-65535L\naL-' 341 b'65536L\naL2147483647L' 342 b'\naL-2147483647L\naL-2' 343 b'147483648L\na(Vabc\np4' 344 b'\ng4\nccopy_reg\n_recon' 345 b'structor\np5\n(c__main' 346 b'__\nC\np6\nc__builtin__' 347 b'\nobject\np7\nNtp8\nRp9\n' 348 b'(dp10\nVfoo\np11\nL1L\ns' 349 b'Vbar\np12\nL2L\nsbg9\ntp' 350 b'13\nag13\naL5L\na.' 351) 352 353# Disassembly of DATA0 354DATA0_DIS = """\ 355 0: ( MARK 356 1: l LIST (MARK at 0) 357 2: p PUT 0 358 5: L LONG 0 359 9: a APPEND 360 10: L LONG 1 361 14: a APPEND 362 15: F FLOAT 2.0 363 20: a APPEND 364 21: c GLOBAL '__builtin__ complex' 365 42: p PUT 1 366 45: ( MARK 367 46: F FLOAT 3.0 368 51: F FLOAT 0.0 369 56: t TUPLE (MARK at 45) 370 57: p PUT 2 371 60: R REDUCE 372 61: p PUT 3 373 64: a APPEND 374 65: L LONG 1 375 69: a APPEND 376 70: L LONG -1 377 75: a APPEND 378 76: L LONG 255 379 82: a APPEND 380 83: L LONG -255 381 90: a APPEND 382 91: L LONG -256 383 98: a APPEND 384 99: L LONG 65535 385 107: a APPEND 386 108: L LONG -65535 387 117: a APPEND 388 118: L LONG -65536 389 127: a APPEND 390 128: L LONG 2147483647 391 141: a APPEND 392 142: L LONG -2147483647 393 156: a APPEND 394 157: L LONG -2147483648 395 171: a APPEND 396 172: ( MARK 397 173: V UNICODE 'abc' 398 178: p PUT 4 399 181: g GET 4 400 184: c GLOBAL 'copy_reg _reconstructor' 401 209: p PUT 5 402 212: ( MARK 403 213: c GLOBAL '__main__ C' 404 225: p PUT 6 405 228: c GLOBAL '__builtin__ object' 406 248: p PUT 7 407 251: N NONE 408 252: t TUPLE (MARK at 212) 409 253: p PUT 8 410 256: R REDUCE 411 257: p PUT 9 412 260: ( MARK 413 261: d DICT (MARK at 260) 414 262: p PUT 10 415 266: V UNICODE 'foo' 416 271: p PUT 11 417 275: L LONG 1 418 279: s SETITEM 419 280: V UNICODE 'bar' 420 285: p PUT 12 421 289: L LONG 2 422 293: s SETITEM 423 294: b BUILD 424 295: g GET 9 425 298: t TUPLE (MARK at 172) 426 299: p PUT 13 427 303: a APPEND 428 304: g GET 13 429 308: a APPEND 430 309: L LONG 5 431 313: a APPEND 432 314: . STOP 433highest protocol among opcodes = 0 434""" 435 436DATA1 = ( 437 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__' 438 b'builtin__\ncomplex\nq\x01' 439 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t' 440 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ' 441 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff' 442 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab' 443 b'cq\x04h\x04ccopy_reg\n_reco' 444 b'nstructor\nq\x05(c__main' 445 b'__\nC\nq\x06c__builtin__\n' 446 b'object\nq\x07Ntq\x08Rq\t}q\n(' 447 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar' 448 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.' 449) 450 451# Disassembly of DATA1 452DATA1_DIS = """\ 453 0: ] EMPTY_LIST 454 1: q BINPUT 0 455 3: ( MARK 456 4: K BININT1 0 457 6: K BININT1 1 458 8: G BINFLOAT 2.0 459 17: c GLOBAL '__builtin__ complex' 460 38: q BINPUT 1 461 40: ( MARK 462 41: G BINFLOAT 3.0 463 50: G BINFLOAT 0.0 464 59: t TUPLE (MARK at 40) 465 60: q BINPUT 2 466 62: R REDUCE 467 63: q BINPUT 3 468 65: K BININT1 1 469 67: J BININT -1 470 72: K BININT1 255 471 74: J BININT -255 472 79: J BININT -256 473 84: M BININT2 65535 474 87: J BININT -65535 475 92: J BININT -65536 476 97: J BININT 2147483647 477 102: J BININT -2147483647 478 107: J BININT -2147483648 479 112: ( MARK 480 113: X BINUNICODE 'abc' 481 121: q BINPUT 4 482 123: h BINGET 4 483 125: c GLOBAL 'copy_reg _reconstructor' 484 150: q BINPUT 5 485 152: ( MARK 486 153: c GLOBAL '__main__ C' 487 165: q BINPUT 6 488 167: c GLOBAL '__builtin__ object' 489 187: q BINPUT 7 490 189: N NONE 491 190: t TUPLE (MARK at 152) 492 191: q BINPUT 8 493 193: R REDUCE 494 194: q BINPUT 9 495 196: } EMPTY_DICT 496 197: q BINPUT 10 497 199: ( MARK 498 200: X BINUNICODE 'foo' 499 208: q BINPUT 11 500 210: K BININT1 1 501 212: X BINUNICODE 'bar' 502 220: q BINPUT 12 503 222: K BININT1 2 504 224: u SETITEMS (MARK at 199) 505 225: b BUILD 506 226: h BINGET 9 507 228: t TUPLE (MARK at 112) 508 229: q BINPUT 13 509 231: h BINGET 13 510 233: K BININT1 5 511 235: e APPENDS (MARK at 3) 512 236: . STOP 513highest protocol among opcodes = 1 514""" 515 516DATA2 = ( 517 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 518 b'__builtin__\ncomplex\n' 519 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00' 520 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff' 521 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff' 522 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a' 523 b'bcq\x04h\x04c__main__\nC\nq\x05' 524 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01' 525 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh' 526 b'\nK\x05e.' 527) 528 529# Disassembly of DATA2 530DATA2_DIS = """\ 531 0: \x80 PROTO 2 532 2: ] EMPTY_LIST 533 3: q BINPUT 0 534 5: ( MARK 535 6: K BININT1 0 536 8: K BININT1 1 537 10: G BINFLOAT 2.0 538 19: c GLOBAL '__builtin__ complex' 539 40: q BINPUT 1 540 42: G BINFLOAT 3.0 541 51: G BINFLOAT 0.0 542 60: \x86 TUPLE2 543 61: q BINPUT 2 544 63: R REDUCE 545 64: q BINPUT 3 546 66: K BININT1 1 547 68: J BININT -1 548 73: K BININT1 255 549 75: J BININT -255 550 80: J BININT -256 551 85: M BININT2 65535 552 88: J BININT -65535 553 93: J BININT -65536 554 98: J BININT 2147483647 555 103: J BININT -2147483647 556 108: J BININT -2147483648 557 113: ( MARK 558 114: X BINUNICODE 'abc' 559 122: q BINPUT 4 560 124: h BINGET 4 561 126: c GLOBAL '__main__ C' 562 138: q BINPUT 5 563 140: ) EMPTY_TUPLE 564 141: \x81 NEWOBJ 565 142: q BINPUT 6 566 144: } EMPTY_DICT 567 145: q BINPUT 7 568 147: ( MARK 569 148: X BINUNICODE 'foo' 570 156: q BINPUT 8 571 158: K BININT1 1 572 160: X BINUNICODE 'bar' 573 168: q BINPUT 9 574 170: K BININT1 2 575 172: u SETITEMS (MARK at 147) 576 173: b BUILD 577 174: h BINGET 6 578 176: t TUPLE (MARK at 113) 579 177: q BINPUT 10 580 179: h BINGET 10 581 181: K BININT1 5 582 183: e APPENDS (MARK at 5) 583 184: . STOP 584highest protocol among opcodes = 2 585""" 586 587DATA3 = ( 588 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 589 b'builtins\ncomplex\nq\x01G' 590 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02' 591 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff' 592 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f' 593 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq' 594 b'\x04h\x04c__main__\nC\nq\x05)\x81q' 595 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00' 596 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05' 597 b'e.' 598) 599 600# Disassembly of DATA3 601DATA3_DIS = """\ 602 0: \x80 PROTO 3 603 2: ] EMPTY_LIST 604 3: q BINPUT 0 605 5: ( MARK 606 6: K BININT1 0 607 8: K BININT1 1 608 10: G BINFLOAT 2.0 609 19: c GLOBAL 'builtins complex' 610 37: q BINPUT 1 611 39: G BINFLOAT 3.0 612 48: G BINFLOAT 0.0 613 57: \x86 TUPLE2 614 58: q BINPUT 2 615 60: R REDUCE 616 61: q BINPUT 3 617 63: K BININT1 1 618 65: J BININT -1 619 70: K BININT1 255 620 72: J BININT -255 621 77: J BININT -256 622 82: M BININT2 65535 623 85: J BININT -65535 624 90: J BININT -65536 625 95: J BININT 2147483647 626 100: J BININT -2147483647 627 105: J BININT -2147483648 628 110: ( MARK 629 111: X BINUNICODE 'abc' 630 119: q BINPUT 4 631 121: h BINGET 4 632 123: c GLOBAL '__main__ C' 633 135: q BINPUT 5 634 137: ) EMPTY_TUPLE 635 138: \x81 NEWOBJ 636 139: q BINPUT 6 637 141: } EMPTY_DICT 638 142: q BINPUT 7 639 144: ( MARK 640 145: X BINUNICODE 'bar' 641 153: q BINPUT 8 642 155: K BININT1 2 643 157: X BINUNICODE 'foo' 644 165: q BINPUT 9 645 167: K BININT1 1 646 169: u SETITEMS (MARK at 144) 647 170: b BUILD 648 171: h BINGET 6 649 173: t TUPLE (MARK at 110) 650 174: q BINPUT 10 651 176: h BINGET 10 652 178: K BININT1 5 653 180: e APPENDS (MARK at 5) 654 181: . STOP 655highest protocol among opcodes = 2 656""" 657 658DATA4 = ( 659 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@' 660 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07' 661 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G' 662 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK' 663 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ' 664 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(' 665 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c' 666 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c' 667 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.' 668) 669 670# Disassembly of DATA4 671DATA4_DIS = """\ 672 0: \x80 PROTO 4 673 2: \x95 FRAME 168 674 11: ] EMPTY_LIST 675 12: \x94 MEMOIZE 676 13: ( MARK 677 14: K BININT1 0 678 16: K BININT1 1 679 18: G BINFLOAT 2.0 680 27: \x8c SHORT_BINUNICODE 'builtins' 681 37: \x94 MEMOIZE 682 38: \x8c SHORT_BINUNICODE 'complex' 683 47: \x94 MEMOIZE 684 48: \x93 STACK_GLOBAL 685 49: \x94 MEMOIZE 686 50: G BINFLOAT 3.0 687 59: G BINFLOAT 0.0 688 68: \x86 TUPLE2 689 69: \x94 MEMOIZE 690 70: R REDUCE 691 71: \x94 MEMOIZE 692 72: K BININT1 1 693 74: J BININT -1 694 79: K BININT1 255 695 81: J BININT -255 696 86: J BININT -256 697 91: M BININT2 65535 698 94: J BININT -65535 699 99: J BININT -65536 700 104: J BININT 2147483647 701 109: J BININT -2147483647 702 114: J BININT -2147483648 703 119: ( MARK 704 120: \x8c SHORT_BINUNICODE 'abc' 705 125: \x94 MEMOIZE 706 126: h BINGET 6 707 128: \x8c SHORT_BINUNICODE '__main__' 708 138: \x94 MEMOIZE 709 139: \x8c SHORT_BINUNICODE 'C' 710 142: \x94 MEMOIZE 711 143: \x93 STACK_GLOBAL 712 144: \x94 MEMOIZE 713 145: ) EMPTY_TUPLE 714 146: \x81 NEWOBJ 715 147: \x94 MEMOIZE 716 148: } EMPTY_DICT 717 149: \x94 MEMOIZE 718 150: ( MARK 719 151: \x8c SHORT_BINUNICODE 'bar' 720 156: \x94 MEMOIZE 721 157: K BININT1 2 722 159: \x8c SHORT_BINUNICODE 'foo' 723 164: \x94 MEMOIZE 724 165: K BININT1 1 725 167: u SETITEMS (MARK at 150) 726 168: b BUILD 727 169: h BINGET 10 728 171: t TUPLE (MARK at 119) 729 172: \x94 MEMOIZE 730 173: h BINGET 14 731 175: K BININT1 5 732 177: e APPENDS (MARK at 13) 733 178: . STOP 734highest protocol among opcodes = 4 735""" 736 737# set([1,2]) pickled from 2.x with protocol 2 738DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.' 739 740# xrange(5) pickled from 2.x with protocol 2 741DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.' 742 743# a SimpleCookie() object pickled from 2.x with protocol 2 744DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key' 745 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U' 746 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07' 747 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U' 748 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b' 749 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.') 750 751# set([3]) pickled from 2.x with protocol 2 752DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' 753 754python2_exceptions_without_args = ( 755 ArithmeticError, 756 AssertionError, 757 AttributeError, 758 BaseException, 759 BufferError, 760 BytesWarning, 761 DeprecationWarning, 762 EOFError, 763 EnvironmentError, 764 Exception, 765 FloatingPointError, 766 FutureWarning, 767 GeneratorExit, 768 IOError, 769 ImportError, 770 ImportWarning, 771 IndentationError, 772 IndexError, 773 KeyError, 774 KeyboardInterrupt, 775 LookupError, 776 MemoryError, 777 NameError, 778 NotImplementedError, 779 OSError, 780 OverflowError, 781 PendingDeprecationWarning, 782 ReferenceError, 783 RuntimeError, 784 RuntimeWarning, 785 # StandardError is gone in Python 3, we map it to Exception 786 StopIteration, 787 SyntaxError, 788 SyntaxWarning, 789 SystemError, 790 SystemExit, 791 TabError, 792 TypeError, 793 UnboundLocalError, 794 UnicodeError, 795 UnicodeWarning, 796 UserWarning, 797 ValueError, 798 Warning, 799 ZeroDivisionError, 800) 801 802exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.' 803 804# UnicodeEncodeError object pickled from 2.x with protocol 2 805DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n' 806 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01' 807 b'U\x03badq\x03tq\x04Rq\x05.') 808 809 810def create_data(): 811 c = C() 812 c.foo = 1 813 c.bar = 2 814 x = [0, 1, 2.0, 3.0+0j] 815 # Append some integer test cases at cPickle.c's internal size 816 # cutoffs. 817 uint1max = 0xff 818 uint2max = 0xffff 819 int4max = 0x7fffffff 820 x.extend([1, -1, 821 uint1max, -uint1max, -uint1max-1, 822 uint2max, -uint2max, -uint2max-1, 823 int4max, -int4max, -int4max-1]) 824 y = ('abc', 'abc', c, c) 825 x.append(y) 826 x.append(y) 827 x.append(5) 828 return x 829 830 831class AbstractUnpickleTests: 832 # Subclass must define self.loads. 833 834 _testdata = create_data() 835 836 def assert_is_copy(self, obj, objcopy, msg=None): 837 """Utility method to verify if two objects are copies of each others. 838 """ 839 if msg is None: 840 msg = "{!r} is not a copy of {!r}".format(obj, objcopy) 841 self.assertEqual(obj, objcopy, msg=msg) 842 self.assertIs(type(obj), type(objcopy), msg=msg) 843 if hasattr(obj, '__dict__'): 844 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg) 845 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg) 846 if hasattr(obj, '__slots__'): 847 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg) 848 for slot in obj.__slots__: 849 self.assertEqual( 850 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg) 851 self.assertEqual(getattr(obj, slot, None), 852 getattr(objcopy, slot, None), msg=msg) 853 854 def check_unpickling_error(self, errors, data): 855 with self.subTest(data=data), \ 856 self.assertRaises(errors): 857 try: 858 self.loads(data) 859 except BaseException as exc: 860 if support.verbose > 1: 861 print('%-32r - %s: %s' % 862 (data, exc.__class__.__name__, exc)) 863 raise 864 865 def test_load_from_data0(self): 866 self.assert_is_copy(self._testdata, self.loads(DATA0)) 867 868 def test_load_from_data1(self): 869 self.assert_is_copy(self._testdata, self.loads(DATA1)) 870 871 def test_load_from_data2(self): 872 self.assert_is_copy(self._testdata, self.loads(DATA2)) 873 874 def test_load_from_data3(self): 875 self.assert_is_copy(self._testdata, self.loads(DATA3)) 876 877 def test_load_from_data4(self): 878 self.assert_is_copy(self._testdata, self.loads(DATA4)) 879 880 def test_load_classic_instance(self): 881 # See issue5180. Test loading 2.x pickles that 882 # contain an instance of old style class. 883 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 884 xname = X.__name__.encode('ascii') 885 # Protocol 0 (text mode pickle): 886 """ 887 0: ( MARK 888 1: i INST '__main__ X' (MARK at 0) 889 13: p PUT 0 890 16: ( MARK 891 17: d DICT (MARK at 16) 892 18: p PUT 1 893 21: b BUILD 894 22: . STOP 895 """ 896 pickle0 = (b"(i__main__\n" 897 b"X\n" 898 b"p0\n" 899 b"(dp1\nb.").replace(b'X', xname) 900 self.assert_is_copy(X(*args), self.loads(pickle0)) 901 902 # Protocol 1 (binary mode pickle) 903 """ 904 0: ( MARK 905 1: c GLOBAL '__main__ X' 906 13: q BINPUT 0 907 15: o OBJ (MARK at 0) 908 16: q BINPUT 1 909 18: } EMPTY_DICT 910 19: q BINPUT 2 911 21: b BUILD 912 22: . STOP 913 """ 914 pickle1 = (b'(c__main__\n' 915 b'X\n' 916 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 917 self.assert_is_copy(X(*args), self.loads(pickle1)) 918 919 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1) 920 """ 921 0: \x80 PROTO 2 922 2: ( MARK 923 3: c GLOBAL '__main__ X' 924 15: q BINPUT 0 925 17: o OBJ (MARK at 2) 926 18: q BINPUT 1 927 20: } EMPTY_DICT 928 21: q BINPUT 2 929 23: b BUILD 930 24: . STOP 931 """ 932 pickle2 = (b'\x80\x02(c__main__\n' 933 b'X\n' 934 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 935 self.assert_is_copy(X(*args), self.loads(pickle2)) 936 937 def test_maxint64(self): 938 maxint64 = (1 << 63) - 1 939 data = b'I' + str(maxint64).encode("ascii") + b'\n.' 940 got = self.loads(data) 941 self.assert_is_copy(maxint64, got) 942 943 # Try too with a bogus literal. 944 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' 945 self.check_unpickling_error(ValueError, data) 946 947 def test_unpickle_from_2x(self): 948 # Unpickle non-trivial data from Python 2.x. 949 loaded = self.loads(DATA_SET) 950 self.assertEqual(loaded, set([1, 2])) 951 loaded = self.loads(DATA_XRANGE) 952 self.assertEqual(type(loaded), type(range(0))) 953 self.assertEqual(list(loaded), list(range(5))) 954 loaded = self.loads(DATA_COOKIE) 955 self.assertEqual(type(loaded), SimpleCookie) 956 self.assertEqual(list(loaded.keys()), ["key"]) 957 self.assertEqual(loaded["key"].value, "value") 958 959 # Exception objects without arguments pickled from 2.x with protocol 2 960 for exc in python2_exceptions_without_args: 961 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii")) 962 loaded = self.loads(data) 963 self.assertIs(type(loaded), exc) 964 965 # StandardError is mapped to Exception, test that separately 966 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError')) 967 self.assertIs(type(loaded), Exception) 968 969 loaded = self.loads(DATA_UEERR) 970 self.assertIs(type(loaded), UnicodeEncodeError) 971 self.assertEqual(loaded.object, "foo") 972 self.assertEqual(loaded.encoding, "ascii") 973 self.assertEqual(loaded.start, 0) 974 self.assertEqual(loaded.end, 1) 975 self.assertEqual(loaded.reason, "bad") 976 977 def test_load_python2_str_as_bytes(self): 978 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) 979 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.", 980 encoding="bytes"), b'a\x00\xa0') 981 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1) 982 self.assertEqual(self.loads(b'U\x03a\x00\xa0.', 983 encoding="bytes"), b'a\x00\xa0') 984 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2) 985 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.', 986 encoding="bytes"), b'a\x00\xa0') 987 988 def test_load_python2_unicode_as_str(self): 989 # From Python 2: pickle.dumps(u'π', protocol=0) 990 self.assertEqual(self.loads(b'V\\u03c0\n.', 991 encoding='bytes'), 'π') 992 # From Python 2: pickle.dumps(u'π', protocol=1) 993 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.', 994 encoding="bytes"), 'π') 995 # From Python 2: pickle.dumps(u'π', protocol=2) 996 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.', 997 encoding="bytes"), 'π') 998 999 def test_load_long_python2_str_as_bytes(self): 1000 # From Python 2: pickle.dumps('x' * 300, protocol=1) 1001 self.assertEqual(self.loads(pickle.BINSTRING + 1002 struct.pack("<I", 300) + 1003 b'x' * 300 + pickle.STOP, 1004 encoding='bytes'), b'x' * 300) 1005 1006 def test_constants(self): 1007 self.assertIsNone(self.loads(b'N.')) 1008 self.assertIs(self.loads(b'\x88.'), True) 1009 self.assertIs(self.loads(b'\x89.'), False) 1010 self.assertIs(self.loads(b'I01\n.'), True) 1011 self.assertIs(self.loads(b'I00\n.'), False) 1012 1013 def test_empty_bytestring(self): 1014 # issue 11286 1015 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') 1016 self.assertEqual(empty, '') 1017 1018 def test_short_binbytes(self): 1019 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.' 1020 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1021 1022 def test_binbytes(self): 1023 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.' 1024 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1025 1026 @requires_32b 1027 def test_negative_32b_binbytes(self): 1028 # On 32-bit builds, a BINBYTES of 2**31 or more is refused 1029 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' 1030 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1031 dumped) 1032 1033 @requires_32b 1034 def test_negative_32b_binunicode(self): 1035 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused 1036 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' 1037 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1038 dumped) 1039 1040 def test_short_binunicode(self): 1041 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' 1042 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1043 1044 def test_misc_get(self): 1045 self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0') 1046 self.check_unpickling_error(pickle.UnpicklingError, b'jens:') 1047 self.check_unpickling_error(pickle.UnpicklingError, b'hens:') 1048 self.assert_is_copy([(100,), (100,)], 1049 self.loads(b'((Kdtp0\nh\x00l.))')) 1050 1051 def test_binbytes8(self): 1052 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1053 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1054 1055 def test_binunicode8(self): 1056 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1057 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1058 1059 def test_bytearray8(self): 1060 dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.' 1061 self.assertEqual(self.loads(dumped), bytearray(b'xxx')) 1062 1063 @requires_32b 1064 def test_large_32b_binbytes8(self): 1065 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1066 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1067 dumped) 1068 1069 @requires_32b 1070 def test_large_32b_bytearray8(self): 1071 dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1072 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1073 dumped) 1074 1075 @requires_32b 1076 def test_large_32b_binunicode8(self): 1077 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1078 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1079 dumped) 1080 1081 def test_get(self): 1082 pickled = b'((lp100000\ng100000\nt.' 1083 unpickled = self.loads(pickled) 1084 self.assertEqual(unpickled, ([],)*2) 1085 self.assertIs(unpickled[0], unpickled[1]) 1086 1087 def test_binget(self): 1088 pickled = b'(]q\xffh\xfft.' 1089 unpickled = self.loads(pickled) 1090 self.assertEqual(unpickled, ([],)*2) 1091 self.assertIs(unpickled[0], unpickled[1]) 1092 1093 def test_long_binget(self): 1094 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.' 1095 unpickled = self.loads(pickled) 1096 self.assertEqual(unpickled, ([],)*2) 1097 self.assertIs(unpickled[0], unpickled[1]) 1098 1099 def test_dup(self): 1100 pickled = b'((l2t.' 1101 unpickled = self.loads(pickled) 1102 self.assertEqual(unpickled, ([],)*2) 1103 self.assertIs(unpickled[0], unpickled[1]) 1104 1105 def test_negative_put(self): 1106 # Issue #12847 1107 dumped = b'Va\np-1\n.' 1108 self.check_unpickling_error(ValueError, dumped) 1109 1110 @requires_32b 1111 def test_negative_32b_binput(self): 1112 # Issue #12847 1113 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' 1114 self.check_unpickling_error(ValueError, dumped) 1115 1116 def test_badly_escaped_string(self): 1117 self.check_unpickling_error(ValueError, b"S'\\'\n.") 1118 1119 def test_badly_quoted_string(self): 1120 # Issue #17710 1121 badpickles = [b"S'\n.", 1122 b'S"\n.', 1123 b'S\' \n.', 1124 b'S" \n.', 1125 b'S\'"\n.', 1126 b'S"\'\n.', 1127 b"S' ' \n.", 1128 b'S" " \n.', 1129 b"S ''\n.", 1130 b'S ""\n.', 1131 b'S \n.', 1132 b'S\n.', 1133 b'S.'] 1134 for p in badpickles: 1135 self.check_unpickling_error(pickle.UnpicklingError, p) 1136 1137 def test_correctly_quoted_string(self): 1138 goodpickles = [(b"S''\n.", ''), 1139 (b'S""\n.', ''), 1140 (b'S"\\n"\n.', '\n'), 1141 (b"S'\\n'\n.", '\n')] 1142 for p, expected in goodpickles: 1143 self.assertEqual(self.loads(p), expected) 1144 1145 def test_frame_readline(self): 1146 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.' 1147 # 0: \x80 PROTO 4 1148 # 2: \x95 FRAME 5 1149 # 11: I INT 42 1150 # 15: . STOP 1151 self.assertEqual(self.loads(pickled), 42) 1152 1153 def test_compat_unpickle(self): 1154 # xrange(1, 7) 1155 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' 1156 unpickled = self.loads(pickled) 1157 self.assertIs(type(unpickled), range) 1158 self.assertEqual(unpickled, range(1, 7)) 1159 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6]) 1160 # reduce 1161 pickled = b'\x80\x02c__builtin__\nreduce\n.' 1162 self.assertIs(self.loads(pickled), functools.reduce) 1163 # whichdb.whichdb 1164 pickled = b'\x80\x02cwhichdb\nwhichdb\n.' 1165 self.assertIs(self.loads(pickled), dbm.whichdb) 1166 # Exception(), StandardError() 1167 for name in (b'Exception', b'StandardError'): 1168 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.') 1169 unpickled = self.loads(pickled) 1170 self.assertIs(type(unpickled), Exception) 1171 self.assertEqual(str(unpickled), 'ugh') 1172 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2}) 1173 for name in (b'UserDict', b'IterableUserDict'): 1174 pickled = (b'\x80\x02(cUserDict\n' + name + 1175 b'\no}U\x04data}K\x01K\x02ssb.') 1176 unpickled = self.loads(pickled) 1177 self.assertIs(type(unpickled), collections.UserDict) 1178 self.assertEqual(unpickled, collections.UserDict({1: 2})) 1179 1180 def test_bad_reduce(self): 1181 self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) 1182 self.check_unpickling_error(TypeError, b'N)R.') 1183 self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.') 1184 1185 def test_bad_newobj(self): 1186 error = (pickle.UnpicklingError, TypeError) 1187 self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0) 1188 self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.') 1189 self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.') 1190 1191 def test_bad_newobj_ex(self): 1192 error = (pickle.UnpicklingError, TypeError) 1193 self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0) 1194 self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.') 1195 self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') 1196 self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') 1197 1198 def test_bad_stack(self): 1199 badpickles = [ 1200 b'.', # STOP 1201 b'0', # POP 1202 b'1', # POP_MARK 1203 b'2', # DUP 1204 b'(2', 1205 b'R', # REDUCE 1206 b')R', 1207 b'a', # APPEND 1208 b'Na', 1209 b'b', # BUILD 1210 b'Nb', 1211 b'd', # DICT 1212 b'e', # APPENDS 1213 b'(e', 1214 b'ibuiltins\nlist\n', # INST 1215 b'l', # LIST 1216 b'o', # OBJ 1217 b'(o', 1218 b'p1\n', # PUT 1219 b'q\x00', # BINPUT 1220 b'r\x00\x00\x00\x00', # LONG_BINPUT 1221 b's', # SETITEM 1222 b'Ns', 1223 b'NNs', 1224 b't', # TUPLE 1225 b'u', # SETITEMS 1226 b'(u', 1227 b'}(Nu', 1228 b'\x81', # NEWOBJ 1229 b')\x81', 1230 b'\x85', # TUPLE1 1231 b'\x86', # TUPLE2 1232 b'N\x86', 1233 b'\x87', # TUPLE3 1234 b'N\x87', 1235 b'NN\x87', 1236 b'\x90', # ADDITEMS 1237 b'(\x90', 1238 b'\x91', # FROZENSET 1239 b'\x92', # NEWOBJ_EX 1240 b')}\x92', 1241 b'\x93', # STACK_GLOBAL 1242 b'Vlist\n\x93', 1243 b'\x94', # MEMOIZE 1244 ] 1245 for p in badpickles: 1246 self.check_unpickling_error(self.bad_stack_errors, p) 1247 1248 def test_bad_mark(self): 1249 badpickles = [ 1250 b'N(.', # STOP 1251 b'N(2', # DUP 1252 b'cbuiltins\nlist\n)(R', # REDUCE 1253 b'cbuiltins\nlist\n()R', 1254 b']N(a', # APPEND 1255 # BUILD 1256 b'cbuiltins\nValueError\n)R}(b', 1257 b'cbuiltins\nValueError\n)R(}b', 1258 b'(Nd', # DICT 1259 b'N(p1\n', # PUT 1260 b'N(q\x00', # BINPUT 1261 b'N(r\x00\x00\x00\x00', # LONG_BINPUT 1262 b'}NN(s', # SETITEM 1263 b'}N(Ns', 1264 b'}(NNs', 1265 b'}((u', # SETITEMS 1266 b'cbuiltins\nlist\n)(\x81', # NEWOBJ 1267 b'cbuiltins\nlist\n()\x81', 1268 b'N(\x85', # TUPLE1 1269 b'NN(\x86', # TUPLE2 1270 b'N(N\x86', 1271 b'NNN(\x87', # TUPLE3 1272 b'NN(N\x87', 1273 b'N(NN\x87', 1274 b']((\x90', # ADDITEMS 1275 # NEWOBJ_EX 1276 b'cbuiltins\nlist\n)}(\x92', 1277 b'cbuiltins\nlist\n)(}\x92', 1278 b'cbuiltins\nlist\n()}\x92', 1279 # STACK_GLOBAL 1280 b'Vbuiltins\n(Vlist\n\x93', 1281 b'Vbuiltins\nVlist\n(\x93', 1282 b'N(\x94', # MEMOIZE 1283 ] 1284 for p in badpickles: 1285 self.check_unpickling_error(self.bad_stack_errors, p) 1286 1287 def test_truncated_data(self): 1288 self.check_unpickling_error(EOFError, b'') 1289 self.check_unpickling_error(EOFError, b'N') 1290 badpickles = [ 1291 b'B', # BINBYTES 1292 b'B\x03\x00\x00', 1293 b'B\x03\x00\x00\x00', 1294 b'B\x03\x00\x00\x00ab', 1295 b'C', # SHORT_BINBYTES 1296 b'C\x03', 1297 b'C\x03ab', 1298 b'F', # FLOAT 1299 b'F0.0', 1300 b'F0.00', 1301 b'G', # BINFLOAT 1302 b'G\x00\x00\x00\x00\x00\x00\x00', 1303 b'I', # INT 1304 b'I0', 1305 b'J', # BININT 1306 b'J\x00\x00\x00', 1307 b'K', # BININT1 1308 b'L', # LONG 1309 b'L0', 1310 b'L10', 1311 b'L0L', 1312 b'L10L', 1313 b'M', # BININT2 1314 b'M\x00', 1315 # b'P', # PERSID 1316 # b'Pabc', 1317 b'S', # STRING 1318 b"S'abc'", 1319 b'T', # BINSTRING 1320 b'T\x03\x00\x00', 1321 b'T\x03\x00\x00\x00', 1322 b'T\x03\x00\x00\x00ab', 1323 b'U', # SHORT_BINSTRING 1324 b'U\x03', 1325 b'U\x03ab', 1326 b'V', # UNICODE 1327 b'Vabc', 1328 b'X', # BINUNICODE 1329 b'X\x03\x00\x00', 1330 b'X\x03\x00\x00\x00', 1331 b'X\x03\x00\x00\x00ab', 1332 b'(c', # GLOBAL 1333 b'(cbuiltins', 1334 b'(cbuiltins\n', 1335 b'(cbuiltins\nlist', 1336 b'Ng', # GET 1337 b'Ng0', 1338 b'(i', # INST 1339 b'(ibuiltins', 1340 b'(ibuiltins\n', 1341 b'(ibuiltins\nlist', 1342 b'Nh', # BINGET 1343 b'Nj', # LONG_BINGET 1344 b'Nj\x00\x00\x00', 1345 b'Np', # PUT 1346 b'Np0', 1347 b'Nq', # BINPUT 1348 b'Nr', # LONG_BINPUT 1349 b'Nr\x00\x00\x00', 1350 b'\x80', # PROTO 1351 b'\x82', # EXT1 1352 b'\x83', # EXT2 1353 b'\x84\x01', 1354 b'\x84', # EXT4 1355 b'\x84\x01\x00\x00', 1356 b'\x8a', # LONG1 1357 b'\x8b', # LONG4 1358 b'\x8b\x00\x00\x00', 1359 b'\x8c', # SHORT_BINUNICODE 1360 b'\x8c\x03', 1361 b'\x8c\x03ab', 1362 b'\x8d', # BINUNICODE8 1363 b'\x8d\x03\x00\x00\x00\x00\x00\x00', 1364 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', 1365 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', 1366 b'\x8e', # BINBYTES8 1367 b'\x8e\x03\x00\x00\x00\x00\x00\x00', 1368 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', 1369 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', 1370 b'\x96', # BYTEARRAY8 1371 b'\x96\x03\x00\x00\x00\x00\x00\x00', 1372 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00', 1373 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab', 1374 b'\x95', # FRAME 1375 b'\x95\x02\x00\x00\x00\x00\x00\x00', 1376 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', 1377 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', 1378 ] 1379 for p in badpickles: 1380 self.check_unpickling_error(self.truncated_errors, p) 1381 1382 @threading_helper.reap_threads 1383 def test_unpickle_module_race(self): 1384 # https://bugs.python.org/issue34572 1385 locker_module = dedent(""" 1386 import threading 1387 barrier = threading.Barrier(2) 1388 """) 1389 locking_import_module = dedent(""" 1390 import locker 1391 locker.barrier.wait() 1392 class ToBeUnpickled(object): 1393 pass 1394 """) 1395 1396 os.mkdir(TESTFN) 1397 self.addCleanup(shutil.rmtree, TESTFN) 1398 sys.path.insert(0, TESTFN) 1399 self.addCleanup(sys.path.remove, TESTFN) 1400 with open(os.path.join(TESTFN, "locker.py"), "wb") as f: 1401 f.write(locker_module.encode('utf-8')) 1402 with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f: 1403 f.write(locking_import_module.encode('utf-8')) 1404 self.addCleanup(forget, "locker") 1405 self.addCleanup(forget, "locking_import") 1406 1407 import locker 1408 1409 pickle_bytes = ( 1410 b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.') 1411 1412 # Then try to unpickle two of these simultaneously 1413 # One of them will cause the module import, and we want it to block 1414 # until the other one either: 1415 # - fails (before the patch for this issue) 1416 # - blocks on the import lock for the module, as it should 1417 results = [] 1418 barrier = threading.Barrier(3) 1419 def t(): 1420 # This ensures the threads have all started 1421 # presumably barrier release is faster than thread startup 1422 barrier.wait() 1423 results.append(pickle.loads(pickle_bytes)) 1424 1425 t1 = threading.Thread(target=t) 1426 t2 = threading.Thread(target=t) 1427 t1.start() 1428 t2.start() 1429 1430 barrier.wait() 1431 # could have delay here 1432 locker.barrier.wait() 1433 1434 t1.join() 1435 t2.join() 1436 1437 from locking_import import ToBeUnpickled 1438 self.assertEqual( 1439 [type(x) for x in results], 1440 [ToBeUnpickled] * 2) 1441 1442 1443 1444class AbstractPickleTests: 1445 # Subclass must define self.dumps, self.loads. 1446 1447 optimized = False 1448 1449 _testdata = AbstractUnpickleTests._testdata 1450 1451 def setUp(self): 1452 pass 1453 1454 assert_is_copy = AbstractUnpickleTests.assert_is_copy 1455 1456 def test_misc(self): 1457 # test various datatypes not tested by testdata 1458 for proto in protocols: 1459 x = myint(4) 1460 s = self.dumps(x, proto) 1461 y = self.loads(s) 1462 self.assert_is_copy(x, y) 1463 1464 x = (1, ()) 1465 s = self.dumps(x, proto) 1466 y = self.loads(s) 1467 self.assert_is_copy(x, y) 1468 1469 x = initarg(1, x) 1470 s = self.dumps(x, proto) 1471 y = self.loads(s) 1472 self.assert_is_copy(x, y) 1473 1474 # XXX test __reduce__ protocol? 1475 1476 def test_roundtrip_equality(self): 1477 expected = self._testdata 1478 for proto in protocols: 1479 s = self.dumps(expected, proto) 1480 got = self.loads(s) 1481 self.assert_is_copy(expected, got) 1482 1483 # There are gratuitous differences between pickles produced by 1484 # pickle and cPickle, largely because cPickle starts PUT indices at 1485 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 1486 # there's a comment with an exclamation point there whose meaning 1487 # is a mystery. cPickle also suppresses PUT for objects with a refcount 1488 # of 1. 1489 def dont_test_disassembly(self): 1490 from io import StringIO 1491 from pickletools import dis 1492 1493 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 1494 s = self.dumps(self._testdata, proto) 1495 filelike = StringIO() 1496 dis(s, out=filelike) 1497 got = filelike.getvalue() 1498 self.assertEqual(expected, got) 1499 1500 def _test_recursive_list(self, cls, aslist=identity, minprotocol=0): 1501 # List containing itself. 1502 l = cls() 1503 l.append(l) 1504 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1505 s = self.dumps(l, proto) 1506 x = self.loads(s) 1507 self.assertIsInstance(x, cls) 1508 y = aslist(x) 1509 self.assertEqual(len(y), 1) 1510 self.assertIs(y[0], x) 1511 1512 def test_recursive_list(self): 1513 self._test_recursive_list(list) 1514 1515 def test_recursive_list_subclass(self): 1516 self._test_recursive_list(MyList, minprotocol=2) 1517 1518 def test_recursive_list_like(self): 1519 self._test_recursive_list(REX_six, aslist=lambda x: x.items) 1520 1521 def _test_recursive_tuple_and_list(self, cls, aslist=identity, minprotocol=0): 1522 # Tuple containing a list containing the original tuple. 1523 t = (cls(),) 1524 t[0].append(t) 1525 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1526 s = self.dumps(t, proto) 1527 x = self.loads(s) 1528 self.assertIsInstance(x, tuple) 1529 self.assertEqual(len(x), 1) 1530 self.assertIsInstance(x[0], cls) 1531 y = aslist(x[0]) 1532 self.assertEqual(len(y), 1) 1533 self.assertIs(y[0], x) 1534 1535 # List containing a tuple containing the original list. 1536 t, = t 1537 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1538 s = self.dumps(t, proto) 1539 x = self.loads(s) 1540 self.assertIsInstance(x, cls) 1541 y = aslist(x) 1542 self.assertEqual(len(y), 1) 1543 self.assertIsInstance(y[0], tuple) 1544 self.assertEqual(len(y[0]), 1) 1545 self.assertIs(y[0][0], x) 1546 1547 def test_recursive_tuple_and_list(self): 1548 self._test_recursive_tuple_and_list(list) 1549 1550 def test_recursive_tuple_and_list_subclass(self): 1551 self._test_recursive_tuple_and_list(MyList, minprotocol=2) 1552 1553 def test_recursive_tuple_and_list_like(self): 1554 self._test_recursive_tuple_and_list(REX_six, aslist=lambda x: x.items) 1555 1556 def _test_recursive_dict(self, cls, asdict=identity, minprotocol=0): 1557 # Dict containing itself. 1558 d = cls() 1559 d[1] = d 1560 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1561 s = self.dumps(d, proto) 1562 x = self.loads(s) 1563 self.assertIsInstance(x, cls) 1564 y = asdict(x) 1565 self.assertEqual(list(y.keys()), [1]) 1566 self.assertIs(y[1], x) 1567 1568 def test_recursive_dict(self): 1569 self._test_recursive_dict(dict) 1570 1571 def test_recursive_dict_subclass(self): 1572 self._test_recursive_dict(MyDict, minprotocol=2) 1573 1574 def test_recursive_dict_like(self): 1575 self._test_recursive_dict(REX_seven, asdict=lambda x: x.table) 1576 1577 def _test_recursive_tuple_and_dict(self, cls, asdict=identity, minprotocol=0): 1578 # Tuple containing a dict containing the original tuple. 1579 t = (cls(),) 1580 t[0][1] = t 1581 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1582 s = self.dumps(t, proto) 1583 x = self.loads(s) 1584 self.assertIsInstance(x, tuple) 1585 self.assertEqual(len(x), 1) 1586 self.assertIsInstance(x[0], cls) 1587 y = asdict(x[0]) 1588 self.assertEqual(list(y), [1]) 1589 self.assertIs(y[1], x) 1590 1591 # Dict containing a tuple containing the original dict. 1592 t, = t 1593 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1594 s = self.dumps(t, proto) 1595 x = self.loads(s) 1596 self.assertIsInstance(x, cls) 1597 y = asdict(x) 1598 self.assertEqual(list(y), [1]) 1599 self.assertIsInstance(y[1], tuple) 1600 self.assertEqual(len(y[1]), 1) 1601 self.assertIs(y[1][0], x) 1602 1603 def test_recursive_tuple_and_dict(self): 1604 self._test_recursive_tuple_and_dict(dict) 1605 1606 def test_recursive_tuple_and_dict_subclass(self): 1607 self._test_recursive_tuple_and_dict(MyDict, minprotocol=2) 1608 1609 def test_recursive_tuple_and_dict_like(self): 1610 self._test_recursive_tuple_and_dict(REX_seven, asdict=lambda x: x.table) 1611 1612 def _test_recursive_dict_key(self, cls, asdict=identity, minprotocol=0): 1613 # Dict containing an immutable object (as key) containing the original 1614 # dict. 1615 d = cls() 1616 d[K(d)] = 1 1617 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1618 s = self.dumps(d, proto) 1619 x = self.loads(s) 1620 self.assertIsInstance(x, cls) 1621 y = asdict(x) 1622 self.assertEqual(len(y.keys()), 1) 1623 self.assertIsInstance(list(y.keys())[0], K) 1624 self.assertIs(list(y.keys())[0].value, x) 1625 1626 def test_recursive_dict_key(self): 1627 self._test_recursive_dict_key(dict) 1628 1629 def test_recursive_dict_subclass_key(self): 1630 self._test_recursive_dict_key(MyDict, minprotocol=2) 1631 1632 def test_recursive_dict_like_key(self): 1633 self._test_recursive_dict_key(REX_seven, asdict=lambda x: x.table) 1634 1635 def _test_recursive_tuple_and_dict_key(self, cls, asdict=identity, minprotocol=0): 1636 # Tuple containing a dict containing an immutable object (as key) 1637 # containing the original tuple. 1638 t = (cls(),) 1639 t[0][K(t)] = 1 1640 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1641 s = self.dumps(t, proto) 1642 x = self.loads(s) 1643 self.assertIsInstance(x, tuple) 1644 self.assertEqual(len(x), 1) 1645 self.assertIsInstance(x[0], cls) 1646 y = asdict(x[0]) 1647 self.assertEqual(len(y), 1) 1648 self.assertIsInstance(list(y.keys())[0], K) 1649 self.assertIs(list(y.keys())[0].value, x) 1650 1651 # Dict containing an immutable object (as key) containing a tuple 1652 # containing the original dict. 1653 t, = t 1654 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1655 s = self.dumps(t, proto) 1656 x = self.loads(s) 1657 self.assertIsInstance(x, cls) 1658 y = asdict(x) 1659 self.assertEqual(len(y), 1) 1660 self.assertIsInstance(list(y.keys())[0], K) 1661 self.assertIs(list(y.keys())[0].value[0], x) 1662 1663 def test_recursive_tuple_and_dict_key(self): 1664 self._test_recursive_tuple_and_dict_key(dict) 1665 1666 def test_recursive_tuple_and_dict_subclass_key(self): 1667 self._test_recursive_tuple_and_dict_key(MyDict, minprotocol=2) 1668 1669 def test_recursive_tuple_and_dict_like_key(self): 1670 self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table) 1671 1672 def test_recursive_set(self): 1673 # Set containing an immutable object containing the original set. 1674 y = set() 1675 y.add(K(y)) 1676 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1677 s = self.dumps(y, proto) 1678 x = self.loads(s) 1679 self.assertIsInstance(x, set) 1680 self.assertEqual(len(x), 1) 1681 self.assertIsInstance(list(x)[0], K) 1682 self.assertIs(list(x)[0].value, x) 1683 1684 # Immutable object containing a set containing the original object. 1685 y, = y 1686 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1687 s = self.dumps(y, proto) 1688 x = self.loads(s) 1689 self.assertIsInstance(x, K) 1690 self.assertIsInstance(x.value, set) 1691 self.assertEqual(len(x.value), 1) 1692 self.assertIs(list(x.value)[0], x) 1693 1694 def test_recursive_inst(self): 1695 # Mutable object containing itself. 1696 i = Object() 1697 i.attr = i 1698 for proto in protocols: 1699 s = self.dumps(i, proto) 1700 x = self.loads(s) 1701 self.assertIsInstance(x, Object) 1702 self.assertEqual(dir(x), dir(i)) 1703 self.assertIs(x.attr, x) 1704 1705 def test_recursive_multi(self): 1706 l = [] 1707 d = {1:l} 1708 i = Object() 1709 i.attr = d 1710 l.append(i) 1711 for proto in protocols: 1712 s = self.dumps(l, proto) 1713 x = self.loads(s) 1714 self.assertIsInstance(x, list) 1715 self.assertEqual(len(x), 1) 1716 self.assertEqual(dir(x[0]), dir(i)) 1717 self.assertEqual(list(x[0].attr.keys()), [1]) 1718 self.assertIs(x[0].attr[1], x) 1719 1720 def _test_recursive_collection_and_inst(self, factory): 1721 # Mutable object containing a collection containing the original 1722 # object. 1723 o = Object() 1724 o.attr = factory([o]) 1725 t = type(o.attr) 1726 for proto in protocols: 1727 s = self.dumps(o, proto) 1728 x = self.loads(s) 1729 self.assertIsInstance(x.attr, t) 1730 self.assertEqual(len(x.attr), 1) 1731 self.assertIsInstance(list(x.attr)[0], Object) 1732 self.assertIs(list(x.attr)[0], x) 1733 1734 # Collection containing a mutable object containing the original 1735 # collection. 1736 o = o.attr 1737 for proto in protocols: 1738 s = self.dumps(o, proto) 1739 x = self.loads(s) 1740 self.assertIsInstance(x, t) 1741 self.assertEqual(len(x), 1) 1742 self.assertIsInstance(list(x)[0], Object) 1743 self.assertIs(list(x)[0].attr, x) 1744 1745 def test_recursive_list_and_inst(self): 1746 self._test_recursive_collection_and_inst(list) 1747 1748 def test_recursive_tuple_and_inst(self): 1749 self._test_recursive_collection_and_inst(tuple) 1750 1751 def test_recursive_dict_and_inst(self): 1752 self._test_recursive_collection_and_inst(dict.fromkeys) 1753 1754 def test_recursive_set_and_inst(self): 1755 self._test_recursive_collection_and_inst(set) 1756 1757 def test_recursive_frozenset_and_inst(self): 1758 self._test_recursive_collection_and_inst(frozenset) 1759 1760 def test_recursive_list_subclass_and_inst(self): 1761 self._test_recursive_collection_and_inst(MyList) 1762 1763 def test_recursive_tuple_subclass_and_inst(self): 1764 self._test_recursive_collection_and_inst(MyTuple) 1765 1766 def test_recursive_dict_subclass_and_inst(self): 1767 self._test_recursive_collection_and_inst(MyDict.fromkeys) 1768 1769 def test_recursive_set_subclass_and_inst(self): 1770 self._test_recursive_collection_and_inst(MySet) 1771 1772 def test_recursive_frozenset_subclass_and_inst(self): 1773 self._test_recursive_collection_and_inst(MyFrozenSet) 1774 1775 def test_recursive_inst_state(self): 1776 # Mutable object containing itself. 1777 y = REX_state() 1778 y.state = y 1779 for proto in protocols: 1780 s = self.dumps(y, proto) 1781 x = self.loads(s) 1782 self.assertIsInstance(x, REX_state) 1783 self.assertIs(x.state, x) 1784 1785 def test_recursive_tuple_and_inst_state(self): 1786 # Tuple containing a mutable object containing the original tuple. 1787 t = (REX_state(),) 1788 t[0].state = t 1789 for proto in protocols: 1790 s = self.dumps(t, proto) 1791 x = self.loads(s) 1792 self.assertIsInstance(x, tuple) 1793 self.assertEqual(len(x), 1) 1794 self.assertIsInstance(x[0], REX_state) 1795 self.assertIs(x[0].state, x) 1796 1797 # Mutable object containing a tuple containing the object. 1798 t, = t 1799 for proto in protocols: 1800 s = self.dumps(t, proto) 1801 x = self.loads(s) 1802 self.assertIsInstance(x, REX_state) 1803 self.assertIsInstance(x.state, tuple) 1804 self.assertEqual(len(x.state), 1) 1805 self.assertIs(x.state[0], x) 1806 1807 def test_unicode(self): 1808 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', 1809 '<\\>', '<\\\U00012345>', 1810 # surrogates 1811 '<\udc80>'] 1812 for proto in protocols: 1813 for u in endcases: 1814 p = self.dumps(u, proto) 1815 u2 = self.loads(p) 1816 self.assert_is_copy(u, u2) 1817 1818 def test_unicode_high_plane(self): 1819 t = '\U00012345' 1820 for proto in protocols: 1821 p = self.dumps(t, proto) 1822 t2 = self.loads(p) 1823 self.assert_is_copy(t, t2) 1824 1825 def test_bytes(self): 1826 for proto in protocols: 1827 for s in b'', b'xyz', b'xyz'*100: 1828 p = self.dumps(s, proto) 1829 self.assert_is_copy(s, self.loads(p)) 1830 for s in [bytes([i]) for i in range(256)]: 1831 p = self.dumps(s, proto) 1832 self.assert_is_copy(s, self.loads(p)) 1833 for s in [bytes([i, i]) for i in range(256)]: 1834 p = self.dumps(s, proto) 1835 self.assert_is_copy(s, self.loads(p)) 1836 1837 def test_bytearray(self): 1838 for proto in protocols: 1839 for s in b'', b'xyz', b'xyz'*100: 1840 b = bytearray(s) 1841 p = self.dumps(b, proto) 1842 bb = self.loads(p) 1843 self.assertIsNot(bb, b) 1844 self.assert_is_copy(b, bb) 1845 if proto <= 3: 1846 # bytearray is serialized using a global reference 1847 self.assertIn(b'bytearray', p) 1848 self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p)) 1849 elif proto == 4: 1850 self.assertIn(b'bytearray', p) 1851 self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p)) 1852 elif proto == 5: 1853 self.assertNotIn(b'bytearray', p) 1854 self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) 1855 1856 def test_bytearray_memoization_bug(self): 1857 for proto in protocols: 1858 for s in b'', b'xyz', b'xyz'*100: 1859 b = bytearray(s) 1860 p = self.dumps((b, b), proto) 1861 b1, b2 = self.loads(p) 1862 self.assertIs(b1, b2) 1863 1864 def test_ints(self): 1865 for proto in protocols: 1866 n = sys.maxsize 1867 while n: 1868 for expected in (-n, n): 1869 s = self.dumps(expected, proto) 1870 n2 = self.loads(s) 1871 self.assert_is_copy(expected, n2) 1872 n = n >> 1 1873 1874 def test_long(self): 1875 for proto in protocols: 1876 # 256 bytes is where LONG4 begins. 1877 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 1878 nbase = 1 << nbits 1879 for npos in nbase-1, nbase, nbase+1: 1880 for n in npos, -npos: 1881 pickle = self.dumps(n, proto) 1882 got = self.loads(pickle) 1883 self.assert_is_copy(n, got) 1884 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 1885 # bother with those. 1886 nbase = int("deadbeeffeedface", 16) 1887 nbase += nbase << 1000000 1888 for n in nbase, -nbase: 1889 p = self.dumps(n, 2) 1890 got = self.loads(p) 1891 # assert_is_copy is very expensive here as it precomputes 1892 # a failure message by computing the repr() of n and got, 1893 # we just do the check ourselves. 1894 self.assertIs(type(got), int) 1895 self.assertEqual(n, got) 1896 1897 def test_float(self): 1898 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 1899 3.14, 263.44582062374053, 6.022e23, 1e30] 1900 test_values = test_values + [-x for x in test_values] 1901 for proto in protocols: 1902 for value in test_values: 1903 pickle = self.dumps(value, proto) 1904 got = self.loads(pickle) 1905 self.assert_is_copy(value, got) 1906 1907 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 1908 def test_float_format(self): 1909 # make sure that floats are formatted locale independent with proto 0 1910 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') 1911 1912 def test_reduce(self): 1913 for proto in protocols: 1914 inst = AAA() 1915 dumped = self.dumps(inst, proto) 1916 loaded = self.loads(dumped) 1917 self.assertEqual(loaded, REDUCE_A) 1918 1919 def test_getinitargs(self): 1920 for proto in protocols: 1921 inst = initarg(1, 2) 1922 dumped = self.dumps(inst, proto) 1923 loaded = self.loads(dumped) 1924 self.assert_is_copy(inst, loaded) 1925 1926 def test_metaclass(self): 1927 a = use_metaclass() 1928 for proto in protocols: 1929 s = self.dumps(a, proto) 1930 b = self.loads(s) 1931 self.assertEqual(a.__class__, b.__class__) 1932 1933 def test_dynamic_class(self): 1934 a = create_dynamic_class("my_dynamic_class", (object,)) 1935 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 1936 for proto in protocols: 1937 s = self.dumps(a, proto) 1938 b = self.loads(s) 1939 self.assertEqual(a, b) 1940 self.assertIs(type(a), type(b)) 1941 1942 def test_structseq(self): 1943 import time 1944 import os 1945 1946 t = time.localtime() 1947 for proto in protocols: 1948 s = self.dumps(t, proto) 1949 u = self.loads(s) 1950 self.assert_is_copy(t, u) 1951 t = os.stat(os.curdir) 1952 s = self.dumps(t, proto) 1953 u = self.loads(s) 1954 self.assert_is_copy(t, u) 1955 if hasattr(os, "statvfs"): 1956 t = os.statvfs(os.curdir) 1957 s = self.dumps(t, proto) 1958 u = self.loads(s) 1959 self.assert_is_copy(t, u) 1960 1961 def test_ellipsis(self): 1962 for proto in protocols: 1963 s = self.dumps(..., proto) 1964 u = self.loads(s) 1965 self.assertIs(..., u) 1966 1967 def test_notimplemented(self): 1968 for proto in protocols: 1969 s = self.dumps(NotImplemented, proto) 1970 u = self.loads(s) 1971 self.assertIs(NotImplemented, u) 1972 1973 def test_singleton_types(self): 1974 # Issue #6477: Test that types of built-in singletons can be pickled. 1975 singletons = [None, ..., NotImplemented] 1976 for singleton in singletons: 1977 for proto in protocols: 1978 s = self.dumps(type(singleton), proto) 1979 u = self.loads(s) 1980 self.assertIs(type(singleton), u) 1981 1982 # Tests for protocol 2 1983 1984 def test_proto(self): 1985 for proto in protocols: 1986 pickled = self.dumps(None, proto) 1987 if proto >= 2: 1988 proto_header = pickle.PROTO + bytes([proto]) 1989 self.assertTrue(pickled.startswith(proto_header)) 1990 else: 1991 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) 1992 1993 oob = protocols[-1] + 1 # a future protocol 1994 build_none = pickle.NONE + pickle.STOP 1995 badpickle = pickle.PROTO + bytes([oob]) + build_none 1996 try: 1997 self.loads(badpickle) 1998 except ValueError as err: 1999 self.assertIn("unsupported pickle protocol", str(err)) 2000 else: 2001 self.fail("expected bad protocol number to raise ValueError") 2002 2003 def test_long1(self): 2004 x = 12345678910111213141516178920 2005 for proto in protocols: 2006 s = self.dumps(x, proto) 2007 y = self.loads(s) 2008 self.assert_is_copy(x, y) 2009 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 2010 2011 def test_long4(self): 2012 x = 12345678910111213141516178920 << (256*8) 2013 for proto in protocols: 2014 s = self.dumps(x, proto) 2015 y = self.loads(s) 2016 self.assert_is_copy(x, y) 2017 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 2018 2019 def test_short_tuples(self): 2020 # Map (proto, len(tuple)) to expected opcode. 2021 expected_opcode = {(0, 0): pickle.TUPLE, 2022 (0, 1): pickle.TUPLE, 2023 (0, 2): pickle.TUPLE, 2024 (0, 3): pickle.TUPLE, 2025 (0, 4): pickle.TUPLE, 2026 2027 (1, 0): pickle.EMPTY_TUPLE, 2028 (1, 1): pickle.TUPLE, 2029 (1, 2): pickle.TUPLE, 2030 (1, 3): pickle.TUPLE, 2031 (1, 4): pickle.TUPLE, 2032 2033 (2, 0): pickle.EMPTY_TUPLE, 2034 (2, 1): pickle.TUPLE1, 2035 (2, 2): pickle.TUPLE2, 2036 (2, 3): pickle.TUPLE3, 2037 (2, 4): pickle.TUPLE, 2038 2039 (3, 0): pickle.EMPTY_TUPLE, 2040 (3, 1): pickle.TUPLE1, 2041 (3, 2): pickle.TUPLE2, 2042 (3, 3): pickle.TUPLE3, 2043 (3, 4): pickle.TUPLE, 2044 } 2045 a = () 2046 b = (1,) 2047 c = (1, 2) 2048 d = (1, 2, 3) 2049 e = (1, 2, 3, 4) 2050 for proto in protocols: 2051 for x in a, b, c, d, e: 2052 s = self.dumps(x, proto) 2053 y = self.loads(s) 2054 self.assert_is_copy(x, y) 2055 expected = expected_opcode[min(proto, 3), len(x)] 2056 self.assertTrue(opcode_in_pickle(expected, s)) 2057 2058 def test_singletons(self): 2059 # Map (proto, singleton) to expected opcode. 2060 expected_opcode = {(0, None): pickle.NONE, 2061 (1, None): pickle.NONE, 2062 (2, None): pickle.NONE, 2063 (3, None): pickle.NONE, 2064 2065 (0, True): pickle.INT, 2066 (1, True): pickle.INT, 2067 (2, True): pickle.NEWTRUE, 2068 (3, True): pickle.NEWTRUE, 2069 2070 (0, False): pickle.INT, 2071 (1, False): pickle.INT, 2072 (2, False): pickle.NEWFALSE, 2073 (3, False): pickle.NEWFALSE, 2074 } 2075 for proto in protocols: 2076 for x in None, False, True: 2077 s = self.dumps(x, proto) 2078 y = self.loads(s) 2079 self.assertTrue(x is y, (proto, x, s, y)) 2080 expected = expected_opcode[min(proto, 3), x] 2081 self.assertTrue(opcode_in_pickle(expected, s)) 2082 2083 def test_newobj_tuple(self): 2084 x = MyTuple([1, 2, 3]) 2085 x.foo = 42 2086 x.bar = "hello" 2087 for proto in protocols: 2088 s = self.dumps(x, proto) 2089 y = self.loads(s) 2090 self.assert_is_copy(x, y) 2091 2092 def test_newobj_list(self): 2093 x = MyList([1, 2, 3]) 2094 x.foo = 42 2095 x.bar = "hello" 2096 for proto in protocols: 2097 s = self.dumps(x, proto) 2098 y = self.loads(s) 2099 self.assert_is_copy(x, y) 2100 2101 def test_newobj_generic(self): 2102 for proto in protocols: 2103 for C in myclasses: 2104 B = C.__base__ 2105 x = C(C.sample) 2106 x.foo = 42 2107 s = self.dumps(x, proto) 2108 y = self.loads(s) 2109 detail = (proto, C, B, x, y, type(y)) 2110 self.assert_is_copy(x, y) # XXX revisit 2111 self.assertEqual(B(x), B(y), detail) 2112 self.assertEqual(x.__dict__, y.__dict__, detail) 2113 2114 def test_newobj_proxies(self): 2115 # NEWOBJ should use the __class__ rather than the raw type 2116 classes = myclasses[:] 2117 # Cannot create weakproxies to these classes 2118 for c in (MyInt, MyTuple): 2119 classes.remove(c) 2120 for proto in protocols: 2121 for C in classes: 2122 B = C.__base__ 2123 x = C(C.sample) 2124 x.foo = 42 2125 p = weakref.proxy(x) 2126 s = self.dumps(p, proto) 2127 y = self.loads(s) 2128 self.assertEqual(type(y), type(x)) # rather than type(p) 2129 detail = (proto, C, B, x, y, type(y)) 2130 self.assertEqual(B(x), B(y), detail) 2131 self.assertEqual(x.__dict__, y.__dict__, detail) 2132 2133 def test_newobj_overridden_new(self): 2134 # Test that Python class with C implemented __new__ is pickleable 2135 for proto in protocols: 2136 x = MyIntWithNew2(1) 2137 x.foo = 42 2138 s = self.dumps(x, proto) 2139 y = self.loads(s) 2140 self.assertIs(type(y), MyIntWithNew2) 2141 self.assertEqual(int(y), 1) 2142 self.assertEqual(y.foo, 42) 2143 2144 def test_newobj_not_class(self): 2145 # Issue 24552 2146 global SimpleNewObj 2147 save = SimpleNewObj 2148 o = SimpleNewObj.__new__(SimpleNewObj) 2149 b = self.dumps(o, 4) 2150 try: 2151 SimpleNewObj = 42 2152 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) 2153 finally: 2154 SimpleNewObj = save 2155 2156 # Register a type with copyreg, with extension code extcode. Pickle 2157 # an object of that type. Check that the resulting pickle uses opcode 2158 # (EXT[124]) under proto 2, and not in proto 1. 2159 2160 def produce_global_ext(self, extcode, opcode): 2161 e = ExtensionSaver(extcode) 2162 try: 2163 copyreg.add_extension(__name__, "MyList", extcode) 2164 x = MyList([1, 2, 3]) 2165 x.foo = 42 2166 x.bar = "hello" 2167 2168 # Dump using protocol 1 for comparison. 2169 s1 = self.dumps(x, 1) 2170 self.assertIn(__name__.encode("utf-8"), s1) 2171 self.assertIn(b"MyList", s1) 2172 self.assertFalse(opcode_in_pickle(opcode, s1)) 2173 2174 y = self.loads(s1) 2175 self.assert_is_copy(x, y) 2176 2177 # Dump using protocol 2 for test. 2178 s2 = self.dumps(x, 2) 2179 self.assertNotIn(__name__.encode("utf-8"), s2) 2180 self.assertNotIn(b"MyList", s2) 2181 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) 2182 2183 y = self.loads(s2) 2184 self.assert_is_copy(x, y) 2185 finally: 2186 e.restore() 2187 2188 def test_global_ext1(self): 2189 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 2190 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 2191 2192 def test_global_ext2(self): 2193 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 2194 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 2195 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 2196 2197 def test_global_ext4(self): 2198 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 2199 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 2200 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 2201 2202 def test_list_chunking(self): 2203 n = 10 # too small to chunk 2204 x = list(range(n)) 2205 for proto in protocols: 2206 s = self.dumps(x, proto) 2207 y = self.loads(s) 2208 self.assert_is_copy(x, y) 2209 num_appends = count_opcode(pickle.APPENDS, s) 2210 self.assertEqual(num_appends, proto > 0) 2211 2212 n = 2500 # expect at least two chunks when proto > 0 2213 x = list(range(n)) 2214 for proto in protocols: 2215 s = self.dumps(x, proto) 2216 y = self.loads(s) 2217 self.assert_is_copy(x, y) 2218 num_appends = count_opcode(pickle.APPENDS, s) 2219 if proto == 0: 2220 self.assertEqual(num_appends, 0) 2221 else: 2222 self.assertTrue(num_appends >= 2) 2223 2224 def test_dict_chunking(self): 2225 n = 10 # too small to chunk 2226 x = dict.fromkeys(range(n)) 2227 for proto in protocols: 2228 s = self.dumps(x, proto) 2229 self.assertIsInstance(s, bytes_types) 2230 y = self.loads(s) 2231 self.assert_is_copy(x, y) 2232 num_setitems = count_opcode(pickle.SETITEMS, s) 2233 self.assertEqual(num_setitems, proto > 0) 2234 2235 n = 2500 # expect at least two chunks when proto > 0 2236 x = dict.fromkeys(range(n)) 2237 for proto in protocols: 2238 s = self.dumps(x, proto) 2239 y = self.loads(s) 2240 self.assert_is_copy(x, y) 2241 num_setitems = count_opcode(pickle.SETITEMS, s) 2242 if proto == 0: 2243 self.assertEqual(num_setitems, 0) 2244 else: 2245 self.assertTrue(num_setitems >= 2) 2246 2247 def test_set_chunking(self): 2248 n = 10 # too small to chunk 2249 x = set(range(n)) 2250 for proto in protocols: 2251 s = self.dumps(x, proto) 2252 y = self.loads(s) 2253 self.assert_is_copy(x, y) 2254 num_additems = count_opcode(pickle.ADDITEMS, s) 2255 if proto < 4: 2256 self.assertEqual(num_additems, 0) 2257 else: 2258 self.assertEqual(num_additems, 1) 2259 2260 n = 2500 # expect at least two chunks when proto >= 4 2261 x = set(range(n)) 2262 for proto in protocols: 2263 s = self.dumps(x, proto) 2264 y = self.loads(s) 2265 self.assert_is_copy(x, y) 2266 num_additems = count_opcode(pickle.ADDITEMS, s) 2267 if proto < 4: 2268 self.assertEqual(num_additems, 0) 2269 else: 2270 self.assertGreaterEqual(num_additems, 2) 2271 2272 def test_simple_newobj(self): 2273 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__ 2274 x.abc = 666 2275 for proto in protocols: 2276 with self.subTest(proto=proto): 2277 s = self.dumps(x, proto) 2278 if proto < 1: 2279 self.assertIn(b'\nI64206', s) # INT 2280 else: 2281 self.assertIn(b'M\xce\xfa', s) # BININT2 2282 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2283 2 <= proto) 2284 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2285 y = self.loads(s) # will raise TypeError if __init__ called 2286 self.assert_is_copy(x, y) 2287 2288 def test_complex_newobj(self): 2289 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__ 2290 x.abc = 666 2291 for proto in protocols: 2292 with self.subTest(proto=proto): 2293 s = self.dumps(x, proto) 2294 if proto < 1: 2295 self.assertIn(b'\nI64206', s) # INT 2296 elif proto < 2: 2297 self.assertIn(b'M\xce\xfa', s) # BININT2 2298 elif proto < 4: 2299 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2300 else: 2301 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2302 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2303 2 <= proto) 2304 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2305 y = self.loads(s) # will raise TypeError if __init__ called 2306 self.assert_is_copy(x, y) 2307 2308 def test_complex_newobj_ex(self): 2309 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ 2310 x.abc = 666 2311 for proto in protocols: 2312 with self.subTest(proto=proto): 2313 s = self.dumps(x, proto) 2314 if proto < 1: 2315 self.assertIn(b'\nI64206', s) # INT 2316 elif proto < 2: 2317 self.assertIn(b'M\xce\xfa', s) # BININT2 2318 elif proto < 4: 2319 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2320 else: 2321 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2322 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) 2323 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), 2324 4 <= proto) 2325 y = self.loads(s) # will raise TypeError if __init__ called 2326 self.assert_is_copy(x, y) 2327 2328 def test_newobj_list_slots(self): 2329 x = SlotList([1, 2, 3]) 2330 x.foo = 42 2331 x.bar = "hello" 2332 s = self.dumps(x, 2) 2333 y = self.loads(s) 2334 self.assert_is_copy(x, y) 2335 2336 def test_reduce_overrides_default_reduce_ex(self): 2337 for proto in protocols: 2338 x = REX_one() 2339 self.assertEqual(x._reduce_called, 0) 2340 s = self.dumps(x, proto) 2341 self.assertEqual(x._reduce_called, 1) 2342 y = self.loads(s) 2343 self.assertEqual(y._reduce_called, 0) 2344 2345 def test_reduce_ex_called(self): 2346 for proto in protocols: 2347 x = REX_two() 2348 self.assertEqual(x._proto, None) 2349 s = self.dumps(x, proto) 2350 self.assertEqual(x._proto, proto) 2351 y = self.loads(s) 2352 self.assertEqual(y._proto, None) 2353 2354 def test_reduce_ex_overrides_reduce(self): 2355 for proto in protocols: 2356 x = REX_three() 2357 self.assertEqual(x._proto, None) 2358 s = self.dumps(x, proto) 2359 self.assertEqual(x._proto, proto) 2360 y = self.loads(s) 2361 self.assertEqual(y._proto, None) 2362 2363 def test_reduce_ex_calls_base(self): 2364 for proto in protocols: 2365 x = REX_four() 2366 self.assertEqual(x._proto, None) 2367 s = self.dumps(x, proto) 2368 self.assertEqual(x._proto, proto) 2369 y = self.loads(s) 2370 self.assertEqual(y._proto, proto) 2371 2372 def test_reduce_calls_base(self): 2373 for proto in protocols: 2374 x = REX_five() 2375 self.assertEqual(x._reduce_called, 0) 2376 s = self.dumps(x, proto) 2377 self.assertEqual(x._reduce_called, 1) 2378 y = self.loads(s) 2379 self.assertEqual(y._reduce_called, 1) 2380 2381 @no_tracing 2382 def test_bad_getattr(self): 2383 # Issue #3514: crash when there is an infinite loop in __getattr__ 2384 x = BadGetattr() 2385 for proto in protocols: 2386 with support.infinite_recursion(): 2387 self.assertRaises(RuntimeError, self.dumps, x, proto) 2388 2389 def test_reduce_bad_iterator(self): 2390 # Issue4176: crash when 4th and 5th items of __reduce__() 2391 # are not iterators 2392 class C(object): 2393 def __reduce__(self): 2394 # 4th item is not an iterator 2395 return list, (), None, [], None 2396 class D(object): 2397 def __reduce__(self): 2398 # 5th item is not an iterator 2399 return dict, (), None, None, [] 2400 2401 # Python implementation is less strict and also accepts iterables. 2402 for proto in protocols: 2403 try: 2404 self.dumps(C(), proto) 2405 except pickle.PicklingError: 2406 pass 2407 try: 2408 self.dumps(D(), proto) 2409 except pickle.PicklingError: 2410 pass 2411 2412 def test_many_puts_and_gets(self): 2413 # Test that internal data structures correctly deal with lots of 2414 # puts/gets. 2415 keys = ("aaa" + str(i) for i in range(100)) 2416 large_dict = dict((k, [4, 5, 6]) for k in keys) 2417 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 2418 2419 for proto in protocols: 2420 with self.subTest(proto=proto): 2421 dumped = self.dumps(obj, proto) 2422 loaded = self.loads(dumped) 2423 self.assert_is_copy(obj, loaded) 2424 2425 def test_attribute_name_interning(self): 2426 # Test that attribute names of pickled objects are interned when 2427 # unpickling. 2428 for proto in protocols: 2429 x = C() 2430 x.foo = 42 2431 x.bar = "hello" 2432 s = self.dumps(x, proto) 2433 y = self.loads(s) 2434 x_keys = sorted(x.__dict__) 2435 y_keys = sorted(y.__dict__) 2436 for x_key, y_key in zip(x_keys, y_keys): 2437 self.assertIs(x_key, y_key) 2438 2439 def test_pickle_to_2x(self): 2440 # Pickle non-trivial data with protocol 2, expecting that it yields 2441 # the same result as Python 2.x did. 2442 # NOTE: this test is a bit too strong since we can produce different 2443 # bytecode that 2.x will still understand. 2444 dumped = self.dumps(range(5), 2) 2445 self.assertEqual(dumped, DATA_XRANGE) 2446 dumped = self.dumps(set([3]), 2) 2447 self.assertEqual(dumped, DATA_SET2) 2448 2449 def test_large_pickles(self): 2450 # Test the correctness of internal buffering routines when handling 2451 # large data. 2452 for proto in protocols: 2453 data = (1, min, b'xy' * (30 * 1024), len) 2454 dumped = self.dumps(data, proto) 2455 loaded = self.loads(dumped) 2456 self.assertEqual(len(loaded), len(data)) 2457 self.assertEqual(loaded, data) 2458 2459 def test_int_pickling_efficiency(self): 2460 # Test compacity of int representation (see issue #12744) 2461 for proto in protocols: 2462 with self.subTest(proto=proto): 2463 pickles = [self.dumps(2**n, proto) for n in range(70)] 2464 sizes = list(map(len, pickles)) 2465 # the size function is monotonic 2466 self.assertEqual(sorted(sizes), sizes) 2467 if proto >= 2: 2468 for p in pickles: 2469 self.assertFalse(opcode_in_pickle(pickle.LONG, p)) 2470 2471 def _check_pickling_with_opcode(self, obj, opcode, proto): 2472 pickled = self.dumps(obj, proto) 2473 self.assertTrue(opcode_in_pickle(opcode, pickled)) 2474 unpickled = self.loads(pickled) 2475 self.assertEqual(obj, unpickled) 2476 2477 def test_appends_on_non_lists(self): 2478 # Issue #17720 2479 obj = REX_six([1, 2, 3]) 2480 for proto in protocols: 2481 if proto == 0: 2482 self._check_pickling_with_opcode(obj, pickle.APPEND, proto) 2483 else: 2484 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 2485 2486 def test_setitems_on_non_dicts(self): 2487 obj = REX_seven({1: -1, 2: -2, 3: -3}) 2488 for proto in protocols: 2489 if proto == 0: 2490 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 2491 else: 2492 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 2493 2494 # Exercise framing (proto >= 4) for significant workloads 2495 2496 FRAME_SIZE_MIN = 4 2497 FRAME_SIZE_TARGET = 64 * 1024 2498 2499 def check_frame_opcodes(self, pickled): 2500 """ 2501 Check the arguments of FRAME opcodes in a protocol 4+ pickle. 2502 2503 Note that binary objects that are larger than FRAME_SIZE_TARGET are not 2504 framed by default and are therefore considered a frame by themselves in 2505 the following consistency check. 2506 """ 2507 frame_end = frameless_start = None 2508 frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8', 2509 'BINUNICODE8', 'BYTEARRAY8'} 2510 for op, arg, pos in pickletools.genops(pickled): 2511 if frame_end is not None: 2512 self.assertLessEqual(pos, frame_end) 2513 if pos == frame_end: 2514 frame_end = None 2515 2516 if frame_end is not None: # framed 2517 self.assertNotEqual(op.name, 'FRAME') 2518 if op.name in frameless_opcodes: 2519 # Only short bytes and str objects should be written 2520 # in a frame 2521 self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET) 2522 2523 else: # not framed 2524 if (op.name == 'FRAME' or 2525 (op.name in frameless_opcodes and 2526 len(arg) > self.FRAME_SIZE_TARGET)): 2527 # Frame or large bytes or str object 2528 if frameless_start is not None: 2529 # Only short data should be written outside of a frame 2530 self.assertLess(pos - frameless_start, 2531 self.FRAME_SIZE_MIN) 2532 frameless_start = None 2533 elif frameless_start is None and op.name != 'PROTO': 2534 frameless_start = pos 2535 2536 if op.name == 'FRAME': 2537 self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN) 2538 frame_end = pos + 9 + arg 2539 2540 pos = len(pickled) 2541 if frame_end is not None: 2542 self.assertEqual(frame_end, pos) 2543 elif frameless_start is not None: 2544 self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) 2545 2546 @support.skip_if_pgo_task 2547 def test_framing_many_objects(self): 2548 obj = list(range(10**5)) 2549 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2550 with self.subTest(proto=proto): 2551 pickled = self.dumps(obj, proto) 2552 unpickled = self.loads(pickled) 2553 self.assertEqual(obj, unpickled) 2554 bytes_per_frame = (len(pickled) / 2555 count_opcode(pickle.FRAME, pickled)) 2556 self.assertGreater(bytes_per_frame, 2557 self.FRAME_SIZE_TARGET / 2) 2558 self.assertLessEqual(bytes_per_frame, 2559 self.FRAME_SIZE_TARGET * 1) 2560 self.check_frame_opcodes(pickled) 2561 2562 def test_framing_large_objects(self): 2563 N = 1024 * 1024 2564 small_items = [[i] for i in range(10)] 2565 obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] 2566 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2567 for fast in [False, True]: 2568 with self.subTest(proto=proto, fast=fast): 2569 if not fast: 2570 # fast=False by default. 2571 # This covers in-memory pickling with pickle.dumps(). 2572 pickled = self.dumps(obj, proto) 2573 else: 2574 # Pickler is required when fast=True. 2575 if not hasattr(self, 'pickler'): 2576 continue 2577 buf = io.BytesIO() 2578 pickler = self.pickler(buf, protocol=proto) 2579 pickler.fast = fast 2580 pickler.dump(obj) 2581 pickled = buf.getvalue() 2582 unpickled = self.loads(pickled) 2583 # More informative error message in case of failure. 2584 self.assertEqual([len(x) for x in obj], 2585 [len(x) for x in unpickled]) 2586 # Perform full equality check if the lengths match. 2587 self.assertEqual(obj, unpickled) 2588 n_frames = count_opcode(pickle.FRAME, pickled) 2589 # A single frame for small objects between 2590 # first two large objects. 2591 self.assertEqual(n_frames, 1) 2592 self.check_frame_opcodes(pickled) 2593 2594 def test_optional_frames(self): 2595 if pickle.HIGHEST_PROTOCOL < 4: 2596 return 2597 2598 def remove_frames(pickled, keep_frame=None): 2599 """Remove frame opcodes from the given pickle.""" 2600 frame_starts = [] 2601 # 1 byte for the opcode and 8 for the argument 2602 frame_opcode_size = 9 2603 for opcode, _, pos in pickletools.genops(pickled): 2604 if opcode.name == 'FRAME': 2605 frame_starts.append(pos) 2606 2607 newpickle = bytearray() 2608 last_frame_end = 0 2609 for i, pos in enumerate(frame_starts): 2610 if keep_frame and keep_frame(i): 2611 continue 2612 newpickle += pickled[last_frame_end:pos] 2613 last_frame_end = pos + frame_opcode_size 2614 newpickle += pickled[last_frame_end:] 2615 return newpickle 2616 2617 frame_size = self.FRAME_SIZE_TARGET 2618 num_frames = 20 2619 # Large byte objects (dict values) intermittent with small objects 2620 # (dict keys) 2621 for bytes_type in (bytes, bytearray): 2622 obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)} 2623 2624 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2625 pickled = self.dumps(obj, proto) 2626 2627 frameless_pickle = remove_frames(pickled) 2628 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0) 2629 self.assertEqual(obj, self.loads(frameless_pickle)) 2630 2631 some_frames_pickle = remove_frames(pickled, lambda i: i % 2) 2632 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle), 2633 count_opcode(pickle.FRAME, pickled)) 2634 self.assertEqual(obj, self.loads(some_frames_pickle)) 2635 2636 @support.skip_if_pgo_task 2637 def test_framed_write_sizes_with_delayed_writer(self): 2638 class ChunkAccumulator: 2639 """Accumulate pickler output in a list of raw chunks.""" 2640 def __init__(self): 2641 self.chunks = [] 2642 def write(self, chunk): 2643 self.chunks.append(chunk) 2644 def concatenate_chunks(self): 2645 return b"".join(self.chunks) 2646 2647 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2648 objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)}) 2649 for i in range(int(1e4))] 2650 # Add a large unique ASCII string 2651 objects.append('0123456789abcdef' * 2652 (self.FRAME_SIZE_TARGET // 16 + 1)) 2653 2654 # Protocol 4 packs groups of small objects into frames and issues 2655 # calls to write only once or twice per frame: 2656 # The C pickler issues one call to write per-frame (header and 2657 # contents) while Python pickler issues two calls to write: one for 2658 # the frame header and one for the frame binary contents. 2659 writer = ChunkAccumulator() 2660 self.pickler(writer, proto).dump(objects) 2661 2662 # Actually read the binary content of the chunks after the end 2663 # of the call to dump: any memoryview passed to write should not 2664 # be released otherwise this delayed access would not be possible. 2665 pickled = writer.concatenate_chunks() 2666 reconstructed = self.loads(pickled) 2667 self.assertEqual(reconstructed, objects) 2668 self.assertGreater(len(writer.chunks), 1) 2669 2670 # memoryviews should own the memory. 2671 del objects 2672 support.gc_collect() 2673 self.assertEqual(writer.concatenate_chunks(), pickled) 2674 2675 n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1 2676 # There should be at least one call to write per frame 2677 self.assertGreaterEqual(len(writer.chunks), n_frames) 2678 2679 # but not too many either: there can be one for the proto, 2680 # one per-frame header, one per frame for the actual contents, 2681 # and two for the header. 2682 self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3) 2683 2684 chunk_sizes = [len(c) for c in writer.chunks] 2685 large_sizes = [s for s in chunk_sizes 2686 if s >= self.FRAME_SIZE_TARGET] 2687 medium_sizes = [s for s in chunk_sizes 2688 if 9 < s < self.FRAME_SIZE_TARGET] 2689 small_sizes = [s for s in chunk_sizes if s <= 9] 2690 2691 # Large chunks should not be too large: 2692 for chunk_size in large_sizes: 2693 self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET, 2694 chunk_sizes) 2695 # There shouldn't bee too many small chunks: the protocol header, 2696 # the frame headers and the large string headers are written 2697 # in small chunks. 2698 self.assertLessEqual(len(small_sizes), 2699 len(large_sizes) + len(medium_sizes) + 3, 2700 chunk_sizes) 2701 2702 def test_nested_names(self): 2703 global Nested 2704 class Nested: 2705 class A: 2706 class B: 2707 class C: 2708 pass 2709 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2710 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: 2711 with self.subTest(proto=proto, obj=obj): 2712 unpickled = self.loads(self.dumps(obj, proto)) 2713 self.assertIs(obj, unpickled) 2714 2715 def test_recursive_nested_names(self): 2716 global Recursive 2717 class Recursive: 2718 pass 2719 Recursive.mod = sys.modules[Recursive.__module__] 2720 Recursive.__qualname__ = 'Recursive.mod.Recursive' 2721 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2722 with self.subTest(proto=proto): 2723 unpickled = self.loads(self.dumps(Recursive, proto)) 2724 self.assertIs(unpickled, Recursive) 2725 del Recursive.mod # break reference loop 2726 2727 def test_py_methods(self): 2728 global PyMethodsTest 2729 class PyMethodsTest: 2730 @staticmethod 2731 def cheese(): 2732 return "cheese" 2733 @classmethod 2734 def wine(cls): 2735 assert cls is PyMethodsTest 2736 return "wine" 2737 def biscuits(self): 2738 assert isinstance(self, PyMethodsTest) 2739 return "biscuits" 2740 class Nested: 2741 "Nested class" 2742 @staticmethod 2743 def ketchup(): 2744 return "ketchup" 2745 @classmethod 2746 def maple(cls): 2747 assert cls is PyMethodsTest.Nested 2748 return "maple" 2749 def pie(self): 2750 assert isinstance(self, PyMethodsTest.Nested) 2751 return "pie" 2752 2753 py_methods = ( 2754 PyMethodsTest.cheese, 2755 PyMethodsTest.wine, 2756 PyMethodsTest().biscuits, 2757 PyMethodsTest.Nested.ketchup, 2758 PyMethodsTest.Nested.maple, 2759 PyMethodsTest.Nested().pie 2760 ) 2761 py_unbound_methods = ( 2762 (PyMethodsTest.biscuits, PyMethodsTest), 2763 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) 2764 ) 2765 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2766 for method in py_methods: 2767 with self.subTest(proto=proto, method=method): 2768 unpickled = self.loads(self.dumps(method, proto)) 2769 self.assertEqual(method(), unpickled()) 2770 for method, cls in py_unbound_methods: 2771 obj = cls() 2772 with self.subTest(proto=proto, method=method): 2773 unpickled = self.loads(self.dumps(method, proto)) 2774 self.assertEqual(method(obj), unpickled(obj)) 2775 2776 def test_c_methods(self): 2777 global Subclass 2778 class Subclass(tuple): 2779 class Nested(str): 2780 pass 2781 2782 c_methods = ( 2783 # bound built-in method 2784 ("abcd".index, ("c",)), 2785 # unbound built-in method 2786 (str.index, ("abcd", "c")), 2787 # bound "slot" method 2788 ([1, 2, 3].__len__, ()), 2789 # unbound "slot" method 2790 (list.__len__, ([1, 2, 3],)), 2791 # bound "coexist" method 2792 ({1, 2}.__contains__, (2,)), 2793 # unbound "coexist" method 2794 (set.__contains__, ({1, 2}, 2)), 2795 # built-in class method 2796 (dict.fromkeys, (("a", 1), ("b", 2))), 2797 # built-in static method 2798 (bytearray.maketrans, (b"abc", b"xyz")), 2799 # subclass methods 2800 (Subclass([1,2,2]).count, (2,)), 2801 (Subclass.count, (Subclass([1,2,2]), 2)), 2802 (Subclass.Nested("sweet").count, ("e",)), 2803 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), 2804 ) 2805 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2806 for method, args in c_methods: 2807 with self.subTest(proto=proto, method=method): 2808 unpickled = self.loads(self.dumps(method, proto)) 2809 self.assertEqual(method(*args), unpickled(*args)) 2810 2811 def test_compat_pickle(self): 2812 tests = [ 2813 (range(1, 7), '__builtin__', 'xrange'), 2814 (map(int, '123'), 'itertools', 'imap'), 2815 (functools.reduce, '__builtin__', 'reduce'), 2816 (dbm.whichdb, 'whichdb', 'whichdb'), 2817 (Exception(), 'exceptions', 'Exception'), 2818 (collections.UserDict(), 'UserDict', 'IterableUserDict'), 2819 (collections.UserList(), 'UserList', 'UserList'), 2820 (collections.defaultdict(), 'collections', 'defaultdict'), 2821 ] 2822 for val, mod, name in tests: 2823 for proto in range(3): 2824 with self.subTest(type=type(val), proto=proto): 2825 pickled = self.dumps(val, proto) 2826 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) 2827 self.assertIs(type(self.loads(pickled)), type(val)) 2828 2829 def test_local_lookup_error(self): 2830 # Test that whichmodule() errors out cleanly when looking up 2831 # an assumed globally-reachable object fails. 2832 def f(): 2833 pass 2834 # Since the function is local, lookup will fail 2835 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2836 with self.assertRaises((AttributeError, pickle.PicklingError)): 2837 pickletools.dis(self.dumps(f, proto)) 2838 # Same without a __module__ attribute (exercises a different path 2839 # in _pickle.c). 2840 del f.__module__ 2841 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2842 with self.assertRaises((AttributeError, pickle.PicklingError)): 2843 pickletools.dis(self.dumps(f, proto)) 2844 # Yet a different path. 2845 f.__name__ = f.__qualname__ 2846 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2847 with self.assertRaises((AttributeError, pickle.PicklingError)): 2848 pickletools.dis(self.dumps(f, proto)) 2849 2850 # 2851 # PEP 574 tests below 2852 # 2853 2854 def buffer_like_objects(self): 2855 # Yield buffer-like objects with the bytestring "abcdef" in them 2856 bytestring = b"abcdefgh" 2857 yield ZeroCopyBytes(bytestring) 2858 yield ZeroCopyBytearray(bytestring) 2859 if _testbuffer is not None: 2860 items = list(bytestring) 2861 value = int.from_bytes(bytestring, byteorder='little') 2862 for flags in (0, _testbuffer.ND_WRITABLE): 2863 # 1-D, contiguous 2864 yield PicklableNDArray(items, format='B', shape=(8,), 2865 flags=flags) 2866 # 2-D, C-contiguous 2867 yield PicklableNDArray(items, format='B', shape=(4, 2), 2868 strides=(2, 1), flags=flags) 2869 # 2-D, Fortran-contiguous 2870 yield PicklableNDArray(items, format='B', 2871 shape=(4, 2), strides=(1, 4), 2872 flags=flags) 2873 2874 def test_in_band_buffers(self): 2875 # Test in-band buffers (PEP 574) 2876 for obj in self.buffer_like_objects(): 2877 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2878 data = self.dumps(obj, proto) 2879 if obj.c_contiguous and proto >= 5: 2880 # The raw memory bytes are serialized in physical order 2881 self.assertIn(b"abcdefgh", data) 2882 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0) 2883 if proto >= 5: 2884 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 2885 1 if obj.readonly else 0) 2886 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 2887 0 if obj.readonly else 1) 2888 # Return a true value from buffer_callback should have 2889 # the same effect 2890 def buffer_callback(obj): 2891 return True 2892 data2 = self.dumps(obj, proto, 2893 buffer_callback=buffer_callback) 2894 self.assertEqual(data2, data) 2895 2896 new = self.loads(data) 2897 # It's a copy 2898 self.assertIsNot(new, obj) 2899 self.assertIs(type(new), type(obj)) 2900 self.assertEqual(new, obj) 2901 2902 # XXX Unfortunately cannot test non-contiguous array 2903 # (see comment in PicklableNDArray.__reduce_ex__) 2904 2905 def test_oob_buffers(self): 2906 # Test out-of-band buffers (PEP 574) 2907 for obj in self.buffer_like_objects(): 2908 for proto in range(0, 5): 2909 # Need protocol >= 5 for buffer_callback 2910 with self.assertRaises(ValueError): 2911 self.dumps(obj, proto, 2912 buffer_callback=[].append) 2913 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2914 buffers = [] 2915 buffer_callback = lambda pb: buffers.append(pb.raw()) 2916 data = self.dumps(obj, proto, 2917 buffer_callback=buffer_callback) 2918 self.assertNotIn(b"abcdefgh", data) 2919 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0) 2920 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0) 2921 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1) 2922 self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data), 2923 1 if obj.readonly else 0) 2924 2925 if obj.c_contiguous: 2926 self.assertEqual(bytes(buffers[0]), b"abcdefgh") 2927 # Need buffers argument to unpickle properly 2928 with self.assertRaises(pickle.UnpicklingError): 2929 self.loads(data) 2930 2931 new = self.loads(data, buffers=buffers) 2932 if obj.zero_copy_reconstruct: 2933 # Zero-copy achieved 2934 self.assertIs(new, obj) 2935 else: 2936 self.assertIs(type(new), type(obj)) 2937 self.assertEqual(new, obj) 2938 # Non-sequence buffers accepted too 2939 new = self.loads(data, buffers=iter(buffers)) 2940 if obj.zero_copy_reconstruct: 2941 # Zero-copy achieved 2942 self.assertIs(new, obj) 2943 else: 2944 self.assertIs(type(new), type(obj)) 2945 self.assertEqual(new, obj) 2946 2947 def test_oob_buffers_writable_to_readonly(self): 2948 # Test reconstructing readonly object from writable buffer 2949 obj = ZeroCopyBytes(b"foobar") 2950 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2951 buffers = [] 2952 buffer_callback = buffers.append 2953 data = self.dumps(obj, proto, buffer_callback=buffer_callback) 2954 2955 buffers = map(bytearray, buffers) 2956 new = self.loads(data, buffers=buffers) 2957 self.assertIs(type(new), type(obj)) 2958 self.assertEqual(new, obj) 2959 2960 def test_picklebuffer_error(self): 2961 # PickleBuffer forbidden with protocol < 5 2962 pb = pickle.PickleBuffer(b"foobar") 2963 for proto in range(0, 5): 2964 with self.assertRaises(pickle.PickleError): 2965 self.dumps(pb, proto) 2966 2967 def test_buffer_callback_error(self): 2968 def buffer_callback(buffers): 2969 1/0 2970 pb = pickle.PickleBuffer(b"foobar") 2971 with self.assertRaises(ZeroDivisionError): 2972 self.dumps(pb, 5, buffer_callback=buffer_callback) 2973 2974 def test_buffers_error(self): 2975 pb = pickle.PickleBuffer(b"foobar") 2976 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2977 data = self.dumps(pb, proto, buffer_callback=[].append) 2978 # Non iterable buffers 2979 with self.assertRaises(TypeError): 2980 self.loads(data, buffers=object()) 2981 # Buffer iterable exhausts too early 2982 with self.assertRaises(pickle.UnpicklingError): 2983 self.loads(data, buffers=[]) 2984 2985 def test_inband_accept_default_buffers_argument(self): 2986 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2987 data_pickled = self.dumps(1, proto, buffer_callback=None) 2988 data = self.loads(data_pickled, buffers=None) 2989 2990 @unittest.skipIf(np is None, "Test needs Numpy") 2991 def test_buffers_numpy(self): 2992 def check_no_copy(x, y): 2993 np.testing.assert_equal(x, y) 2994 self.assertEqual(x.ctypes.data, y.ctypes.data) 2995 2996 def check_copy(x, y): 2997 np.testing.assert_equal(x, y) 2998 self.assertNotEqual(x.ctypes.data, y.ctypes.data) 2999 3000 def check_array(arr): 3001 # In-band 3002 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3003 data = self.dumps(arr, proto) 3004 new = self.loads(data) 3005 check_copy(arr, new) 3006 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3007 buffer_callback = lambda _: True 3008 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 3009 new = self.loads(data) 3010 check_copy(arr, new) 3011 # Out-of-band 3012 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3013 buffers = [] 3014 buffer_callback = buffers.append 3015 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 3016 new = self.loads(data, buffers=buffers) 3017 if arr.flags.c_contiguous or arr.flags.f_contiguous: 3018 check_no_copy(arr, new) 3019 else: 3020 check_copy(arr, new) 3021 3022 # 1-D 3023 arr = np.arange(6) 3024 check_array(arr) 3025 # 1-D, non-contiguous 3026 check_array(arr[::2]) 3027 # 2-D, C-contiguous 3028 arr = np.arange(12).reshape((3, 4)) 3029 check_array(arr) 3030 # 2-D, F-contiguous 3031 check_array(arr.T) 3032 # 2-D, non-contiguous 3033 check_array(arr[::2]) 3034 3035 3036class BigmemPickleTests: 3037 3038 # Binary protocols can serialize longs of up to 2 GiB-1 3039 3040 @bigmemtest(size=_2G, memuse=3.6, dry_run=False) 3041 def test_huge_long_32b(self, size): 3042 data = 1 << (8 * size) 3043 try: 3044 for proto in protocols: 3045 if proto < 2: 3046 continue 3047 with self.subTest(proto=proto): 3048 with self.assertRaises((ValueError, OverflowError)): 3049 self.dumps(data, protocol=proto) 3050 finally: 3051 data = None 3052 3053 # Protocol 3 can serialize up to 4 GiB-1 as a bytes object 3054 # (older protocols don't have a dedicated opcode for bytes and are 3055 # too inefficient) 3056 3057 @bigmemtest(size=_2G, memuse=2.5, dry_run=False) 3058 def test_huge_bytes_32b(self, size): 3059 data = b"abcd" * (size // 4) 3060 try: 3061 for proto in protocols: 3062 if proto < 3: 3063 continue 3064 with self.subTest(proto=proto): 3065 try: 3066 pickled = self.dumps(data, protocol=proto) 3067 header = (pickle.BINBYTES + 3068 struct.pack("<I", len(data))) 3069 data_start = pickled.index(data) 3070 self.assertEqual( 3071 header, 3072 pickled[data_start-len(header):data_start]) 3073 finally: 3074 pickled = None 3075 finally: 3076 data = None 3077 3078 @bigmemtest(size=_4G, memuse=2.5, dry_run=False) 3079 def test_huge_bytes_64b(self, size): 3080 data = b"acbd" * (size // 4) 3081 try: 3082 for proto in protocols: 3083 if proto < 3: 3084 continue 3085 with self.subTest(proto=proto): 3086 if proto == 3: 3087 # Protocol 3 does not support large bytes objects. 3088 # Verify that we do not crash when processing one. 3089 with self.assertRaises((ValueError, OverflowError)): 3090 self.dumps(data, protocol=proto) 3091 continue 3092 try: 3093 pickled = self.dumps(data, protocol=proto) 3094 header = (pickle.BINBYTES8 + 3095 struct.pack("<Q", len(data))) 3096 data_start = pickled.index(data) 3097 self.assertEqual( 3098 header, 3099 pickled[data_start-len(header):data_start]) 3100 finally: 3101 pickled = None 3102 finally: 3103 data = None 3104 3105 # All protocols use 1-byte per printable ASCII character; we add another 3106 # byte because the encoded form has to be copied into the internal buffer. 3107 3108 @bigmemtest(size=_2G, memuse=8, dry_run=False) 3109 def test_huge_str_32b(self, size): 3110 data = "abcd" * (size // 4) 3111 try: 3112 for proto in protocols: 3113 if proto == 0: 3114 continue 3115 with self.subTest(proto=proto): 3116 try: 3117 pickled = self.dumps(data, protocol=proto) 3118 header = (pickle.BINUNICODE + 3119 struct.pack("<I", len(data))) 3120 data_start = pickled.index(b'abcd') 3121 self.assertEqual( 3122 header, 3123 pickled[data_start-len(header):data_start]) 3124 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3125 pickled.index(b"abcd")), len(data)) 3126 finally: 3127 pickled = None 3128 finally: 3129 data = None 3130 3131 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes 3132 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge 3133 # unicode strings however. 3134 3135 @bigmemtest(size=_4G, memuse=8, dry_run=False) 3136 def test_huge_str_64b(self, size): 3137 data = "abcd" * (size // 4) 3138 try: 3139 for proto in protocols: 3140 if proto == 0: 3141 continue 3142 with self.subTest(proto=proto): 3143 if proto < 4: 3144 with self.assertRaises((ValueError, OverflowError)): 3145 self.dumps(data, protocol=proto) 3146 continue 3147 try: 3148 pickled = self.dumps(data, protocol=proto) 3149 header = (pickle.BINUNICODE8 + 3150 struct.pack("<Q", len(data))) 3151 data_start = pickled.index(b'abcd') 3152 self.assertEqual( 3153 header, 3154 pickled[data_start-len(header):data_start]) 3155 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3156 pickled.index(b"abcd")), len(data)) 3157 finally: 3158 pickled = None 3159 finally: 3160 data = None 3161 3162 3163# Test classes for reduce_ex 3164 3165class REX_one(object): 3166 """No __reduce_ex__ here, but inheriting it from object""" 3167 _reduce_called = 0 3168 def __reduce__(self): 3169 self._reduce_called = 1 3170 return REX_one, () 3171 3172class REX_two(object): 3173 """No __reduce__ here, but inheriting it from object""" 3174 _proto = None 3175 def __reduce_ex__(self, proto): 3176 self._proto = proto 3177 return REX_two, () 3178 3179class REX_three(object): 3180 _proto = None 3181 def __reduce_ex__(self, proto): 3182 self._proto = proto 3183 return REX_two, () 3184 def __reduce__(self): 3185 raise TestFailed("This __reduce__ shouldn't be called") 3186 3187class REX_four(object): 3188 """Calling base class method should succeed""" 3189 _proto = None 3190 def __reduce_ex__(self, proto): 3191 self._proto = proto 3192 return object.__reduce_ex__(self, proto) 3193 3194class REX_five(object): 3195 """This one used to fail with infinite recursion""" 3196 _reduce_called = 0 3197 def __reduce__(self): 3198 self._reduce_called = 1 3199 return object.__reduce__(self) 3200 3201class REX_six(object): 3202 """This class is used to check the 4th argument (list iterator) of 3203 the reduce protocol. 3204 """ 3205 def __init__(self, items=None): 3206 self.items = items if items is not None else [] 3207 def __eq__(self, other): 3208 return type(self) is type(other) and self.items == other.items 3209 def append(self, item): 3210 self.items.append(item) 3211 def __reduce__(self): 3212 return type(self), (), None, iter(self.items), None 3213 3214class REX_seven(object): 3215 """This class is used to check the 5th argument (dict iterator) of 3216 the reduce protocol. 3217 """ 3218 def __init__(self, table=None): 3219 self.table = table if table is not None else {} 3220 def __eq__(self, other): 3221 return type(self) is type(other) and self.table == other.table 3222 def __setitem__(self, key, value): 3223 self.table[key] = value 3224 def __reduce__(self): 3225 return type(self), (), None, None, iter(self.table.items()) 3226 3227class REX_state(object): 3228 """This class is used to check the 3th argument (state) of 3229 the reduce protocol. 3230 """ 3231 def __init__(self, state=None): 3232 self.state = state 3233 def __eq__(self, other): 3234 return type(self) is type(other) and self.state == other.state 3235 def __setstate__(self, state): 3236 self.state = state 3237 def __reduce__(self): 3238 return type(self), (), self.state 3239 3240 3241# Test classes for newobj 3242 3243class MyInt(int): 3244 sample = 1 3245 3246class MyFloat(float): 3247 sample = 1.0 3248 3249class MyComplex(complex): 3250 sample = 1.0 + 0.0j 3251 3252class MyStr(str): 3253 sample = "hello" 3254 3255class MyUnicode(str): 3256 sample = "hello \u1234" 3257 3258class MyTuple(tuple): 3259 sample = (1, 2, 3) 3260 3261class MyList(list): 3262 sample = [1, 2, 3] 3263 3264class MyDict(dict): 3265 sample = {"a": 1, "b": 2} 3266 3267class MySet(set): 3268 sample = {"a", "b"} 3269 3270class MyFrozenSet(frozenset): 3271 sample = frozenset({"a", "b"}) 3272 3273myclasses = [MyInt, MyFloat, 3274 MyComplex, 3275 MyStr, MyUnicode, 3276 MyTuple, MyList, MyDict, MySet, MyFrozenSet] 3277 3278class MyIntWithNew(int): 3279 def __new__(cls, value): 3280 raise AssertionError 3281 3282class MyIntWithNew2(MyIntWithNew): 3283 __new__ = int.__new__ 3284 3285 3286class SlotList(MyList): 3287 __slots__ = ["foo"] 3288 3289class SimpleNewObj(int): 3290 def __init__(self, *args, **kwargs): 3291 # raise an error, to make sure this isn't called 3292 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 3293 def __eq__(self, other): 3294 return int(self) == int(other) and self.__dict__ == other.__dict__ 3295 3296class ComplexNewObj(SimpleNewObj): 3297 def __getnewargs__(self): 3298 return ('%X' % self, 16) 3299 3300class ComplexNewObjEx(SimpleNewObj): 3301 def __getnewargs_ex__(self): 3302 return ('%X' % self,), {'base': 16} 3303 3304class BadGetattr: 3305 def __getattr__(self, key): 3306 self.foo 3307 3308 3309class AbstractPickleModuleTests: 3310 3311 def test_dump_closed_file(self): 3312 f = open(TESTFN, "wb") 3313 try: 3314 f.close() 3315 self.assertRaises(ValueError, self.dump, 123, f) 3316 finally: 3317 os_helper.unlink(TESTFN) 3318 3319 def test_load_closed_file(self): 3320 f = open(TESTFN, "wb") 3321 try: 3322 f.close() 3323 self.assertRaises(ValueError, self.dump, 123, f) 3324 finally: 3325 os_helper.unlink(TESTFN) 3326 3327 def test_load_from_and_dump_to_file(self): 3328 stream = io.BytesIO() 3329 data = [123, {}, 124] 3330 self.dump(data, stream) 3331 stream.seek(0) 3332 unpickled = self.load(stream) 3333 self.assertEqual(unpickled, data) 3334 3335 def test_highest_protocol(self): 3336 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 3337 self.assertEqual(pickle.HIGHEST_PROTOCOL, 5) 3338 3339 def test_callapi(self): 3340 f = io.BytesIO() 3341 # With and without keyword arguments 3342 self.dump(123, f, -1) 3343 self.dump(123, file=f, protocol=-1) 3344 self.dumps(123, -1) 3345 self.dumps(123, protocol=-1) 3346 self.Pickler(f, -1) 3347 self.Pickler(f, protocol=-1) 3348 3349 def test_dump_text_file(self): 3350 f = open(TESTFN, "w") 3351 try: 3352 for proto in protocols: 3353 self.assertRaises(TypeError, self.dump, 123, f, proto) 3354 finally: 3355 f.close() 3356 os_helper.unlink(TESTFN) 3357 3358 def test_incomplete_input(self): 3359 s = io.BytesIO(b"X''.") 3360 self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s) 3361 3362 def test_bad_init(self): 3363 # Test issue3664 (pickle can segfault from a badly initialized Pickler). 3364 # Override initialization without calling __init__() of the superclass. 3365 class BadPickler(self.Pickler): 3366 def __init__(self): pass 3367 3368 class BadUnpickler(self.Unpickler): 3369 def __init__(self): pass 3370 3371 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) 3372 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) 3373 3374 def check_dumps_loads_oob_buffers(self, dumps, loads): 3375 # No need to do the full gamut of tests here, just enough to 3376 # check that dumps() and loads() redirect their arguments 3377 # to the underlying Pickler and Unpickler, respectively. 3378 obj = ZeroCopyBytes(b"foo") 3379 3380 for proto in range(0, 5): 3381 # Need protocol >= 5 for buffer_callback 3382 with self.assertRaises(ValueError): 3383 dumps(obj, protocol=proto, 3384 buffer_callback=[].append) 3385 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3386 buffers = [] 3387 buffer_callback = buffers.append 3388 data = dumps(obj, protocol=proto, 3389 buffer_callback=buffer_callback) 3390 self.assertNotIn(b"foo", data) 3391 self.assertEqual(bytes(buffers[0]), b"foo") 3392 # Need buffers argument to unpickle properly 3393 with self.assertRaises(pickle.UnpicklingError): 3394 loads(data) 3395 new = loads(data, buffers=buffers) 3396 self.assertIs(new, obj) 3397 3398 def test_dumps_loads_oob_buffers(self): 3399 # Test out-of-band buffers (PEP 574) with top-level dumps() and loads() 3400 self.check_dumps_loads_oob_buffers(self.dumps, self.loads) 3401 3402 def test_dump_load_oob_buffers(self): 3403 # Test out-of-band buffers (PEP 574) with top-level dump() and load() 3404 def dumps(obj, **kwargs): 3405 f = io.BytesIO() 3406 self.dump(obj, f, **kwargs) 3407 return f.getvalue() 3408 3409 def loads(data, **kwargs): 3410 f = io.BytesIO(data) 3411 return self.load(f, **kwargs) 3412 3413 self.check_dumps_loads_oob_buffers(dumps, loads) 3414 3415 3416class AbstractPersistentPicklerTests: 3417 3418 # This class defines persistent_id() and persistent_load() 3419 # functions that should be used by the pickler. All even integers 3420 # are pickled using persistent ids. 3421 3422 def persistent_id(self, object): 3423 if isinstance(object, int) and object % 2 == 0: 3424 self.id_count += 1 3425 return str(object) 3426 elif object == "test_false_value": 3427 self.false_count += 1 3428 return "" 3429 else: 3430 return None 3431 3432 def persistent_load(self, oid): 3433 if not oid: 3434 self.load_false_count += 1 3435 return "test_false_value" 3436 else: 3437 self.load_count += 1 3438 object = int(oid) 3439 assert object % 2 == 0 3440 return object 3441 3442 def test_persistence(self): 3443 L = list(range(10)) + ["test_false_value"] 3444 for proto in protocols: 3445 self.id_count = 0 3446 self.false_count = 0 3447 self.load_false_count = 0 3448 self.load_count = 0 3449 self.assertEqual(self.loads(self.dumps(L, proto)), L) 3450 self.assertEqual(self.id_count, 5) 3451 self.assertEqual(self.false_count, 1) 3452 self.assertEqual(self.load_count, 5) 3453 self.assertEqual(self.load_false_count, 1) 3454 3455 3456class AbstractIdentityPersistentPicklerTests: 3457 3458 def persistent_id(self, obj): 3459 return obj 3460 3461 def persistent_load(self, pid): 3462 return pid 3463 3464 def _check_return_correct_type(self, obj, proto): 3465 unpickled = self.loads(self.dumps(obj, proto)) 3466 self.assertIsInstance(unpickled, type(obj)) 3467 self.assertEqual(unpickled, obj) 3468 3469 def test_return_correct_type(self): 3470 for proto in protocols: 3471 # Protocol 0 supports only ASCII strings. 3472 if proto == 0: 3473 self._check_return_correct_type("abc", 0) 3474 else: 3475 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]: 3476 self._check_return_correct_type(obj, proto) 3477 3478 def test_protocol0_is_ascii_only(self): 3479 non_ascii_str = "\N{EMPTY SET}" 3480 self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) 3481 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' 3482 self.assertRaises(pickle.UnpicklingError, self.loads, pickled) 3483 3484 3485class AbstractPicklerUnpicklerObjectTests: 3486 3487 pickler_class = None 3488 unpickler_class = None 3489 3490 def setUp(self): 3491 assert self.pickler_class 3492 assert self.unpickler_class 3493 3494 def test_clear_pickler_memo(self): 3495 # To test whether clear_memo() has any effect, we pickle an object, 3496 # then pickle it again without clearing the memo; the two serialized 3497 # forms should be different. If we clear_memo() and then pickle the 3498 # object again, the third serialized form should be identical to the 3499 # first one we obtained. 3500 data = ["abcdefg", "abcdefg", 44] 3501 for proto in protocols: 3502 f = io.BytesIO() 3503 pickler = self.pickler_class(f, proto) 3504 3505 pickler.dump(data) 3506 first_pickled = f.getvalue() 3507 3508 # Reset BytesIO object. 3509 f.seek(0) 3510 f.truncate() 3511 3512 pickler.dump(data) 3513 second_pickled = f.getvalue() 3514 3515 # Reset the Pickler and BytesIO objects. 3516 pickler.clear_memo() 3517 f.seek(0) 3518 f.truncate() 3519 3520 pickler.dump(data) 3521 third_pickled = f.getvalue() 3522 3523 self.assertNotEqual(first_pickled, second_pickled) 3524 self.assertEqual(first_pickled, third_pickled) 3525 3526 def test_priming_pickler_memo(self): 3527 # Verify that we can set the Pickler's memo attribute. 3528 data = ["abcdefg", "abcdefg", 44] 3529 f = io.BytesIO() 3530 pickler = self.pickler_class(f) 3531 3532 pickler.dump(data) 3533 first_pickled = f.getvalue() 3534 3535 f = io.BytesIO() 3536 primed = self.pickler_class(f) 3537 primed.memo = pickler.memo 3538 3539 primed.dump(data) 3540 primed_pickled = f.getvalue() 3541 3542 self.assertNotEqual(first_pickled, primed_pickled) 3543 3544 def test_priming_unpickler_memo(self): 3545 # Verify that we can set the Unpickler's memo attribute. 3546 data = ["abcdefg", "abcdefg", 44] 3547 f = io.BytesIO() 3548 pickler = self.pickler_class(f) 3549 3550 pickler.dump(data) 3551 first_pickled = f.getvalue() 3552 3553 f = io.BytesIO() 3554 primed = self.pickler_class(f) 3555 primed.memo = pickler.memo 3556 3557 primed.dump(data) 3558 primed_pickled = f.getvalue() 3559 3560 unpickler = self.unpickler_class(io.BytesIO(first_pickled)) 3561 unpickled_data1 = unpickler.load() 3562 3563 self.assertEqual(unpickled_data1, data) 3564 3565 primed = self.unpickler_class(io.BytesIO(primed_pickled)) 3566 primed.memo = unpickler.memo 3567 unpickled_data2 = primed.load() 3568 3569 primed.memo.clear() 3570 3571 self.assertEqual(unpickled_data2, data) 3572 self.assertTrue(unpickled_data2 is unpickled_data1) 3573 3574 def test_reusing_unpickler_objects(self): 3575 data1 = ["abcdefg", "abcdefg", 44] 3576 f = io.BytesIO() 3577 pickler = self.pickler_class(f) 3578 pickler.dump(data1) 3579 pickled1 = f.getvalue() 3580 3581 data2 = ["abcdefg", 44, 44] 3582 f = io.BytesIO() 3583 pickler = self.pickler_class(f) 3584 pickler.dump(data2) 3585 pickled2 = f.getvalue() 3586 3587 f = io.BytesIO() 3588 f.write(pickled1) 3589 f.seek(0) 3590 unpickler = self.unpickler_class(f) 3591 self.assertEqual(unpickler.load(), data1) 3592 3593 f.seek(0) 3594 f.truncate() 3595 f.write(pickled2) 3596 f.seek(0) 3597 self.assertEqual(unpickler.load(), data2) 3598 3599 def _check_multiple_unpicklings(self, ioclass, *, seekable=True): 3600 for proto in protocols: 3601 with self.subTest(proto=proto): 3602 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len] 3603 f = ioclass() 3604 pickler = self.pickler_class(f, protocol=proto) 3605 pickler.dump(data1) 3606 pickled = f.getvalue() 3607 3608 N = 5 3609 f = ioclass(pickled * N) 3610 unpickler = self.unpickler_class(f) 3611 for i in range(N): 3612 if seekable: 3613 pos = f.tell() 3614 self.assertEqual(unpickler.load(), data1) 3615 if seekable: 3616 self.assertEqual(f.tell(), pos + len(pickled)) 3617 self.assertRaises(EOFError, unpickler.load) 3618 3619 def test_multiple_unpicklings_seekable(self): 3620 self._check_multiple_unpicklings(io.BytesIO) 3621 3622 def test_multiple_unpicklings_unseekable(self): 3623 self._check_multiple_unpicklings(UnseekableIO, seekable=False) 3624 3625 def test_multiple_unpicklings_minimal(self): 3626 # File-like object that doesn't support peek() and readinto() 3627 # (bpo-39681) 3628 self._check_multiple_unpicklings(MinimalIO, seekable=False) 3629 3630 def test_unpickling_buffering_readline(self): 3631 # Issue #12687: the unpickler's buffering logic could fail with 3632 # text mode opcodes. 3633 data = list(range(10)) 3634 for proto in protocols: 3635 for buf_size in range(1, 11): 3636 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size) 3637 pickler = self.pickler_class(f, protocol=proto) 3638 pickler.dump(data) 3639 f.seek(0) 3640 unpickler = self.unpickler_class(f) 3641 self.assertEqual(unpickler.load(), data) 3642 3643 3644# Tests for dispatch_table attribute 3645 3646REDUCE_A = 'reduce_A' 3647 3648class AAA(object): 3649 def __reduce__(self): 3650 return str, (REDUCE_A,) 3651 3652class BBB(object): 3653 def __init__(self): 3654 # Add an instance attribute to enable state-saving routines at pickling 3655 # time. 3656 self.a = "some attribute" 3657 3658 def __setstate__(self, state): 3659 self.a = "BBB.__setstate__" 3660 3661 3662def setstate_bbb(obj, state): 3663 """Custom state setter for BBB objects 3664 3665 Such callable may be created by other persons than the ones who created the 3666 BBB class. If passed as the state_setter item of a custom reducer, this 3667 allows for custom state setting behavior of BBB objects. One can think of 3668 it as the analogous of list_setitems or dict_setitems but for foreign 3669 classes/functions. 3670 """ 3671 obj.a = "custom state_setter" 3672 3673 3674 3675class AbstractCustomPicklerClass: 3676 """Pickler implementing a reducing hook using reducer_override.""" 3677 def reducer_override(self, obj): 3678 obj_name = getattr(obj, "__name__", None) 3679 3680 if obj_name == 'f': 3681 # asking the pickler to save f as 5 3682 return int, (5, ) 3683 3684 if obj_name == 'MyClass': 3685 return str, ('some str',) 3686 3687 elif obj_name == 'g': 3688 # in this case, the callback returns an invalid result (not a 2-5 3689 # tuple or a string), the pickler should raise a proper error. 3690 return False 3691 3692 elif obj_name == 'h': 3693 # Simulate a case when the reducer fails. The error should 3694 # be propagated to the original ``dump`` call. 3695 raise ValueError('The reducer just failed') 3696 3697 return NotImplemented 3698 3699class AbstractHookTests: 3700 def test_pickler_hook(self): 3701 # test the ability of a custom, user-defined CPickler subclass to 3702 # override the default reducing routines of any type using the method 3703 # reducer_override 3704 3705 def f(): 3706 pass 3707 3708 def g(): 3709 pass 3710 3711 def h(): 3712 pass 3713 3714 class MyClass: 3715 pass 3716 3717 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3718 with self.subTest(proto=proto): 3719 bio = io.BytesIO() 3720 p = self.pickler_class(bio, proto) 3721 3722 p.dump([f, MyClass, math.log]) 3723 new_f, some_str, math_log = pickle.loads(bio.getvalue()) 3724 3725 self.assertEqual(new_f, 5) 3726 self.assertEqual(some_str, 'some str') 3727 # math.log does not have its usual reducer overridden, so the 3728 # custom reduction callback should silently direct the pickler 3729 # to the default pickling by attribute, by returning 3730 # NotImplemented 3731 self.assertIs(math_log, math.log) 3732 3733 with self.assertRaises(pickle.PicklingError): 3734 p.dump(g) 3735 3736 with self.assertRaisesRegex( 3737 ValueError, 'The reducer just failed'): 3738 p.dump(h) 3739 3740 @support.cpython_only 3741 def test_reducer_override_no_reference_cycle(self): 3742 # bpo-39492: reducer_override used to induce a spurious reference cycle 3743 # inside the Pickler object, that could prevent all serialized objects 3744 # from being garbage-collected without explicitly invoking gc.collect. 3745 3746 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3747 with self.subTest(proto=proto): 3748 def f(): 3749 pass 3750 3751 wr = weakref.ref(f) 3752 3753 bio = io.BytesIO() 3754 p = self.pickler_class(bio, proto) 3755 p.dump(f) 3756 new_f = pickle.loads(bio.getvalue()) 3757 assert new_f == 5 3758 3759 del p 3760 del f 3761 3762 self.assertIsNone(wr()) 3763 3764 3765class AbstractDispatchTableTests: 3766 3767 def test_default_dispatch_table(self): 3768 # No dispatch_table attribute by default 3769 f = io.BytesIO() 3770 p = self.pickler_class(f, 0) 3771 with self.assertRaises(AttributeError): 3772 p.dispatch_table 3773 self.assertFalse(hasattr(p, 'dispatch_table')) 3774 3775 def test_class_dispatch_table(self): 3776 # A dispatch_table attribute can be specified class-wide 3777 dt = self.get_dispatch_table() 3778 3779 class MyPickler(self.pickler_class): 3780 dispatch_table = dt 3781 3782 def dumps(obj, protocol=None): 3783 f = io.BytesIO() 3784 p = MyPickler(f, protocol) 3785 self.assertEqual(p.dispatch_table, dt) 3786 p.dump(obj) 3787 return f.getvalue() 3788 3789 self._test_dispatch_table(dumps, dt) 3790 3791 def test_instance_dispatch_table(self): 3792 # A dispatch_table attribute can also be specified instance-wide 3793 dt = self.get_dispatch_table() 3794 3795 def dumps(obj, protocol=None): 3796 f = io.BytesIO() 3797 p = self.pickler_class(f, protocol) 3798 p.dispatch_table = dt 3799 self.assertEqual(p.dispatch_table, dt) 3800 p.dump(obj) 3801 return f.getvalue() 3802 3803 self._test_dispatch_table(dumps, dt) 3804 3805 def _test_dispatch_table(self, dumps, dispatch_table): 3806 def custom_load_dump(obj): 3807 return pickle.loads(dumps(obj, 0)) 3808 3809 def default_load_dump(obj): 3810 return pickle.loads(pickle.dumps(obj, 0)) 3811 3812 # pickling complex numbers using protocol 0 relies on copyreg 3813 # so check pickling a complex number still works 3814 z = 1 + 2j 3815 self.assertEqual(custom_load_dump(z), z) 3816 self.assertEqual(default_load_dump(z), z) 3817 3818 # modify pickling of complex 3819 REDUCE_1 = 'reduce_1' 3820 def reduce_1(obj): 3821 return str, (REDUCE_1,) 3822 dispatch_table[complex] = reduce_1 3823 self.assertEqual(custom_load_dump(z), REDUCE_1) 3824 self.assertEqual(default_load_dump(z), z) 3825 3826 # check picklability of AAA and BBB 3827 a = AAA() 3828 b = BBB() 3829 self.assertEqual(custom_load_dump(a), REDUCE_A) 3830 self.assertIsInstance(custom_load_dump(b), BBB) 3831 self.assertEqual(default_load_dump(a), REDUCE_A) 3832 self.assertIsInstance(default_load_dump(b), BBB) 3833 3834 # modify pickling of BBB 3835 dispatch_table[BBB] = reduce_1 3836 self.assertEqual(custom_load_dump(a), REDUCE_A) 3837 self.assertEqual(custom_load_dump(b), REDUCE_1) 3838 self.assertEqual(default_load_dump(a), REDUCE_A) 3839 self.assertIsInstance(default_load_dump(b), BBB) 3840 3841 # revert pickling of BBB and modify pickling of AAA 3842 REDUCE_2 = 'reduce_2' 3843 def reduce_2(obj): 3844 return str, (REDUCE_2,) 3845 dispatch_table[AAA] = reduce_2 3846 del dispatch_table[BBB] 3847 self.assertEqual(custom_load_dump(a), REDUCE_2) 3848 self.assertIsInstance(custom_load_dump(b), BBB) 3849 self.assertEqual(default_load_dump(a), REDUCE_A) 3850 self.assertIsInstance(default_load_dump(b), BBB) 3851 3852 # End-to-end testing of save_reduce with the state_setter keyword 3853 # argument. This is a dispatch_table test as the primary goal of 3854 # state_setter is to tweak objects reduction behavior. 3855 # In particular, state_setter is useful when the default __setstate__ 3856 # behavior is not flexible enough. 3857 3858 # No custom reducer for b has been registered for now, so 3859 # BBB.__setstate__ should be used at unpickling time 3860 self.assertEqual(default_load_dump(b).a, "BBB.__setstate__") 3861 3862 def reduce_bbb(obj): 3863 return BBB, (), obj.__dict__, None, None, setstate_bbb 3864 3865 dispatch_table[BBB] = reduce_bbb 3866 3867 # The custom reducer reduce_bbb includes a state setter, that should 3868 # have priority over BBB.__setstate__ 3869 self.assertEqual(custom_load_dump(b).a, "custom state_setter") 3870 3871 3872if __name__ == "__main__": 3873 # Print some stuff that can be used to rewrite DATA{0,1,2} 3874 from pickletools import dis 3875 x = create_data() 3876 for i in range(pickle.HIGHEST_PROTOCOL+1): 3877 p = pickle.dumps(x, i) 3878 print("DATA{0} = (".format(i)) 3879 for j in range(0, len(p), 20): 3880 b = bytes(p[j:j+20]) 3881 print(" {0!r}".format(b)) 3882 print(")") 3883 print() 3884 print("# Disassembly of DATA{0}".format(i)) 3885 print("DATA{0}_DIS = \"\"\"\\".format(i)) 3886 dis(p) 3887 print("\"\"\"") 3888 print() 3889