1import collections 2import copyreg 3import dbm 4import io 5import functools 6import os 7import math 8import pickle 9import pickletools 10import shutil 11import struct 12import sys 13import threading 14import unittest 15import weakref 16from textwrap import dedent 17from http.cookies import SimpleCookie 18 19try: 20 import _testbuffer 21except ImportError: 22 _testbuffer = None 23 24from test import support 25from test.support import ( 26 TestFailed, TESTFN, run_with_locale, no_tracing, 27 _2G, _4G, bigmemtest, reap_threads, forget, 28 save_restore_warnings_filters 29 ) 30 31from pickle import bytes_types 32 33 34# bpo-41003: Save/restore warnings filters to leave them unchanged. 35# Ignore filters installed by numpy. 36try: 37 with save_restore_warnings_filters(): 38 import numpy as np 39except ImportError: 40 np = None 41 42 43requires_32b = unittest.skipUnless(sys.maxsize < 2**32, 44 "test is only meaningful on 32-bit builds") 45 46# Tests that try a number of pickle protocols should have a 47# for proto in protocols: 48# kind of outer loop. 49protocols = range(pickle.HIGHEST_PROTOCOL + 1) 50 51 52# Return True if opcode code appears in the pickle, else False. 53def opcode_in_pickle(code, pickle): 54 for op, dummy, dummy in pickletools.genops(pickle): 55 if op.code == code.decode("latin-1"): 56 return True 57 return False 58 59# Return the number of times opcode code appears in pickle. 60def count_opcode(code, pickle): 61 n = 0 62 for op, dummy, dummy in pickletools.genops(pickle): 63 if op.code == code.decode("latin-1"): 64 n += 1 65 return n 66 67 68def identity(x): 69 return x 70 71 72class UnseekableIO(io.BytesIO): 73 def peek(self, *args): 74 raise NotImplementedError 75 76 def seekable(self): 77 return False 78 79 def seek(self, *args): 80 raise io.UnsupportedOperation 81 82 def tell(self): 83 raise io.UnsupportedOperation 84 85 86class MinimalIO(object): 87 """ 88 A file-like object that doesn't support readinto(). 89 """ 90 def __init__(self, *args): 91 self._bio = io.BytesIO(*args) 92 self.getvalue = self._bio.getvalue 93 self.read = self._bio.read 94 self.readline = self._bio.readline 95 self.write = self._bio.write 96 97 98# We can't very well test the extension registry without putting known stuff 99# in it, but we have to be careful to restore its original state. Code 100# should do this: 101# 102# e = ExtensionSaver(extension_code) 103# try: 104# fiddle w/ the extension registry's stuff for extension_code 105# finally: 106# e.restore() 107 108class ExtensionSaver: 109 # Remember current registration for code (if any), and remove it (if 110 # there is one). 111 def __init__(self, code): 112 self.code = code 113 if code in copyreg._inverted_registry: 114 self.pair = copyreg._inverted_registry[code] 115 copyreg.remove_extension(self.pair[0], self.pair[1], code) 116 else: 117 self.pair = None 118 119 # Restore previous registration for code. 120 def restore(self): 121 code = self.code 122 curpair = copyreg._inverted_registry.get(code) 123 if curpair is not None: 124 copyreg.remove_extension(curpair[0], curpair[1], code) 125 pair = self.pair 126 if pair is not None: 127 copyreg.add_extension(pair[0], pair[1], code) 128 129class C: 130 def __eq__(self, other): 131 return self.__dict__ == other.__dict__ 132 133class D(C): 134 def __init__(self, arg): 135 pass 136 137class E(C): 138 def __getinitargs__(self): 139 return () 140 141# Simple mutable object. 142class Object: 143 pass 144 145# Hashable immutable key object containing unheshable mutable data. 146class K: 147 def __init__(self, value): 148 self.value = value 149 150 def __reduce__(self): 151 # Shouldn't support the recursion itself 152 return K, (self.value,) 153 154import __main__ 155__main__.C = C 156C.__module__ = "__main__" 157__main__.D = D 158D.__module__ = "__main__" 159__main__.E = E 160E.__module__ = "__main__" 161 162class myint(int): 163 def __init__(self, x): 164 self.str = str(x) 165 166class initarg(C): 167 168 def __init__(self, a, b): 169 self.a = a 170 self.b = b 171 172 def __getinitargs__(self): 173 return self.a, self.b 174 175class metaclass(type): 176 pass 177 178class use_metaclass(object, metaclass=metaclass): 179 pass 180 181class pickling_metaclass(type): 182 def __eq__(self, other): 183 return (type(self) == type(other) and 184 self.reduce_args == other.reduce_args) 185 186 def __reduce__(self): 187 return (create_dynamic_class, self.reduce_args) 188 189def create_dynamic_class(name, bases): 190 result = pickling_metaclass(name, bases, dict()) 191 result.reduce_args = (name, bases) 192 return result 193 194 195class ZeroCopyBytes(bytes): 196 readonly = True 197 c_contiguous = True 198 f_contiguous = True 199 zero_copy_reconstruct = True 200 201 def __reduce_ex__(self, protocol): 202 if protocol >= 5: 203 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 204 else: 205 return type(self)._reconstruct, (bytes(self),) 206 207 def __repr__(self): 208 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 209 210 __str__ = __repr__ 211 212 @classmethod 213 def _reconstruct(cls, obj): 214 with memoryview(obj) as m: 215 obj = m.obj 216 if type(obj) is cls: 217 # Zero-copy 218 return obj 219 else: 220 return cls(obj) 221 222 223class ZeroCopyBytearray(bytearray): 224 readonly = False 225 c_contiguous = True 226 f_contiguous = True 227 zero_copy_reconstruct = True 228 229 def __reduce_ex__(self, protocol): 230 if protocol >= 5: 231 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 232 else: 233 return type(self)._reconstruct, (bytes(self),) 234 235 def __repr__(self): 236 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 237 238 __str__ = __repr__ 239 240 @classmethod 241 def _reconstruct(cls, obj): 242 with memoryview(obj) as m: 243 obj = m.obj 244 if type(obj) is cls: 245 # Zero-copy 246 return obj 247 else: 248 return cls(obj) 249 250 251if _testbuffer is not None: 252 253 class PicklableNDArray: 254 # A not-really-zero-copy picklable ndarray, as the ndarray() 255 # constructor doesn't allow for it 256 257 zero_copy_reconstruct = False 258 259 def __init__(self, *args, **kwargs): 260 self.array = _testbuffer.ndarray(*args, **kwargs) 261 262 def __getitem__(self, idx): 263 cls = type(self) 264 new = cls.__new__(cls) 265 new.array = self.array[idx] 266 return new 267 268 @property 269 def readonly(self): 270 return self.array.readonly 271 272 @property 273 def c_contiguous(self): 274 return self.array.c_contiguous 275 276 @property 277 def f_contiguous(self): 278 return self.array.f_contiguous 279 280 def __eq__(self, other): 281 if not isinstance(other, PicklableNDArray): 282 return NotImplemented 283 return (other.array.format == self.array.format and 284 other.array.shape == self.array.shape and 285 other.array.strides == self.array.strides and 286 other.array.readonly == self.array.readonly and 287 other.array.tobytes() == self.array.tobytes()) 288 289 def __ne__(self, other): 290 if not isinstance(other, PicklableNDArray): 291 return NotImplemented 292 return not (self == other) 293 294 def __repr__(self): 295 return (f"{type(self)}(shape={self.array.shape}," 296 f"strides={self.array.strides}, " 297 f"bytes={self.array.tobytes()})") 298 299 def __reduce_ex__(self, protocol): 300 if not self.array.contiguous: 301 raise NotImplementedError("Reconstructing a non-contiguous " 302 "ndarray does not seem possible") 303 ndarray_kwargs = {"shape": self.array.shape, 304 "strides": self.array.strides, 305 "format": self.array.format, 306 "flags": (0 if self.readonly 307 else _testbuffer.ND_WRITABLE)} 308 pb = pickle.PickleBuffer(self.array) 309 if protocol >= 5: 310 return (type(self)._reconstruct, 311 (pb, ndarray_kwargs)) 312 else: 313 # Need to serialize the bytes in physical order 314 with pb.raw() as m: 315 return (type(self)._reconstruct, 316 (m.tobytes(), ndarray_kwargs)) 317 318 @classmethod 319 def _reconstruct(cls, obj, kwargs): 320 with memoryview(obj) as m: 321 # For some reason, ndarray() wants a list of integers... 322 # XXX This only works if format == 'B' 323 items = list(m.tobytes()) 324 return cls(items, **kwargs) 325 326 327# DATA0 .. DATA4 are the pickles we expect under the various protocols, for 328# the object returned by create_data(). 329 330DATA0 = ( 331 b'(lp0\nL0L\naL1L\naF2.0\n' 332 b'ac__builtin__\ncomple' 333 b'x\np1\n(F3.0\nF0.0\ntp2\n' 334 b'Rp3\naL1L\naL-1L\naL255' 335 b'L\naL-255L\naL-256L\naL' 336 b'65535L\naL-65535L\naL-' 337 b'65536L\naL2147483647L' 338 b'\naL-2147483647L\naL-2' 339 b'147483648L\na(Vabc\np4' 340 b'\ng4\nccopy_reg\n_recon' 341 b'structor\np5\n(c__main' 342 b'__\nC\np6\nc__builtin__' 343 b'\nobject\np7\nNtp8\nRp9\n' 344 b'(dp10\nVfoo\np11\nL1L\ns' 345 b'Vbar\np12\nL2L\nsbg9\ntp' 346 b'13\nag13\naL5L\na.' 347) 348 349# Disassembly of DATA0 350DATA0_DIS = """\ 351 0: ( MARK 352 1: l LIST (MARK at 0) 353 2: p PUT 0 354 5: L LONG 0 355 9: a APPEND 356 10: L LONG 1 357 14: a APPEND 358 15: F FLOAT 2.0 359 20: a APPEND 360 21: c GLOBAL '__builtin__ complex' 361 42: p PUT 1 362 45: ( MARK 363 46: F FLOAT 3.0 364 51: F FLOAT 0.0 365 56: t TUPLE (MARK at 45) 366 57: p PUT 2 367 60: R REDUCE 368 61: p PUT 3 369 64: a APPEND 370 65: L LONG 1 371 69: a APPEND 372 70: L LONG -1 373 75: a APPEND 374 76: L LONG 255 375 82: a APPEND 376 83: L LONG -255 377 90: a APPEND 378 91: L LONG -256 379 98: a APPEND 380 99: L LONG 65535 381 107: a APPEND 382 108: L LONG -65535 383 117: a APPEND 384 118: L LONG -65536 385 127: a APPEND 386 128: L LONG 2147483647 387 141: a APPEND 388 142: L LONG -2147483647 389 156: a APPEND 390 157: L LONG -2147483648 391 171: a APPEND 392 172: ( MARK 393 173: V UNICODE 'abc' 394 178: p PUT 4 395 181: g GET 4 396 184: c GLOBAL 'copy_reg _reconstructor' 397 209: p PUT 5 398 212: ( MARK 399 213: c GLOBAL '__main__ C' 400 225: p PUT 6 401 228: c GLOBAL '__builtin__ object' 402 248: p PUT 7 403 251: N NONE 404 252: t TUPLE (MARK at 212) 405 253: p PUT 8 406 256: R REDUCE 407 257: p PUT 9 408 260: ( MARK 409 261: d DICT (MARK at 260) 410 262: p PUT 10 411 266: V UNICODE 'foo' 412 271: p PUT 11 413 275: L LONG 1 414 279: s SETITEM 415 280: V UNICODE 'bar' 416 285: p PUT 12 417 289: L LONG 2 418 293: s SETITEM 419 294: b BUILD 420 295: g GET 9 421 298: t TUPLE (MARK at 172) 422 299: p PUT 13 423 303: a APPEND 424 304: g GET 13 425 308: a APPEND 426 309: L LONG 5 427 313: a APPEND 428 314: . STOP 429highest protocol among opcodes = 0 430""" 431 432DATA1 = ( 433 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__' 434 b'builtin__\ncomplex\nq\x01' 435 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t' 436 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ' 437 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff' 438 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab' 439 b'cq\x04h\x04ccopy_reg\n_reco' 440 b'nstructor\nq\x05(c__main' 441 b'__\nC\nq\x06c__builtin__\n' 442 b'object\nq\x07Ntq\x08Rq\t}q\n(' 443 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar' 444 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.' 445) 446 447# Disassembly of DATA1 448DATA1_DIS = """\ 449 0: ] EMPTY_LIST 450 1: q BINPUT 0 451 3: ( MARK 452 4: K BININT1 0 453 6: K BININT1 1 454 8: G BINFLOAT 2.0 455 17: c GLOBAL '__builtin__ complex' 456 38: q BINPUT 1 457 40: ( MARK 458 41: G BINFLOAT 3.0 459 50: G BINFLOAT 0.0 460 59: t TUPLE (MARK at 40) 461 60: q BINPUT 2 462 62: R REDUCE 463 63: q BINPUT 3 464 65: K BININT1 1 465 67: J BININT -1 466 72: K BININT1 255 467 74: J BININT -255 468 79: J BININT -256 469 84: M BININT2 65535 470 87: J BININT -65535 471 92: J BININT -65536 472 97: J BININT 2147483647 473 102: J BININT -2147483647 474 107: J BININT -2147483648 475 112: ( MARK 476 113: X BINUNICODE 'abc' 477 121: q BINPUT 4 478 123: h BINGET 4 479 125: c GLOBAL 'copy_reg _reconstructor' 480 150: q BINPUT 5 481 152: ( MARK 482 153: c GLOBAL '__main__ C' 483 165: q BINPUT 6 484 167: c GLOBAL '__builtin__ object' 485 187: q BINPUT 7 486 189: N NONE 487 190: t TUPLE (MARK at 152) 488 191: q BINPUT 8 489 193: R REDUCE 490 194: q BINPUT 9 491 196: } EMPTY_DICT 492 197: q BINPUT 10 493 199: ( MARK 494 200: X BINUNICODE 'foo' 495 208: q BINPUT 11 496 210: K BININT1 1 497 212: X BINUNICODE 'bar' 498 220: q BINPUT 12 499 222: K BININT1 2 500 224: u SETITEMS (MARK at 199) 501 225: b BUILD 502 226: h BINGET 9 503 228: t TUPLE (MARK at 112) 504 229: q BINPUT 13 505 231: h BINGET 13 506 233: K BININT1 5 507 235: e APPENDS (MARK at 3) 508 236: . STOP 509highest protocol among opcodes = 1 510""" 511 512DATA2 = ( 513 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 514 b'__builtin__\ncomplex\n' 515 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00' 516 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff' 517 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff' 518 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a' 519 b'bcq\x04h\x04c__main__\nC\nq\x05' 520 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01' 521 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh' 522 b'\nK\x05e.' 523) 524 525# Disassembly of DATA2 526DATA2_DIS = """\ 527 0: \x80 PROTO 2 528 2: ] EMPTY_LIST 529 3: q BINPUT 0 530 5: ( MARK 531 6: K BININT1 0 532 8: K BININT1 1 533 10: G BINFLOAT 2.0 534 19: c GLOBAL '__builtin__ complex' 535 40: q BINPUT 1 536 42: G BINFLOAT 3.0 537 51: G BINFLOAT 0.0 538 60: \x86 TUPLE2 539 61: q BINPUT 2 540 63: R REDUCE 541 64: q BINPUT 3 542 66: K BININT1 1 543 68: J BININT -1 544 73: K BININT1 255 545 75: J BININT -255 546 80: J BININT -256 547 85: M BININT2 65535 548 88: J BININT -65535 549 93: J BININT -65536 550 98: J BININT 2147483647 551 103: J BININT -2147483647 552 108: J BININT -2147483648 553 113: ( MARK 554 114: X BINUNICODE 'abc' 555 122: q BINPUT 4 556 124: h BINGET 4 557 126: c GLOBAL '__main__ C' 558 138: q BINPUT 5 559 140: ) EMPTY_TUPLE 560 141: \x81 NEWOBJ 561 142: q BINPUT 6 562 144: } EMPTY_DICT 563 145: q BINPUT 7 564 147: ( MARK 565 148: X BINUNICODE 'foo' 566 156: q BINPUT 8 567 158: K BININT1 1 568 160: X BINUNICODE 'bar' 569 168: q BINPUT 9 570 170: K BININT1 2 571 172: u SETITEMS (MARK at 147) 572 173: b BUILD 573 174: h BINGET 6 574 176: t TUPLE (MARK at 113) 575 177: q BINPUT 10 576 179: h BINGET 10 577 181: K BININT1 5 578 183: e APPENDS (MARK at 5) 579 184: . STOP 580highest protocol among opcodes = 2 581""" 582 583DATA3 = ( 584 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 585 b'builtins\ncomplex\nq\x01G' 586 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02' 587 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff' 588 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f' 589 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq' 590 b'\x04h\x04c__main__\nC\nq\x05)\x81q' 591 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00' 592 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05' 593 b'e.' 594) 595 596# Disassembly of DATA3 597DATA3_DIS = """\ 598 0: \x80 PROTO 3 599 2: ] EMPTY_LIST 600 3: q BINPUT 0 601 5: ( MARK 602 6: K BININT1 0 603 8: K BININT1 1 604 10: G BINFLOAT 2.0 605 19: c GLOBAL 'builtins complex' 606 37: q BINPUT 1 607 39: G BINFLOAT 3.0 608 48: G BINFLOAT 0.0 609 57: \x86 TUPLE2 610 58: q BINPUT 2 611 60: R REDUCE 612 61: q BINPUT 3 613 63: K BININT1 1 614 65: J BININT -1 615 70: K BININT1 255 616 72: J BININT -255 617 77: J BININT -256 618 82: M BININT2 65535 619 85: J BININT -65535 620 90: J BININT -65536 621 95: J BININT 2147483647 622 100: J BININT -2147483647 623 105: J BININT -2147483648 624 110: ( MARK 625 111: X BINUNICODE 'abc' 626 119: q BINPUT 4 627 121: h BINGET 4 628 123: c GLOBAL '__main__ C' 629 135: q BINPUT 5 630 137: ) EMPTY_TUPLE 631 138: \x81 NEWOBJ 632 139: q BINPUT 6 633 141: } EMPTY_DICT 634 142: q BINPUT 7 635 144: ( MARK 636 145: X BINUNICODE 'bar' 637 153: q BINPUT 8 638 155: K BININT1 2 639 157: X BINUNICODE 'foo' 640 165: q BINPUT 9 641 167: K BININT1 1 642 169: u SETITEMS (MARK at 144) 643 170: b BUILD 644 171: h BINGET 6 645 173: t TUPLE (MARK at 110) 646 174: q BINPUT 10 647 176: h BINGET 10 648 178: K BININT1 5 649 180: e APPENDS (MARK at 5) 650 181: . STOP 651highest protocol among opcodes = 2 652""" 653 654DATA4 = ( 655 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@' 656 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07' 657 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G' 658 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK' 659 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ' 660 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(' 661 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c' 662 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c' 663 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.' 664) 665 666# Disassembly of DATA4 667DATA4_DIS = """\ 668 0: \x80 PROTO 4 669 2: \x95 FRAME 168 670 11: ] EMPTY_LIST 671 12: \x94 MEMOIZE 672 13: ( MARK 673 14: K BININT1 0 674 16: K BININT1 1 675 18: G BINFLOAT 2.0 676 27: \x8c SHORT_BINUNICODE 'builtins' 677 37: \x94 MEMOIZE 678 38: \x8c SHORT_BINUNICODE 'complex' 679 47: \x94 MEMOIZE 680 48: \x93 STACK_GLOBAL 681 49: \x94 MEMOIZE 682 50: G BINFLOAT 3.0 683 59: G BINFLOAT 0.0 684 68: \x86 TUPLE2 685 69: \x94 MEMOIZE 686 70: R REDUCE 687 71: \x94 MEMOIZE 688 72: K BININT1 1 689 74: J BININT -1 690 79: K BININT1 255 691 81: J BININT -255 692 86: J BININT -256 693 91: M BININT2 65535 694 94: J BININT -65535 695 99: J BININT -65536 696 104: J BININT 2147483647 697 109: J BININT -2147483647 698 114: J BININT -2147483648 699 119: ( MARK 700 120: \x8c SHORT_BINUNICODE 'abc' 701 125: \x94 MEMOIZE 702 126: h BINGET 6 703 128: \x8c SHORT_BINUNICODE '__main__' 704 138: \x94 MEMOIZE 705 139: \x8c SHORT_BINUNICODE 'C' 706 142: \x94 MEMOIZE 707 143: \x93 STACK_GLOBAL 708 144: \x94 MEMOIZE 709 145: ) EMPTY_TUPLE 710 146: \x81 NEWOBJ 711 147: \x94 MEMOIZE 712 148: } EMPTY_DICT 713 149: \x94 MEMOIZE 714 150: ( MARK 715 151: \x8c SHORT_BINUNICODE 'bar' 716 156: \x94 MEMOIZE 717 157: K BININT1 2 718 159: \x8c SHORT_BINUNICODE 'foo' 719 164: \x94 MEMOIZE 720 165: K BININT1 1 721 167: u SETITEMS (MARK at 150) 722 168: b BUILD 723 169: h BINGET 10 724 171: t TUPLE (MARK at 119) 725 172: \x94 MEMOIZE 726 173: h BINGET 14 727 175: K BININT1 5 728 177: e APPENDS (MARK at 13) 729 178: . STOP 730highest protocol among opcodes = 4 731""" 732 733# set([1,2]) pickled from 2.x with protocol 2 734DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.' 735 736# xrange(5) pickled from 2.x with protocol 2 737DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.' 738 739# a SimpleCookie() object pickled from 2.x with protocol 2 740DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key' 741 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U' 742 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07' 743 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U' 744 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b' 745 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.') 746 747# set([3]) pickled from 2.x with protocol 2 748DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' 749 750python2_exceptions_without_args = ( 751 ArithmeticError, 752 AssertionError, 753 AttributeError, 754 BaseException, 755 BufferError, 756 BytesWarning, 757 DeprecationWarning, 758 EOFError, 759 EnvironmentError, 760 Exception, 761 FloatingPointError, 762 FutureWarning, 763 GeneratorExit, 764 IOError, 765 ImportError, 766 ImportWarning, 767 IndentationError, 768 IndexError, 769 KeyError, 770 KeyboardInterrupt, 771 LookupError, 772 MemoryError, 773 NameError, 774 NotImplementedError, 775 OSError, 776 OverflowError, 777 PendingDeprecationWarning, 778 ReferenceError, 779 RuntimeError, 780 RuntimeWarning, 781 # StandardError is gone in Python 3, we map it to Exception 782 StopIteration, 783 SyntaxError, 784 SyntaxWarning, 785 SystemError, 786 SystemExit, 787 TabError, 788 TypeError, 789 UnboundLocalError, 790 UnicodeError, 791 UnicodeWarning, 792 UserWarning, 793 ValueError, 794 Warning, 795 ZeroDivisionError, 796) 797 798exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.' 799 800# UnicodeEncodeError object pickled from 2.x with protocol 2 801DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n' 802 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01' 803 b'U\x03badq\x03tq\x04Rq\x05.') 804 805 806def create_data(): 807 c = C() 808 c.foo = 1 809 c.bar = 2 810 x = [0, 1, 2.0, 3.0+0j] 811 # Append some integer test cases at cPickle.c's internal size 812 # cutoffs. 813 uint1max = 0xff 814 uint2max = 0xffff 815 int4max = 0x7fffffff 816 x.extend([1, -1, 817 uint1max, -uint1max, -uint1max-1, 818 uint2max, -uint2max, -uint2max-1, 819 int4max, -int4max, -int4max-1]) 820 y = ('abc', 'abc', c, c) 821 x.append(y) 822 x.append(y) 823 x.append(5) 824 return x 825 826 827class AbstractUnpickleTests: 828 # Subclass must define self.loads. 829 830 _testdata = create_data() 831 832 def assert_is_copy(self, obj, objcopy, msg=None): 833 """Utility method to verify if two objects are copies of each others. 834 """ 835 if msg is None: 836 msg = "{!r} is not a copy of {!r}".format(obj, objcopy) 837 self.assertEqual(obj, objcopy, msg=msg) 838 self.assertIs(type(obj), type(objcopy), msg=msg) 839 if hasattr(obj, '__dict__'): 840 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg) 841 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg) 842 if hasattr(obj, '__slots__'): 843 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg) 844 for slot in obj.__slots__: 845 self.assertEqual( 846 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg) 847 self.assertEqual(getattr(obj, slot, None), 848 getattr(objcopy, slot, None), msg=msg) 849 850 def check_unpickling_error(self, errors, data): 851 with self.subTest(data=data), \ 852 self.assertRaises(errors): 853 try: 854 self.loads(data) 855 except BaseException as exc: 856 if support.verbose > 1: 857 print('%-32r - %s: %s' % 858 (data, exc.__class__.__name__, exc)) 859 raise 860 861 def test_load_from_data0(self): 862 self.assert_is_copy(self._testdata, self.loads(DATA0)) 863 864 def test_load_from_data1(self): 865 self.assert_is_copy(self._testdata, self.loads(DATA1)) 866 867 def test_load_from_data2(self): 868 self.assert_is_copy(self._testdata, self.loads(DATA2)) 869 870 def test_load_from_data3(self): 871 self.assert_is_copy(self._testdata, self.loads(DATA3)) 872 873 def test_load_from_data4(self): 874 self.assert_is_copy(self._testdata, self.loads(DATA4)) 875 876 def test_load_classic_instance(self): 877 # See issue5180. Test loading 2.x pickles that 878 # contain an instance of old style class. 879 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 880 xname = X.__name__.encode('ascii') 881 # Protocol 0 (text mode pickle): 882 """ 883 0: ( MARK 884 1: i INST '__main__ X' (MARK at 0) 885 13: p PUT 0 886 16: ( MARK 887 17: d DICT (MARK at 16) 888 18: p PUT 1 889 21: b BUILD 890 22: . STOP 891 """ 892 pickle0 = (b"(i__main__\n" 893 b"X\n" 894 b"p0\n" 895 b"(dp1\nb.").replace(b'X', xname) 896 self.assert_is_copy(X(*args), self.loads(pickle0)) 897 898 # Protocol 1 (binary mode pickle) 899 """ 900 0: ( MARK 901 1: c GLOBAL '__main__ X' 902 13: q BINPUT 0 903 15: o OBJ (MARK at 0) 904 16: q BINPUT 1 905 18: } EMPTY_DICT 906 19: q BINPUT 2 907 21: b BUILD 908 22: . STOP 909 """ 910 pickle1 = (b'(c__main__\n' 911 b'X\n' 912 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 913 self.assert_is_copy(X(*args), self.loads(pickle1)) 914 915 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1) 916 """ 917 0: \x80 PROTO 2 918 2: ( MARK 919 3: c GLOBAL '__main__ X' 920 15: q BINPUT 0 921 17: o OBJ (MARK at 2) 922 18: q BINPUT 1 923 20: } EMPTY_DICT 924 21: q BINPUT 2 925 23: b BUILD 926 24: . STOP 927 """ 928 pickle2 = (b'\x80\x02(c__main__\n' 929 b'X\n' 930 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 931 self.assert_is_copy(X(*args), self.loads(pickle2)) 932 933 def test_maxint64(self): 934 maxint64 = (1 << 63) - 1 935 data = b'I' + str(maxint64).encode("ascii") + b'\n.' 936 got = self.loads(data) 937 self.assert_is_copy(maxint64, got) 938 939 # Try too with a bogus literal. 940 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' 941 self.check_unpickling_error(ValueError, data) 942 943 def test_unpickle_from_2x(self): 944 # Unpickle non-trivial data from Python 2.x. 945 loaded = self.loads(DATA_SET) 946 self.assertEqual(loaded, set([1, 2])) 947 loaded = self.loads(DATA_XRANGE) 948 self.assertEqual(type(loaded), type(range(0))) 949 self.assertEqual(list(loaded), list(range(5))) 950 loaded = self.loads(DATA_COOKIE) 951 self.assertEqual(type(loaded), SimpleCookie) 952 self.assertEqual(list(loaded.keys()), ["key"]) 953 self.assertEqual(loaded["key"].value, "value") 954 955 # Exception objects without arguments pickled from 2.x with protocol 2 956 for exc in python2_exceptions_without_args: 957 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii")) 958 loaded = self.loads(data) 959 self.assertIs(type(loaded), exc) 960 961 # StandardError is mapped to Exception, test that separately 962 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError')) 963 self.assertIs(type(loaded), Exception) 964 965 loaded = self.loads(DATA_UEERR) 966 self.assertIs(type(loaded), UnicodeEncodeError) 967 self.assertEqual(loaded.object, "foo") 968 self.assertEqual(loaded.encoding, "ascii") 969 self.assertEqual(loaded.start, 0) 970 self.assertEqual(loaded.end, 1) 971 self.assertEqual(loaded.reason, "bad") 972 973 def test_load_python2_str_as_bytes(self): 974 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) 975 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.", 976 encoding="bytes"), b'a\x00\xa0') 977 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1) 978 self.assertEqual(self.loads(b'U\x03a\x00\xa0.', 979 encoding="bytes"), b'a\x00\xa0') 980 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2) 981 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.', 982 encoding="bytes"), b'a\x00\xa0') 983 984 def test_load_python2_unicode_as_str(self): 985 # From Python 2: pickle.dumps(u'π', protocol=0) 986 self.assertEqual(self.loads(b'V\\u03c0\n.', 987 encoding='bytes'), 'π') 988 # From Python 2: pickle.dumps(u'π', protocol=1) 989 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.', 990 encoding="bytes"), 'π') 991 # From Python 2: pickle.dumps(u'π', protocol=2) 992 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.', 993 encoding="bytes"), 'π') 994 995 def test_load_long_python2_str_as_bytes(self): 996 # From Python 2: pickle.dumps('x' * 300, protocol=1) 997 self.assertEqual(self.loads(pickle.BINSTRING + 998 struct.pack("<I", 300) + 999 b'x' * 300 + pickle.STOP, 1000 encoding='bytes'), b'x' * 300) 1001 1002 def test_constants(self): 1003 self.assertIsNone(self.loads(b'N.')) 1004 self.assertIs(self.loads(b'\x88.'), True) 1005 self.assertIs(self.loads(b'\x89.'), False) 1006 self.assertIs(self.loads(b'I01\n.'), True) 1007 self.assertIs(self.loads(b'I00\n.'), False) 1008 1009 def test_empty_bytestring(self): 1010 # issue 11286 1011 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') 1012 self.assertEqual(empty, '') 1013 1014 def test_short_binbytes(self): 1015 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.' 1016 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1017 1018 def test_binbytes(self): 1019 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.' 1020 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1021 1022 @requires_32b 1023 def test_negative_32b_binbytes(self): 1024 # On 32-bit builds, a BINBYTES of 2**31 or more is refused 1025 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' 1026 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1027 dumped) 1028 1029 @requires_32b 1030 def test_negative_32b_binunicode(self): 1031 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused 1032 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' 1033 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1034 dumped) 1035 1036 def test_short_binunicode(self): 1037 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' 1038 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1039 1040 def test_misc_get(self): 1041 self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0') 1042 self.check_unpickling_error(pickle.UnpicklingError, b'jens:') 1043 self.check_unpickling_error(pickle.UnpicklingError, b'hens:') 1044 self.assert_is_copy([(100,), (100,)], 1045 self.loads(b'((Kdtp0\nh\x00l.))')) 1046 1047 def test_binbytes8(self): 1048 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1049 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1050 1051 def test_binunicode8(self): 1052 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1053 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1054 1055 def test_bytearray8(self): 1056 dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.' 1057 self.assertEqual(self.loads(dumped), bytearray(b'xxx')) 1058 1059 @requires_32b 1060 def test_large_32b_binbytes8(self): 1061 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1062 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1063 dumped) 1064 1065 @requires_32b 1066 def test_large_32b_bytearray8(self): 1067 dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1068 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1069 dumped) 1070 1071 @requires_32b 1072 def test_large_32b_binunicode8(self): 1073 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1074 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1075 dumped) 1076 1077 def test_get(self): 1078 pickled = b'((lp100000\ng100000\nt.' 1079 unpickled = self.loads(pickled) 1080 self.assertEqual(unpickled, ([],)*2) 1081 self.assertIs(unpickled[0], unpickled[1]) 1082 1083 def test_binget(self): 1084 pickled = b'(]q\xffh\xfft.' 1085 unpickled = self.loads(pickled) 1086 self.assertEqual(unpickled, ([],)*2) 1087 self.assertIs(unpickled[0], unpickled[1]) 1088 1089 def test_long_binget(self): 1090 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.' 1091 unpickled = self.loads(pickled) 1092 self.assertEqual(unpickled, ([],)*2) 1093 self.assertIs(unpickled[0], unpickled[1]) 1094 1095 def test_dup(self): 1096 pickled = b'((l2t.' 1097 unpickled = self.loads(pickled) 1098 self.assertEqual(unpickled, ([],)*2) 1099 self.assertIs(unpickled[0], unpickled[1]) 1100 1101 def test_negative_put(self): 1102 # Issue #12847 1103 dumped = b'Va\np-1\n.' 1104 self.check_unpickling_error(ValueError, dumped) 1105 1106 @requires_32b 1107 def test_negative_32b_binput(self): 1108 # Issue #12847 1109 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' 1110 self.check_unpickling_error(ValueError, dumped) 1111 1112 def test_badly_escaped_string(self): 1113 self.check_unpickling_error(ValueError, b"S'\\'\n.") 1114 1115 def test_badly_quoted_string(self): 1116 # Issue #17710 1117 badpickles = [b"S'\n.", 1118 b'S"\n.', 1119 b'S\' \n.', 1120 b'S" \n.', 1121 b'S\'"\n.', 1122 b'S"\'\n.', 1123 b"S' ' \n.", 1124 b'S" " \n.', 1125 b"S ''\n.", 1126 b'S ""\n.', 1127 b'S \n.', 1128 b'S\n.', 1129 b'S.'] 1130 for p in badpickles: 1131 self.check_unpickling_error(pickle.UnpicklingError, p) 1132 1133 def test_correctly_quoted_string(self): 1134 goodpickles = [(b"S''\n.", ''), 1135 (b'S""\n.', ''), 1136 (b'S"\\n"\n.', '\n'), 1137 (b"S'\\n'\n.", '\n')] 1138 for p, expected in goodpickles: 1139 self.assertEqual(self.loads(p), expected) 1140 1141 def test_frame_readline(self): 1142 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.' 1143 # 0: \x80 PROTO 4 1144 # 2: \x95 FRAME 5 1145 # 11: I INT 42 1146 # 15: . STOP 1147 self.assertEqual(self.loads(pickled), 42) 1148 1149 def test_compat_unpickle(self): 1150 # xrange(1, 7) 1151 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' 1152 unpickled = self.loads(pickled) 1153 self.assertIs(type(unpickled), range) 1154 self.assertEqual(unpickled, range(1, 7)) 1155 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6]) 1156 # reduce 1157 pickled = b'\x80\x02c__builtin__\nreduce\n.' 1158 self.assertIs(self.loads(pickled), functools.reduce) 1159 # whichdb.whichdb 1160 pickled = b'\x80\x02cwhichdb\nwhichdb\n.' 1161 self.assertIs(self.loads(pickled), dbm.whichdb) 1162 # Exception(), StandardError() 1163 for name in (b'Exception', b'StandardError'): 1164 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.') 1165 unpickled = self.loads(pickled) 1166 self.assertIs(type(unpickled), Exception) 1167 self.assertEqual(str(unpickled), 'ugh') 1168 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2}) 1169 for name in (b'UserDict', b'IterableUserDict'): 1170 pickled = (b'\x80\x02(cUserDict\n' + name + 1171 b'\no}U\x04data}K\x01K\x02ssb.') 1172 unpickled = self.loads(pickled) 1173 self.assertIs(type(unpickled), collections.UserDict) 1174 self.assertEqual(unpickled, collections.UserDict({1: 2})) 1175 1176 def test_bad_reduce(self): 1177 self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) 1178 self.check_unpickling_error(TypeError, b'N)R.') 1179 self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.') 1180 1181 def test_bad_newobj(self): 1182 error = (pickle.UnpicklingError, TypeError) 1183 self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0) 1184 self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.') 1185 self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.') 1186 1187 def test_bad_newobj_ex(self): 1188 error = (pickle.UnpicklingError, TypeError) 1189 self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0) 1190 self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.') 1191 self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') 1192 self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') 1193 1194 def test_bad_stack(self): 1195 badpickles = [ 1196 b'.', # STOP 1197 b'0', # POP 1198 b'1', # POP_MARK 1199 b'2', # DUP 1200 b'(2', 1201 b'R', # REDUCE 1202 b')R', 1203 b'a', # APPEND 1204 b'Na', 1205 b'b', # BUILD 1206 b'Nb', 1207 b'd', # DICT 1208 b'e', # APPENDS 1209 b'(e', 1210 b'ibuiltins\nlist\n', # INST 1211 b'l', # LIST 1212 b'o', # OBJ 1213 b'(o', 1214 b'p1\n', # PUT 1215 b'q\x00', # BINPUT 1216 b'r\x00\x00\x00\x00', # LONG_BINPUT 1217 b's', # SETITEM 1218 b'Ns', 1219 b'NNs', 1220 b't', # TUPLE 1221 b'u', # SETITEMS 1222 b'(u', 1223 b'}(Nu', 1224 b'\x81', # NEWOBJ 1225 b')\x81', 1226 b'\x85', # TUPLE1 1227 b'\x86', # TUPLE2 1228 b'N\x86', 1229 b'\x87', # TUPLE3 1230 b'N\x87', 1231 b'NN\x87', 1232 b'\x90', # ADDITEMS 1233 b'(\x90', 1234 b'\x91', # FROZENSET 1235 b'\x92', # NEWOBJ_EX 1236 b')}\x92', 1237 b'\x93', # STACK_GLOBAL 1238 b'Vlist\n\x93', 1239 b'\x94', # MEMOIZE 1240 ] 1241 for p in badpickles: 1242 self.check_unpickling_error(self.bad_stack_errors, p) 1243 1244 def test_bad_mark(self): 1245 badpickles = [ 1246 b'N(.', # STOP 1247 b'N(2', # DUP 1248 b'cbuiltins\nlist\n)(R', # REDUCE 1249 b'cbuiltins\nlist\n()R', 1250 b']N(a', # APPEND 1251 # BUILD 1252 b'cbuiltins\nValueError\n)R}(b', 1253 b'cbuiltins\nValueError\n)R(}b', 1254 b'(Nd', # DICT 1255 b'N(p1\n', # PUT 1256 b'N(q\x00', # BINPUT 1257 b'N(r\x00\x00\x00\x00', # LONG_BINPUT 1258 b'}NN(s', # SETITEM 1259 b'}N(Ns', 1260 b'}(NNs', 1261 b'}((u', # SETITEMS 1262 b'cbuiltins\nlist\n)(\x81', # NEWOBJ 1263 b'cbuiltins\nlist\n()\x81', 1264 b'N(\x85', # TUPLE1 1265 b'NN(\x86', # TUPLE2 1266 b'N(N\x86', 1267 b'NNN(\x87', # TUPLE3 1268 b'NN(N\x87', 1269 b'N(NN\x87', 1270 b']((\x90', # ADDITEMS 1271 # NEWOBJ_EX 1272 b'cbuiltins\nlist\n)}(\x92', 1273 b'cbuiltins\nlist\n)(}\x92', 1274 b'cbuiltins\nlist\n()}\x92', 1275 # STACK_GLOBAL 1276 b'Vbuiltins\n(Vlist\n\x93', 1277 b'Vbuiltins\nVlist\n(\x93', 1278 b'N(\x94', # MEMOIZE 1279 ] 1280 for p in badpickles: 1281 self.check_unpickling_error(self.bad_stack_errors, p) 1282 1283 def test_truncated_data(self): 1284 self.check_unpickling_error(EOFError, b'') 1285 self.check_unpickling_error(EOFError, b'N') 1286 badpickles = [ 1287 b'B', # BINBYTES 1288 b'B\x03\x00\x00', 1289 b'B\x03\x00\x00\x00', 1290 b'B\x03\x00\x00\x00ab', 1291 b'C', # SHORT_BINBYTES 1292 b'C\x03', 1293 b'C\x03ab', 1294 b'F', # FLOAT 1295 b'F0.0', 1296 b'F0.00', 1297 b'G', # BINFLOAT 1298 b'G\x00\x00\x00\x00\x00\x00\x00', 1299 b'I', # INT 1300 b'I0', 1301 b'J', # BININT 1302 b'J\x00\x00\x00', 1303 b'K', # BININT1 1304 b'L', # LONG 1305 b'L0', 1306 b'L10', 1307 b'L0L', 1308 b'L10L', 1309 b'M', # BININT2 1310 b'M\x00', 1311 # b'P', # PERSID 1312 # b'Pabc', 1313 b'S', # STRING 1314 b"S'abc'", 1315 b'T', # BINSTRING 1316 b'T\x03\x00\x00', 1317 b'T\x03\x00\x00\x00', 1318 b'T\x03\x00\x00\x00ab', 1319 b'U', # SHORT_BINSTRING 1320 b'U\x03', 1321 b'U\x03ab', 1322 b'V', # UNICODE 1323 b'Vabc', 1324 b'X', # BINUNICODE 1325 b'X\x03\x00\x00', 1326 b'X\x03\x00\x00\x00', 1327 b'X\x03\x00\x00\x00ab', 1328 b'(c', # GLOBAL 1329 b'(cbuiltins', 1330 b'(cbuiltins\n', 1331 b'(cbuiltins\nlist', 1332 b'Ng', # GET 1333 b'Ng0', 1334 b'(i', # INST 1335 b'(ibuiltins', 1336 b'(ibuiltins\n', 1337 b'(ibuiltins\nlist', 1338 b'Nh', # BINGET 1339 b'Nj', # LONG_BINGET 1340 b'Nj\x00\x00\x00', 1341 b'Np', # PUT 1342 b'Np0', 1343 b'Nq', # BINPUT 1344 b'Nr', # LONG_BINPUT 1345 b'Nr\x00\x00\x00', 1346 b'\x80', # PROTO 1347 b'\x82', # EXT1 1348 b'\x83', # EXT2 1349 b'\x84\x01', 1350 b'\x84', # EXT4 1351 b'\x84\x01\x00\x00', 1352 b'\x8a', # LONG1 1353 b'\x8b', # LONG4 1354 b'\x8b\x00\x00\x00', 1355 b'\x8c', # SHORT_BINUNICODE 1356 b'\x8c\x03', 1357 b'\x8c\x03ab', 1358 b'\x8d', # BINUNICODE8 1359 b'\x8d\x03\x00\x00\x00\x00\x00\x00', 1360 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', 1361 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', 1362 b'\x8e', # BINBYTES8 1363 b'\x8e\x03\x00\x00\x00\x00\x00\x00', 1364 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', 1365 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', 1366 b'\x96', # BYTEARRAY8 1367 b'\x96\x03\x00\x00\x00\x00\x00\x00', 1368 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00', 1369 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab', 1370 b'\x95', # FRAME 1371 b'\x95\x02\x00\x00\x00\x00\x00\x00', 1372 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', 1373 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', 1374 ] 1375 for p in badpickles: 1376 self.check_unpickling_error(self.truncated_errors, p) 1377 1378 @reap_threads 1379 def test_unpickle_module_race(self): 1380 # https://bugs.python.org/issue34572 1381 locker_module = dedent(""" 1382 import threading 1383 barrier = threading.Barrier(2) 1384 """) 1385 locking_import_module = dedent(""" 1386 import locker 1387 locker.barrier.wait() 1388 class ToBeUnpickled(object): 1389 pass 1390 """) 1391 1392 os.mkdir(TESTFN) 1393 self.addCleanup(shutil.rmtree, TESTFN) 1394 sys.path.insert(0, TESTFN) 1395 self.addCleanup(sys.path.remove, TESTFN) 1396 with open(os.path.join(TESTFN, "locker.py"), "wb") as f: 1397 f.write(locker_module.encode('utf-8')) 1398 with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f: 1399 f.write(locking_import_module.encode('utf-8')) 1400 self.addCleanup(forget, "locker") 1401 self.addCleanup(forget, "locking_import") 1402 1403 import locker 1404 1405 pickle_bytes = ( 1406 b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.') 1407 1408 # Then try to unpickle two of these simultaneously 1409 # One of them will cause the module import, and we want it to block 1410 # until the other one either: 1411 # - fails (before the patch for this issue) 1412 # - blocks on the import lock for the module, as it should 1413 results = [] 1414 barrier = threading.Barrier(3) 1415 def t(): 1416 # This ensures the threads have all started 1417 # presumably barrier release is faster than thread startup 1418 barrier.wait() 1419 results.append(pickle.loads(pickle_bytes)) 1420 1421 t1 = threading.Thread(target=t) 1422 t2 = threading.Thread(target=t) 1423 t1.start() 1424 t2.start() 1425 1426 barrier.wait() 1427 # could have delay here 1428 locker.barrier.wait() 1429 1430 t1.join() 1431 t2.join() 1432 1433 from locking_import import ToBeUnpickled 1434 self.assertEqual( 1435 [type(x) for x in results], 1436 [ToBeUnpickled] * 2) 1437 1438 1439 1440class AbstractPickleTests: 1441 # Subclass must define self.dumps, self.loads. 1442 1443 optimized = False 1444 1445 _testdata = AbstractUnpickleTests._testdata 1446 1447 def setUp(self): 1448 pass 1449 1450 assert_is_copy = AbstractUnpickleTests.assert_is_copy 1451 1452 def test_misc(self): 1453 # test various datatypes not tested by testdata 1454 for proto in protocols: 1455 x = myint(4) 1456 s = self.dumps(x, proto) 1457 y = self.loads(s) 1458 self.assert_is_copy(x, y) 1459 1460 x = (1, ()) 1461 s = self.dumps(x, proto) 1462 y = self.loads(s) 1463 self.assert_is_copy(x, y) 1464 1465 x = initarg(1, x) 1466 s = self.dumps(x, proto) 1467 y = self.loads(s) 1468 self.assert_is_copy(x, y) 1469 1470 # XXX test __reduce__ protocol? 1471 1472 def test_roundtrip_equality(self): 1473 expected = self._testdata 1474 for proto in protocols: 1475 s = self.dumps(expected, proto) 1476 got = self.loads(s) 1477 self.assert_is_copy(expected, got) 1478 1479 # There are gratuitous differences between pickles produced by 1480 # pickle and cPickle, largely because cPickle starts PUT indices at 1481 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 1482 # there's a comment with an exclamation point there whose meaning 1483 # is a mystery. cPickle also suppresses PUT for objects with a refcount 1484 # of 1. 1485 def dont_test_disassembly(self): 1486 from io import StringIO 1487 from pickletools import dis 1488 1489 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 1490 s = self.dumps(self._testdata, proto) 1491 filelike = StringIO() 1492 dis(s, out=filelike) 1493 got = filelike.getvalue() 1494 self.assertEqual(expected, got) 1495 1496 def _test_recursive_list(self, cls, aslist=identity, minprotocol=0): 1497 # List containing itself. 1498 l = cls() 1499 l.append(l) 1500 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1501 s = self.dumps(l, proto) 1502 x = self.loads(s) 1503 self.assertIsInstance(x, cls) 1504 y = aslist(x) 1505 self.assertEqual(len(y), 1) 1506 self.assertIs(y[0], x) 1507 1508 def test_recursive_list(self): 1509 self._test_recursive_list(list) 1510 1511 def test_recursive_list_subclass(self): 1512 self._test_recursive_list(MyList, minprotocol=2) 1513 1514 def test_recursive_list_like(self): 1515 self._test_recursive_list(REX_six, aslist=lambda x: x.items) 1516 1517 def _test_recursive_tuple_and_list(self, cls, aslist=identity, minprotocol=0): 1518 # Tuple containing a list containing the original tuple. 1519 t = (cls(),) 1520 t[0].append(t) 1521 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1522 s = self.dumps(t, proto) 1523 x = self.loads(s) 1524 self.assertIsInstance(x, tuple) 1525 self.assertEqual(len(x), 1) 1526 self.assertIsInstance(x[0], cls) 1527 y = aslist(x[0]) 1528 self.assertEqual(len(y), 1) 1529 self.assertIs(y[0], x) 1530 1531 # List containing a tuple containing the original list. 1532 t, = t 1533 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1534 s = self.dumps(t, proto) 1535 x = self.loads(s) 1536 self.assertIsInstance(x, cls) 1537 y = aslist(x) 1538 self.assertEqual(len(y), 1) 1539 self.assertIsInstance(y[0], tuple) 1540 self.assertEqual(len(y[0]), 1) 1541 self.assertIs(y[0][0], x) 1542 1543 def test_recursive_tuple_and_list(self): 1544 self._test_recursive_tuple_and_list(list) 1545 1546 def test_recursive_tuple_and_list_subclass(self): 1547 self._test_recursive_tuple_and_list(MyList, minprotocol=2) 1548 1549 def test_recursive_tuple_and_list_like(self): 1550 self._test_recursive_tuple_and_list(REX_six, aslist=lambda x: x.items) 1551 1552 def _test_recursive_dict(self, cls, asdict=identity, minprotocol=0): 1553 # Dict containing itself. 1554 d = cls() 1555 d[1] = d 1556 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1557 s = self.dumps(d, proto) 1558 x = self.loads(s) 1559 self.assertIsInstance(x, cls) 1560 y = asdict(x) 1561 self.assertEqual(list(y.keys()), [1]) 1562 self.assertIs(y[1], x) 1563 1564 def test_recursive_dict(self): 1565 self._test_recursive_dict(dict) 1566 1567 def test_recursive_dict_subclass(self): 1568 self._test_recursive_dict(MyDict, minprotocol=2) 1569 1570 def test_recursive_dict_like(self): 1571 self._test_recursive_dict(REX_seven, asdict=lambda x: x.table) 1572 1573 def _test_recursive_tuple_and_dict(self, cls, asdict=identity, minprotocol=0): 1574 # Tuple containing a dict containing the original tuple. 1575 t = (cls(),) 1576 t[0][1] = t 1577 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1578 s = self.dumps(t, proto) 1579 x = self.loads(s) 1580 self.assertIsInstance(x, tuple) 1581 self.assertEqual(len(x), 1) 1582 self.assertIsInstance(x[0], cls) 1583 y = asdict(x[0]) 1584 self.assertEqual(list(y), [1]) 1585 self.assertIs(y[1], x) 1586 1587 # Dict containing a tuple containing the original dict. 1588 t, = t 1589 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1590 s = self.dumps(t, proto) 1591 x = self.loads(s) 1592 self.assertIsInstance(x, cls) 1593 y = asdict(x) 1594 self.assertEqual(list(y), [1]) 1595 self.assertIsInstance(y[1], tuple) 1596 self.assertEqual(len(y[1]), 1) 1597 self.assertIs(y[1][0], x) 1598 1599 def test_recursive_tuple_and_dict(self): 1600 self._test_recursive_tuple_and_dict(dict) 1601 1602 def test_recursive_tuple_and_dict_subclass(self): 1603 self._test_recursive_tuple_and_dict(MyDict, minprotocol=2) 1604 1605 def test_recursive_tuple_and_dict_like(self): 1606 self._test_recursive_tuple_and_dict(REX_seven, asdict=lambda x: x.table) 1607 1608 def _test_recursive_dict_key(self, cls, asdict=identity, minprotocol=0): 1609 # Dict containing an immutable object (as key) containing the original 1610 # dict. 1611 d = cls() 1612 d[K(d)] = 1 1613 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1614 s = self.dumps(d, proto) 1615 x = self.loads(s) 1616 self.assertIsInstance(x, cls) 1617 y = asdict(x) 1618 self.assertEqual(len(y.keys()), 1) 1619 self.assertIsInstance(list(y.keys())[0], K) 1620 self.assertIs(list(y.keys())[0].value, x) 1621 1622 def test_recursive_dict_key(self): 1623 self._test_recursive_dict_key(dict) 1624 1625 def test_recursive_dict_subclass_key(self): 1626 self._test_recursive_dict_key(MyDict, minprotocol=2) 1627 1628 def test_recursive_dict_like_key(self): 1629 self._test_recursive_dict_key(REX_seven, asdict=lambda x: x.table) 1630 1631 def _test_recursive_tuple_and_dict_key(self, cls, asdict=identity, minprotocol=0): 1632 # Tuple containing a dict containing an immutable object (as key) 1633 # containing the original tuple. 1634 t = (cls(),) 1635 t[0][K(t)] = 1 1636 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1637 s = self.dumps(t, proto) 1638 x = self.loads(s) 1639 self.assertIsInstance(x, tuple) 1640 self.assertEqual(len(x), 1) 1641 self.assertIsInstance(x[0], cls) 1642 y = asdict(x[0]) 1643 self.assertEqual(len(y), 1) 1644 self.assertIsInstance(list(y.keys())[0], K) 1645 self.assertIs(list(y.keys())[0].value, x) 1646 1647 # Dict containing an immutable object (as key) containing a tuple 1648 # containing the original dict. 1649 t, = t 1650 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1651 s = self.dumps(t, proto) 1652 x = self.loads(s) 1653 self.assertIsInstance(x, cls) 1654 y = asdict(x) 1655 self.assertEqual(len(y), 1) 1656 self.assertIsInstance(list(y.keys())[0], K) 1657 self.assertIs(list(y.keys())[0].value[0], x) 1658 1659 def test_recursive_tuple_and_dict_key(self): 1660 self._test_recursive_tuple_and_dict_key(dict) 1661 1662 def test_recursive_tuple_and_dict_subclass_key(self): 1663 self._test_recursive_tuple_and_dict_key(MyDict, minprotocol=2) 1664 1665 def test_recursive_tuple_and_dict_like_key(self): 1666 self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table) 1667 1668 def test_recursive_set(self): 1669 # Set containing an immutable object containing the original set. 1670 y = set() 1671 y.add(K(y)) 1672 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1673 s = self.dumps(y, proto) 1674 x = self.loads(s) 1675 self.assertIsInstance(x, set) 1676 self.assertEqual(len(x), 1) 1677 self.assertIsInstance(list(x)[0], K) 1678 self.assertIs(list(x)[0].value, x) 1679 1680 # Immutable object containing a set containing the original object. 1681 y, = y 1682 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1683 s = self.dumps(y, proto) 1684 x = self.loads(s) 1685 self.assertIsInstance(x, K) 1686 self.assertIsInstance(x.value, set) 1687 self.assertEqual(len(x.value), 1) 1688 self.assertIs(list(x.value)[0], x) 1689 1690 def test_recursive_inst(self): 1691 # Mutable object containing itself. 1692 i = Object() 1693 i.attr = i 1694 for proto in protocols: 1695 s = self.dumps(i, proto) 1696 x = self.loads(s) 1697 self.assertIsInstance(x, Object) 1698 self.assertEqual(dir(x), dir(i)) 1699 self.assertIs(x.attr, x) 1700 1701 def test_recursive_multi(self): 1702 l = [] 1703 d = {1:l} 1704 i = Object() 1705 i.attr = d 1706 l.append(i) 1707 for proto in protocols: 1708 s = self.dumps(l, proto) 1709 x = self.loads(s) 1710 self.assertIsInstance(x, list) 1711 self.assertEqual(len(x), 1) 1712 self.assertEqual(dir(x[0]), dir(i)) 1713 self.assertEqual(list(x[0].attr.keys()), [1]) 1714 self.assertIs(x[0].attr[1], x) 1715 1716 def _test_recursive_collection_and_inst(self, factory): 1717 # Mutable object containing a collection containing the original 1718 # object. 1719 o = Object() 1720 o.attr = factory([o]) 1721 t = type(o.attr) 1722 for proto in protocols: 1723 s = self.dumps(o, proto) 1724 x = self.loads(s) 1725 self.assertIsInstance(x.attr, t) 1726 self.assertEqual(len(x.attr), 1) 1727 self.assertIsInstance(list(x.attr)[0], Object) 1728 self.assertIs(list(x.attr)[0], x) 1729 1730 # Collection containing a mutable object containing the original 1731 # collection. 1732 o = o.attr 1733 for proto in protocols: 1734 s = self.dumps(o, proto) 1735 x = self.loads(s) 1736 self.assertIsInstance(x, t) 1737 self.assertEqual(len(x), 1) 1738 self.assertIsInstance(list(x)[0], Object) 1739 self.assertIs(list(x)[0].attr, x) 1740 1741 def test_recursive_list_and_inst(self): 1742 self._test_recursive_collection_and_inst(list) 1743 1744 def test_recursive_tuple_and_inst(self): 1745 self._test_recursive_collection_and_inst(tuple) 1746 1747 def test_recursive_dict_and_inst(self): 1748 self._test_recursive_collection_and_inst(dict.fromkeys) 1749 1750 def test_recursive_set_and_inst(self): 1751 self._test_recursive_collection_and_inst(set) 1752 1753 def test_recursive_frozenset_and_inst(self): 1754 self._test_recursive_collection_and_inst(frozenset) 1755 1756 def test_recursive_list_subclass_and_inst(self): 1757 self._test_recursive_collection_and_inst(MyList) 1758 1759 def test_recursive_tuple_subclass_and_inst(self): 1760 self._test_recursive_collection_and_inst(MyTuple) 1761 1762 def test_recursive_dict_subclass_and_inst(self): 1763 self._test_recursive_collection_and_inst(MyDict.fromkeys) 1764 1765 def test_recursive_set_subclass_and_inst(self): 1766 self._test_recursive_collection_and_inst(MySet) 1767 1768 def test_recursive_frozenset_subclass_and_inst(self): 1769 self._test_recursive_collection_and_inst(MyFrozenSet) 1770 1771 def test_recursive_inst_state(self): 1772 # Mutable object containing itself. 1773 y = REX_state() 1774 y.state = y 1775 for proto in protocols: 1776 s = self.dumps(y, proto) 1777 x = self.loads(s) 1778 self.assertIsInstance(x, REX_state) 1779 self.assertIs(x.state, x) 1780 1781 def test_recursive_tuple_and_inst_state(self): 1782 # Tuple containing a mutable object containing the original tuple. 1783 t = (REX_state(),) 1784 t[0].state = t 1785 for proto in protocols: 1786 s = self.dumps(t, proto) 1787 x = self.loads(s) 1788 self.assertIsInstance(x, tuple) 1789 self.assertEqual(len(x), 1) 1790 self.assertIsInstance(x[0], REX_state) 1791 self.assertIs(x[0].state, x) 1792 1793 # Mutable object containing a tuple containing the object. 1794 t, = t 1795 for proto in protocols: 1796 s = self.dumps(t, proto) 1797 x = self.loads(s) 1798 self.assertIsInstance(x, REX_state) 1799 self.assertIsInstance(x.state, tuple) 1800 self.assertEqual(len(x.state), 1) 1801 self.assertIs(x.state[0], x) 1802 1803 def test_unicode(self): 1804 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', 1805 '<\\>', '<\\\U00012345>', 1806 # surrogates 1807 '<\udc80>'] 1808 for proto in protocols: 1809 for u in endcases: 1810 p = self.dumps(u, proto) 1811 u2 = self.loads(p) 1812 self.assert_is_copy(u, u2) 1813 1814 def test_unicode_high_plane(self): 1815 t = '\U00012345' 1816 for proto in protocols: 1817 p = self.dumps(t, proto) 1818 t2 = self.loads(p) 1819 self.assert_is_copy(t, t2) 1820 1821 def test_bytes(self): 1822 for proto in protocols: 1823 for s in b'', b'xyz', b'xyz'*100: 1824 p = self.dumps(s, proto) 1825 self.assert_is_copy(s, self.loads(p)) 1826 for s in [bytes([i]) for i in range(256)]: 1827 p = self.dumps(s, proto) 1828 self.assert_is_copy(s, self.loads(p)) 1829 for s in [bytes([i, i]) for i in range(256)]: 1830 p = self.dumps(s, proto) 1831 self.assert_is_copy(s, self.loads(p)) 1832 1833 def test_bytearray(self): 1834 for proto in protocols: 1835 for s in b'', b'xyz', b'xyz'*100: 1836 b = bytearray(s) 1837 p = self.dumps(b, proto) 1838 bb = self.loads(p) 1839 self.assertIsNot(bb, b) 1840 self.assert_is_copy(b, bb) 1841 if proto <= 3: 1842 # bytearray is serialized using a global reference 1843 self.assertIn(b'bytearray', p) 1844 self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p)) 1845 elif proto == 4: 1846 self.assertIn(b'bytearray', p) 1847 self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p)) 1848 elif proto == 5: 1849 self.assertNotIn(b'bytearray', p) 1850 self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) 1851 1852 def test_ints(self): 1853 for proto in protocols: 1854 n = sys.maxsize 1855 while n: 1856 for expected in (-n, n): 1857 s = self.dumps(expected, proto) 1858 n2 = self.loads(s) 1859 self.assert_is_copy(expected, n2) 1860 n = n >> 1 1861 1862 def test_long(self): 1863 for proto in protocols: 1864 # 256 bytes is where LONG4 begins. 1865 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 1866 nbase = 1 << nbits 1867 for npos in nbase-1, nbase, nbase+1: 1868 for n in npos, -npos: 1869 pickle = self.dumps(n, proto) 1870 got = self.loads(pickle) 1871 self.assert_is_copy(n, got) 1872 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 1873 # bother with those. 1874 nbase = int("deadbeeffeedface", 16) 1875 nbase += nbase << 1000000 1876 for n in nbase, -nbase: 1877 p = self.dumps(n, 2) 1878 got = self.loads(p) 1879 # assert_is_copy is very expensive here as it precomputes 1880 # a failure message by computing the repr() of n and got, 1881 # we just do the check ourselves. 1882 self.assertIs(type(got), int) 1883 self.assertEqual(n, got) 1884 1885 def test_float(self): 1886 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 1887 3.14, 263.44582062374053, 6.022e23, 1e30] 1888 test_values = test_values + [-x for x in test_values] 1889 for proto in protocols: 1890 for value in test_values: 1891 pickle = self.dumps(value, proto) 1892 got = self.loads(pickle) 1893 self.assert_is_copy(value, got) 1894 1895 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 1896 def test_float_format(self): 1897 # make sure that floats are formatted locale independent with proto 0 1898 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') 1899 1900 def test_reduce(self): 1901 for proto in protocols: 1902 inst = AAA() 1903 dumped = self.dumps(inst, proto) 1904 loaded = self.loads(dumped) 1905 self.assertEqual(loaded, REDUCE_A) 1906 1907 def test_getinitargs(self): 1908 for proto in protocols: 1909 inst = initarg(1, 2) 1910 dumped = self.dumps(inst, proto) 1911 loaded = self.loads(dumped) 1912 self.assert_is_copy(inst, loaded) 1913 1914 def test_metaclass(self): 1915 a = use_metaclass() 1916 for proto in protocols: 1917 s = self.dumps(a, proto) 1918 b = self.loads(s) 1919 self.assertEqual(a.__class__, b.__class__) 1920 1921 def test_dynamic_class(self): 1922 a = create_dynamic_class("my_dynamic_class", (object,)) 1923 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 1924 for proto in protocols: 1925 s = self.dumps(a, proto) 1926 b = self.loads(s) 1927 self.assertEqual(a, b) 1928 self.assertIs(type(a), type(b)) 1929 1930 def test_structseq(self): 1931 import time 1932 import os 1933 1934 t = time.localtime() 1935 for proto in protocols: 1936 s = self.dumps(t, proto) 1937 u = self.loads(s) 1938 self.assert_is_copy(t, u) 1939 t = os.stat(os.curdir) 1940 s = self.dumps(t, proto) 1941 u = self.loads(s) 1942 self.assert_is_copy(t, u) 1943 if hasattr(os, "statvfs"): 1944 t = os.statvfs(os.curdir) 1945 s = self.dumps(t, proto) 1946 u = self.loads(s) 1947 self.assert_is_copy(t, u) 1948 1949 def test_ellipsis(self): 1950 for proto in protocols: 1951 s = self.dumps(..., proto) 1952 u = self.loads(s) 1953 self.assertIs(..., u) 1954 1955 def test_notimplemented(self): 1956 for proto in protocols: 1957 s = self.dumps(NotImplemented, proto) 1958 u = self.loads(s) 1959 self.assertIs(NotImplemented, u) 1960 1961 def test_singleton_types(self): 1962 # Issue #6477: Test that types of built-in singletons can be pickled. 1963 singletons = [None, ..., NotImplemented] 1964 for singleton in singletons: 1965 for proto in protocols: 1966 s = self.dumps(type(singleton), proto) 1967 u = self.loads(s) 1968 self.assertIs(type(singleton), u) 1969 1970 # Tests for protocol 2 1971 1972 def test_proto(self): 1973 for proto in protocols: 1974 pickled = self.dumps(None, proto) 1975 if proto >= 2: 1976 proto_header = pickle.PROTO + bytes([proto]) 1977 self.assertTrue(pickled.startswith(proto_header)) 1978 else: 1979 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) 1980 1981 oob = protocols[-1] + 1 # a future protocol 1982 build_none = pickle.NONE + pickle.STOP 1983 badpickle = pickle.PROTO + bytes([oob]) + build_none 1984 try: 1985 self.loads(badpickle) 1986 except ValueError as err: 1987 self.assertIn("unsupported pickle protocol", str(err)) 1988 else: 1989 self.fail("expected bad protocol number to raise ValueError") 1990 1991 def test_long1(self): 1992 x = 12345678910111213141516178920 1993 for proto in protocols: 1994 s = self.dumps(x, proto) 1995 y = self.loads(s) 1996 self.assert_is_copy(x, y) 1997 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 1998 1999 def test_long4(self): 2000 x = 12345678910111213141516178920 << (256*8) 2001 for proto in protocols: 2002 s = self.dumps(x, proto) 2003 y = self.loads(s) 2004 self.assert_is_copy(x, y) 2005 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 2006 2007 def test_short_tuples(self): 2008 # Map (proto, len(tuple)) to expected opcode. 2009 expected_opcode = {(0, 0): pickle.TUPLE, 2010 (0, 1): pickle.TUPLE, 2011 (0, 2): pickle.TUPLE, 2012 (0, 3): pickle.TUPLE, 2013 (0, 4): pickle.TUPLE, 2014 2015 (1, 0): pickle.EMPTY_TUPLE, 2016 (1, 1): pickle.TUPLE, 2017 (1, 2): pickle.TUPLE, 2018 (1, 3): pickle.TUPLE, 2019 (1, 4): pickle.TUPLE, 2020 2021 (2, 0): pickle.EMPTY_TUPLE, 2022 (2, 1): pickle.TUPLE1, 2023 (2, 2): pickle.TUPLE2, 2024 (2, 3): pickle.TUPLE3, 2025 (2, 4): pickle.TUPLE, 2026 2027 (3, 0): pickle.EMPTY_TUPLE, 2028 (3, 1): pickle.TUPLE1, 2029 (3, 2): pickle.TUPLE2, 2030 (3, 3): pickle.TUPLE3, 2031 (3, 4): pickle.TUPLE, 2032 } 2033 a = () 2034 b = (1,) 2035 c = (1, 2) 2036 d = (1, 2, 3) 2037 e = (1, 2, 3, 4) 2038 for proto in protocols: 2039 for x in a, b, c, d, e: 2040 s = self.dumps(x, proto) 2041 y = self.loads(s) 2042 self.assert_is_copy(x, y) 2043 expected = expected_opcode[min(proto, 3), len(x)] 2044 self.assertTrue(opcode_in_pickle(expected, s)) 2045 2046 def test_singletons(self): 2047 # Map (proto, singleton) to expected opcode. 2048 expected_opcode = {(0, None): pickle.NONE, 2049 (1, None): pickle.NONE, 2050 (2, None): pickle.NONE, 2051 (3, None): pickle.NONE, 2052 2053 (0, True): pickle.INT, 2054 (1, True): pickle.INT, 2055 (2, True): pickle.NEWTRUE, 2056 (3, True): pickle.NEWTRUE, 2057 2058 (0, False): pickle.INT, 2059 (1, False): pickle.INT, 2060 (2, False): pickle.NEWFALSE, 2061 (3, False): pickle.NEWFALSE, 2062 } 2063 for proto in protocols: 2064 for x in None, False, True: 2065 s = self.dumps(x, proto) 2066 y = self.loads(s) 2067 self.assertTrue(x is y, (proto, x, s, y)) 2068 expected = expected_opcode[min(proto, 3), x] 2069 self.assertTrue(opcode_in_pickle(expected, s)) 2070 2071 def test_newobj_tuple(self): 2072 x = MyTuple([1, 2, 3]) 2073 x.foo = 42 2074 x.bar = "hello" 2075 for proto in protocols: 2076 s = self.dumps(x, proto) 2077 y = self.loads(s) 2078 self.assert_is_copy(x, y) 2079 2080 def test_newobj_list(self): 2081 x = MyList([1, 2, 3]) 2082 x.foo = 42 2083 x.bar = "hello" 2084 for proto in protocols: 2085 s = self.dumps(x, proto) 2086 y = self.loads(s) 2087 self.assert_is_copy(x, y) 2088 2089 def test_newobj_generic(self): 2090 for proto in protocols: 2091 for C in myclasses: 2092 B = C.__base__ 2093 x = C(C.sample) 2094 x.foo = 42 2095 s = self.dumps(x, proto) 2096 y = self.loads(s) 2097 detail = (proto, C, B, x, y, type(y)) 2098 self.assert_is_copy(x, y) # XXX revisit 2099 self.assertEqual(B(x), B(y), detail) 2100 self.assertEqual(x.__dict__, y.__dict__, detail) 2101 2102 def test_newobj_proxies(self): 2103 # NEWOBJ should use the __class__ rather than the raw type 2104 classes = myclasses[:] 2105 # Cannot create weakproxies to these classes 2106 for c in (MyInt, MyTuple): 2107 classes.remove(c) 2108 for proto in protocols: 2109 for C in classes: 2110 B = C.__base__ 2111 x = C(C.sample) 2112 x.foo = 42 2113 p = weakref.proxy(x) 2114 s = self.dumps(p, proto) 2115 y = self.loads(s) 2116 self.assertEqual(type(y), type(x)) # rather than type(p) 2117 detail = (proto, C, B, x, y, type(y)) 2118 self.assertEqual(B(x), B(y), detail) 2119 self.assertEqual(x.__dict__, y.__dict__, detail) 2120 2121 def test_newobj_overridden_new(self): 2122 # Test that Python class with C implemented __new__ is pickleable 2123 for proto in protocols: 2124 x = MyIntWithNew2(1) 2125 x.foo = 42 2126 s = self.dumps(x, proto) 2127 y = self.loads(s) 2128 self.assertIs(type(y), MyIntWithNew2) 2129 self.assertEqual(int(y), 1) 2130 self.assertEqual(y.foo, 42) 2131 2132 def test_newobj_not_class(self): 2133 # Issue 24552 2134 global SimpleNewObj 2135 save = SimpleNewObj 2136 o = SimpleNewObj.__new__(SimpleNewObj) 2137 b = self.dumps(o, 4) 2138 try: 2139 SimpleNewObj = 42 2140 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) 2141 finally: 2142 SimpleNewObj = save 2143 2144 # Register a type with copyreg, with extension code extcode. Pickle 2145 # an object of that type. Check that the resulting pickle uses opcode 2146 # (EXT[124]) under proto 2, and not in proto 1. 2147 2148 def produce_global_ext(self, extcode, opcode): 2149 e = ExtensionSaver(extcode) 2150 try: 2151 copyreg.add_extension(__name__, "MyList", extcode) 2152 x = MyList([1, 2, 3]) 2153 x.foo = 42 2154 x.bar = "hello" 2155 2156 # Dump using protocol 1 for comparison. 2157 s1 = self.dumps(x, 1) 2158 self.assertIn(__name__.encode("utf-8"), s1) 2159 self.assertIn(b"MyList", s1) 2160 self.assertFalse(opcode_in_pickle(opcode, s1)) 2161 2162 y = self.loads(s1) 2163 self.assert_is_copy(x, y) 2164 2165 # Dump using protocol 2 for test. 2166 s2 = self.dumps(x, 2) 2167 self.assertNotIn(__name__.encode("utf-8"), s2) 2168 self.assertNotIn(b"MyList", s2) 2169 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) 2170 2171 y = self.loads(s2) 2172 self.assert_is_copy(x, y) 2173 finally: 2174 e.restore() 2175 2176 def test_global_ext1(self): 2177 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 2178 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 2179 2180 def test_global_ext2(self): 2181 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 2182 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 2183 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 2184 2185 def test_global_ext4(self): 2186 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 2187 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 2188 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 2189 2190 def test_list_chunking(self): 2191 n = 10 # too small to chunk 2192 x = list(range(n)) 2193 for proto in protocols: 2194 s = self.dumps(x, proto) 2195 y = self.loads(s) 2196 self.assert_is_copy(x, y) 2197 num_appends = count_opcode(pickle.APPENDS, s) 2198 self.assertEqual(num_appends, proto > 0) 2199 2200 n = 2500 # expect at least two chunks when proto > 0 2201 x = list(range(n)) 2202 for proto in protocols: 2203 s = self.dumps(x, proto) 2204 y = self.loads(s) 2205 self.assert_is_copy(x, y) 2206 num_appends = count_opcode(pickle.APPENDS, s) 2207 if proto == 0: 2208 self.assertEqual(num_appends, 0) 2209 else: 2210 self.assertTrue(num_appends >= 2) 2211 2212 def test_dict_chunking(self): 2213 n = 10 # too small to chunk 2214 x = dict.fromkeys(range(n)) 2215 for proto in protocols: 2216 s = self.dumps(x, proto) 2217 self.assertIsInstance(s, bytes_types) 2218 y = self.loads(s) 2219 self.assert_is_copy(x, y) 2220 num_setitems = count_opcode(pickle.SETITEMS, s) 2221 self.assertEqual(num_setitems, proto > 0) 2222 2223 n = 2500 # expect at least two chunks when proto > 0 2224 x = dict.fromkeys(range(n)) 2225 for proto in protocols: 2226 s = self.dumps(x, proto) 2227 y = self.loads(s) 2228 self.assert_is_copy(x, y) 2229 num_setitems = count_opcode(pickle.SETITEMS, s) 2230 if proto == 0: 2231 self.assertEqual(num_setitems, 0) 2232 else: 2233 self.assertTrue(num_setitems >= 2) 2234 2235 def test_set_chunking(self): 2236 n = 10 # too small to chunk 2237 x = set(range(n)) 2238 for proto in protocols: 2239 s = self.dumps(x, proto) 2240 y = self.loads(s) 2241 self.assert_is_copy(x, y) 2242 num_additems = count_opcode(pickle.ADDITEMS, s) 2243 if proto < 4: 2244 self.assertEqual(num_additems, 0) 2245 else: 2246 self.assertEqual(num_additems, 1) 2247 2248 n = 2500 # expect at least two chunks when proto >= 4 2249 x = set(range(n)) 2250 for proto in protocols: 2251 s = self.dumps(x, proto) 2252 y = self.loads(s) 2253 self.assert_is_copy(x, y) 2254 num_additems = count_opcode(pickle.ADDITEMS, s) 2255 if proto < 4: 2256 self.assertEqual(num_additems, 0) 2257 else: 2258 self.assertGreaterEqual(num_additems, 2) 2259 2260 def test_simple_newobj(self): 2261 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__ 2262 x.abc = 666 2263 for proto in protocols: 2264 with self.subTest(proto=proto): 2265 s = self.dumps(x, proto) 2266 if proto < 1: 2267 self.assertIn(b'\nI64206', s) # INT 2268 else: 2269 self.assertIn(b'M\xce\xfa', s) # BININT2 2270 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2271 2 <= proto) 2272 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2273 y = self.loads(s) # will raise TypeError if __init__ called 2274 self.assert_is_copy(x, y) 2275 2276 def test_complex_newobj(self): 2277 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__ 2278 x.abc = 666 2279 for proto in protocols: 2280 with self.subTest(proto=proto): 2281 s = self.dumps(x, proto) 2282 if proto < 1: 2283 self.assertIn(b'\nI64206', s) # INT 2284 elif proto < 2: 2285 self.assertIn(b'M\xce\xfa', s) # BININT2 2286 elif proto < 4: 2287 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2288 else: 2289 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2290 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2291 2 <= proto) 2292 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2293 y = self.loads(s) # will raise TypeError if __init__ called 2294 self.assert_is_copy(x, y) 2295 2296 def test_complex_newobj_ex(self): 2297 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ 2298 x.abc = 666 2299 for proto in protocols: 2300 with self.subTest(proto=proto): 2301 s = self.dumps(x, proto) 2302 if proto < 1: 2303 self.assertIn(b'\nI64206', s) # INT 2304 elif proto < 2: 2305 self.assertIn(b'M\xce\xfa', s) # BININT2 2306 elif proto < 4: 2307 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2308 else: 2309 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2310 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) 2311 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), 2312 4 <= proto) 2313 y = self.loads(s) # will raise TypeError if __init__ called 2314 self.assert_is_copy(x, y) 2315 2316 def test_newobj_list_slots(self): 2317 x = SlotList([1, 2, 3]) 2318 x.foo = 42 2319 x.bar = "hello" 2320 s = self.dumps(x, 2) 2321 y = self.loads(s) 2322 self.assert_is_copy(x, y) 2323 2324 def test_reduce_overrides_default_reduce_ex(self): 2325 for proto in protocols: 2326 x = REX_one() 2327 self.assertEqual(x._reduce_called, 0) 2328 s = self.dumps(x, proto) 2329 self.assertEqual(x._reduce_called, 1) 2330 y = self.loads(s) 2331 self.assertEqual(y._reduce_called, 0) 2332 2333 def test_reduce_ex_called(self): 2334 for proto in protocols: 2335 x = REX_two() 2336 self.assertEqual(x._proto, None) 2337 s = self.dumps(x, proto) 2338 self.assertEqual(x._proto, proto) 2339 y = self.loads(s) 2340 self.assertEqual(y._proto, None) 2341 2342 def test_reduce_ex_overrides_reduce(self): 2343 for proto in protocols: 2344 x = REX_three() 2345 self.assertEqual(x._proto, None) 2346 s = self.dumps(x, proto) 2347 self.assertEqual(x._proto, proto) 2348 y = self.loads(s) 2349 self.assertEqual(y._proto, None) 2350 2351 def test_reduce_ex_calls_base(self): 2352 for proto in protocols: 2353 x = REX_four() 2354 self.assertEqual(x._proto, None) 2355 s = self.dumps(x, proto) 2356 self.assertEqual(x._proto, proto) 2357 y = self.loads(s) 2358 self.assertEqual(y._proto, proto) 2359 2360 def test_reduce_calls_base(self): 2361 for proto in protocols: 2362 x = REX_five() 2363 self.assertEqual(x._reduce_called, 0) 2364 s = self.dumps(x, proto) 2365 self.assertEqual(x._reduce_called, 1) 2366 y = self.loads(s) 2367 self.assertEqual(y._reduce_called, 1) 2368 2369 @no_tracing 2370 def test_bad_getattr(self): 2371 # Issue #3514: crash when there is an infinite loop in __getattr__ 2372 x = BadGetattr() 2373 for proto in protocols: 2374 with support.infinite_recursion(): 2375 self.assertRaises(RuntimeError, self.dumps, x, proto) 2376 2377 def test_reduce_bad_iterator(self): 2378 # Issue4176: crash when 4th and 5th items of __reduce__() 2379 # are not iterators 2380 class C(object): 2381 def __reduce__(self): 2382 # 4th item is not an iterator 2383 return list, (), None, [], None 2384 class D(object): 2385 def __reduce__(self): 2386 # 5th item is not an iterator 2387 return dict, (), None, None, [] 2388 2389 # Python implementation is less strict and also accepts iterables. 2390 for proto in protocols: 2391 try: 2392 self.dumps(C(), proto) 2393 except pickle.PicklingError: 2394 pass 2395 try: 2396 self.dumps(D(), proto) 2397 except pickle.PicklingError: 2398 pass 2399 2400 def test_many_puts_and_gets(self): 2401 # Test that internal data structures correctly deal with lots of 2402 # puts/gets. 2403 keys = ("aaa" + str(i) for i in range(100)) 2404 large_dict = dict((k, [4, 5, 6]) for k in keys) 2405 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 2406 2407 for proto in protocols: 2408 with self.subTest(proto=proto): 2409 dumped = self.dumps(obj, proto) 2410 loaded = self.loads(dumped) 2411 self.assert_is_copy(obj, loaded) 2412 2413 def test_attribute_name_interning(self): 2414 # Test that attribute names of pickled objects are interned when 2415 # unpickling. 2416 for proto in protocols: 2417 x = C() 2418 x.foo = 42 2419 x.bar = "hello" 2420 s = self.dumps(x, proto) 2421 y = self.loads(s) 2422 x_keys = sorted(x.__dict__) 2423 y_keys = sorted(y.__dict__) 2424 for x_key, y_key in zip(x_keys, y_keys): 2425 self.assertIs(x_key, y_key) 2426 2427 def test_pickle_to_2x(self): 2428 # Pickle non-trivial data with protocol 2, expecting that it yields 2429 # the same result as Python 2.x did. 2430 # NOTE: this test is a bit too strong since we can produce different 2431 # bytecode that 2.x will still understand. 2432 dumped = self.dumps(range(5), 2) 2433 self.assertEqual(dumped, DATA_XRANGE) 2434 dumped = self.dumps(set([3]), 2) 2435 self.assertEqual(dumped, DATA_SET2) 2436 2437 def test_large_pickles(self): 2438 # Test the correctness of internal buffering routines when handling 2439 # large data. 2440 for proto in protocols: 2441 data = (1, min, b'xy' * (30 * 1024), len) 2442 dumped = self.dumps(data, proto) 2443 loaded = self.loads(dumped) 2444 self.assertEqual(len(loaded), len(data)) 2445 self.assertEqual(loaded, data) 2446 2447 def test_int_pickling_efficiency(self): 2448 # Test compacity of int representation (see issue #12744) 2449 for proto in protocols: 2450 with self.subTest(proto=proto): 2451 pickles = [self.dumps(2**n, proto) for n in range(70)] 2452 sizes = list(map(len, pickles)) 2453 # the size function is monotonic 2454 self.assertEqual(sorted(sizes), sizes) 2455 if proto >= 2: 2456 for p in pickles: 2457 self.assertFalse(opcode_in_pickle(pickle.LONG, p)) 2458 2459 def _check_pickling_with_opcode(self, obj, opcode, proto): 2460 pickled = self.dumps(obj, proto) 2461 self.assertTrue(opcode_in_pickle(opcode, pickled)) 2462 unpickled = self.loads(pickled) 2463 self.assertEqual(obj, unpickled) 2464 2465 def test_appends_on_non_lists(self): 2466 # Issue #17720 2467 obj = REX_six([1, 2, 3]) 2468 for proto in protocols: 2469 if proto == 0: 2470 self._check_pickling_with_opcode(obj, pickle.APPEND, proto) 2471 else: 2472 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 2473 2474 def test_setitems_on_non_dicts(self): 2475 obj = REX_seven({1: -1, 2: -2, 3: -3}) 2476 for proto in protocols: 2477 if proto == 0: 2478 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 2479 else: 2480 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 2481 2482 # Exercise framing (proto >= 4) for significant workloads 2483 2484 FRAME_SIZE_MIN = 4 2485 FRAME_SIZE_TARGET = 64 * 1024 2486 2487 def check_frame_opcodes(self, pickled): 2488 """ 2489 Check the arguments of FRAME opcodes in a protocol 4+ pickle. 2490 2491 Note that binary objects that are larger than FRAME_SIZE_TARGET are not 2492 framed by default and are therefore considered a frame by themselves in 2493 the following consistency check. 2494 """ 2495 frame_end = frameless_start = None 2496 frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8', 2497 'BINUNICODE8', 'BYTEARRAY8'} 2498 for op, arg, pos in pickletools.genops(pickled): 2499 if frame_end is not None: 2500 self.assertLessEqual(pos, frame_end) 2501 if pos == frame_end: 2502 frame_end = None 2503 2504 if frame_end is not None: # framed 2505 self.assertNotEqual(op.name, 'FRAME') 2506 if op.name in frameless_opcodes: 2507 # Only short bytes and str objects should be written 2508 # in a frame 2509 self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET) 2510 2511 else: # not framed 2512 if (op.name == 'FRAME' or 2513 (op.name in frameless_opcodes and 2514 len(arg) > self.FRAME_SIZE_TARGET)): 2515 # Frame or large bytes or str object 2516 if frameless_start is not None: 2517 # Only short data should be written outside of a frame 2518 self.assertLess(pos - frameless_start, 2519 self.FRAME_SIZE_MIN) 2520 frameless_start = None 2521 elif frameless_start is None and op.name != 'PROTO': 2522 frameless_start = pos 2523 2524 if op.name == 'FRAME': 2525 self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN) 2526 frame_end = pos + 9 + arg 2527 2528 pos = len(pickled) 2529 if frame_end is not None: 2530 self.assertEqual(frame_end, pos) 2531 elif frameless_start is not None: 2532 self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) 2533 2534 @support.skip_if_pgo_task 2535 def test_framing_many_objects(self): 2536 obj = list(range(10**5)) 2537 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2538 with self.subTest(proto=proto): 2539 pickled = self.dumps(obj, proto) 2540 unpickled = self.loads(pickled) 2541 self.assertEqual(obj, unpickled) 2542 bytes_per_frame = (len(pickled) / 2543 count_opcode(pickle.FRAME, pickled)) 2544 self.assertGreater(bytes_per_frame, 2545 self.FRAME_SIZE_TARGET / 2) 2546 self.assertLessEqual(bytes_per_frame, 2547 self.FRAME_SIZE_TARGET * 1) 2548 self.check_frame_opcodes(pickled) 2549 2550 def test_framing_large_objects(self): 2551 N = 1024 * 1024 2552 small_items = [[i] for i in range(10)] 2553 obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] 2554 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2555 for fast in [False, True]: 2556 with self.subTest(proto=proto, fast=fast): 2557 if not fast: 2558 # fast=False by default. 2559 # This covers in-memory pickling with pickle.dumps(). 2560 pickled = self.dumps(obj, proto) 2561 else: 2562 # Pickler is required when fast=True. 2563 if not hasattr(self, 'pickler'): 2564 continue 2565 buf = io.BytesIO() 2566 pickler = self.pickler(buf, protocol=proto) 2567 pickler.fast = fast 2568 pickler.dump(obj) 2569 pickled = buf.getvalue() 2570 unpickled = self.loads(pickled) 2571 # More informative error message in case of failure. 2572 self.assertEqual([len(x) for x in obj], 2573 [len(x) for x in unpickled]) 2574 # Perform full equality check if the lengths match. 2575 self.assertEqual(obj, unpickled) 2576 n_frames = count_opcode(pickle.FRAME, pickled) 2577 # A single frame for small objects between 2578 # first two large objects. 2579 self.assertEqual(n_frames, 1) 2580 self.check_frame_opcodes(pickled) 2581 2582 def test_optional_frames(self): 2583 if pickle.HIGHEST_PROTOCOL < 4: 2584 return 2585 2586 def remove_frames(pickled, keep_frame=None): 2587 """Remove frame opcodes from the given pickle.""" 2588 frame_starts = [] 2589 # 1 byte for the opcode and 8 for the argument 2590 frame_opcode_size = 9 2591 for opcode, _, pos in pickletools.genops(pickled): 2592 if opcode.name == 'FRAME': 2593 frame_starts.append(pos) 2594 2595 newpickle = bytearray() 2596 last_frame_end = 0 2597 for i, pos in enumerate(frame_starts): 2598 if keep_frame and keep_frame(i): 2599 continue 2600 newpickle += pickled[last_frame_end:pos] 2601 last_frame_end = pos + frame_opcode_size 2602 newpickle += pickled[last_frame_end:] 2603 return newpickle 2604 2605 frame_size = self.FRAME_SIZE_TARGET 2606 num_frames = 20 2607 # Large byte objects (dict values) intermittent with small objects 2608 # (dict keys) 2609 for bytes_type in (bytes, bytearray): 2610 obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)} 2611 2612 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2613 pickled = self.dumps(obj, proto) 2614 2615 frameless_pickle = remove_frames(pickled) 2616 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0) 2617 self.assertEqual(obj, self.loads(frameless_pickle)) 2618 2619 some_frames_pickle = remove_frames(pickled, lambda i: i % 2) 2620 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle), 2621 count_opcode(pickle.FRAME, pickled)) 2622 self.assertEqual(obj, self.loads(some_frames_pickle)) 2623 2624 @support.skip_if_pgo_task 2625 def test_framed_write_sizes_with_delayed_writer(self): 2626 class ChunkAccumulator: 2627 """Accumulate pickler output in a list of raw chunks.""" 2628 def __init__(self): 2629 self.chunks = [] 2630 def write(self, chunk): 2631 self.chunks.append(chunk) 2632 def concatenate_chunks(self): 2633 return b"".join(self.chunks) 2634 2635 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2636 objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)}) 2637 for i in range(int(1e4))] 2638 # Add a large unique ASCII string 2639 objects.append('0123456789abcdef' * 2640 (self.FRAME_SIZE_TARGET // 16 + 1)) 2641 2642 # Protocol 4 packs groups of small objects into frames and issues 2643 # calls to write only once or twice per frame: 2644 # The C pickler issues one call to write per-frame (header and 2645 # contents) while Python pickler issues two calls to write: one for 2646 # the frame header and one for the frame binary contents. 2647 writer = ChunkAccumulator() 2648 self.pickler(writer, proto).dump(objects) 2649 2650 # Actually read the binary content of the chunks after the end 2651 # of the call to dump: any memoryview passed to write should not 2652 # be released otherwise this delayed access would not be possible. 2653 pickled = writer.concatenate_chunks() 2654 reconstructed = self.loads(pickled) 2655 self.assertEqual(reconstructed, objects) 2656 self.assertGreater(len(writer.chunks), 1) 2657 2658 # memoryviews should own the memory. 2659 del objects 2660 support.gc_collect() 2661 self.assertEqual(writer.concatenate_chunks(), pickled) 2662 2663 n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1 2664 # There should be at least one call to write per frame 2665 self.assertGreaterEqual(len(writer.chunks), n_frames) 2666 2667 # but not too many either: there can be one for the proto, 2668 # one per-frame header, one per frame for the actual contents, 2669 # and two for the header. 2670 self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3) 2671 2672 chunk_sizes = [len(c) for c in writer.chunks] 2673 large_sizes = [s for s in chunk_sizes 2674 if s >= self.FRAME_SIZE_TARGET] 2675 medium_sizes = [s for s in chunk_sizes 2676 if 9 < s < self.FRAME_SIZE_TARGET] 2677 small_sizes = [s for s in chunk_sizes if s <= 9] 2678 2679 # Large chunks should not be too large: 2680 for chunk_size in large_sizes: 2681 self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET, 2682 chunk_sizes) 2683 # There shouldn't bee too many small chunks: the protocol header, 2684 # the frame headers and the large string headers are written 2685 # in small chunks. 2686 self.assertLessEqual(len(small_sizes), 2687 len(large_sizes) + len(medium_sizes) + 3, 2688 chunk_sizes) 2689 2690 def test_nested_names(self): 2691 global Nested 2692 class Nested: 2693 class A: 2694 class B: 2695 class C: 2696 pass 2697 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2698 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: 2699 with self.subTest(proto=proto, obj=obj): 2700 unpickled = self.loads(self.dumps(obj, proto)) 2701 self.assertIs(obj, unpickled) 2702 2703 def test_recursive_nested_names(self): 2704 global Recursive 2705 class Recursive: 2706 pass 2707 Recursive.mod = sys.modules[Recursive.__module__] 2708 Recursive.__qualname__ = 'Recursive.mod.Recursive' 2709 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2710 with self.subTest(proto=proto): 2711 unpickled = self.loads(self.dumps(Recursive, proto)) 2712 self.assertIs(unpickled, Recursive) 2713 del Recursive.mod # break reference loop 2714 2715 def test_py_methods(self): 2716 global PyMethodsTest 2717 class PyMethodsTest: 2718 @staticmethod 2719 def cheese(): 2720 return "cheese" 2721 @classmethod 2722 def wine(cls): 2723 assert cls is PyMethodsTest 2724 return "wine" 2725 def biscuits(self): 2726 assert isinstance(self, PyMethodsTest) 2727 return "biscuits" 2728 class Nested: 2729 "Nested class" 2730 @staticmethod 2731 def ketchup(): 2732 return "ketchup" 2733 @classmethod 2734 def maple(cls): 2735 assert cls is PyMethodsTest.Nested 2736 return "maple" 2737 def pie(self): 2738 assert isinstance(self, PyMethodsTest.Nested) 2739 return "pie" 2740 2741 py_methods = ( 2742 PyMethodsTest.cheese, 2743 PyMethodsTest.wine, 2744 PyMethodsTest().biscuits, 2745 PyMethodsTest.Nested.ketchup, 2746 PyMethodsTest.Nested.maple, 2747 PyMethodsTest.Nested().pie 2748 ) 2749 py_unbound_methods = ( 2750 (PyMethodsTest.biscuits, PyMethodsTest), 2751 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) 2752 ) 2753 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2754 for method in py_methods: 2755 with self.subTest(proto=proto, method=method): 2756 unpickled = self.loads(self.dumps(method, proto)) 2757 self.assertEqual(method(), unpickled()) 2758 for method, cls in py_unbound_methods: 2759 obj = cls() 2760 with self.subTest(proto=proto, method=method): 2761 unpickled = self.loads(self.dumps(method, proto)) 2762 self.assertEqual(method(obj), unpickled(obj)) 2763 2764 def test_c_methods(self): 2765 global Subclass 2766 class Subclass(tuple): 2767 class Nested(str): 2768 pass 2769 2770 c_methods = ( 2771 # bound built-in method 2772 ("abcd".index, ("c",)), 2773 # unbound built-in method 2774 (str.index, ("abcd", "c")), 2775 # bound "slot" method 2776 ([1, 2, 3].__len__, ()), 2777 # unbound "slot" method 2778 (list.__len__, ([1, 2, 3],)), 2779 # bound "coexist" method 2780 ({1, 2}.__contains__, (2,)), 2781 # unbound "coexist" method 2782 (set.__contains__, ({1, 2}, 2)), 2783 # built-in class method 2784 (dict.fromkeys, (("a", 1), ("b", 2))), 2785 # built-in static method 2786 (bytearray.maketrans, (b"abc", b"xyz")), 2787 # subclass methods 2788 (Subclass([1,2,2]).count, (2,)), 2789 (Subclass.count, (Subclass([1,2,2]), 2)), 2790 (Subclass.Nested("sweet").count, ("e",)), 2791 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), 2792 ) 2793 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2794 for method, args in c_methods: 2795 with self.subTest(proto=proto, method=method): 2796 unpickled = self.loads(self.dumps(method, proto)) 2797 self.assertEqual(method(*args), unpickled(*args)) 2798 2799 def test_compat_pickle(self): 2800 tests = [ 2801 (range(1, 7), '__builtin__', 'xrange'), 2802 (map(int, '123'), 'itertools', 'imap'), 2803 (functools.reduce, '__builtin__', 'reduce'), 2804 (dbm.whichdb, 'whichdb', 'whichdb'), 2805 (Exception(), 'exceptions', 'Exception'), 2806 (collections.UserDict(), 'UserDict', 'IterableUserDict'), 2807 (collections.UserList(), 'UserList', 'UserList'), 2808 (collections.defaultdict(), 'collections', 'defaultdict'), 2809 ] 2810 for val, mod, name in tests: 2811 for proto in range(3): 2812 with self.subTest(type=type(val), proto=proto): 2813 pickled = self.dumps(val, proto) 2814 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) 2815 self.assertIs(type(self.loads(pickled)), type(val)) 2816 2817 def test_local_lookup_error(self): 2818 # Test that whichmodule() errors out cleanly when looking up 2819 # an assumed globally-reachable object fails. 2820 def f(): 2821 pass 2822 # Since the function is local, lookup will fail 2823 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2824 with self.assertRaises((AttributeError, pickle.PicklingError)): 2825 pickletools.dis(self.dumps(f, proto)) 2826 # Same without a __module__ attribute (exercises a different path 2827 # in _pickle.c). 2828 del f.__module__ 2829 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2830 with self.assertRaises((AttributeError, pickle.PicklingError)): 2831 pickletools.dis(self.dumps(f, proto)) 2832 # Yet a different path. 2833 f.__name__ = f.__qualname__ 2834 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2835 with self.assertRaises((AttributeError, pickle.PicklingError)): 2836 pickletools.dis(self.dumps(f, proto)) 2837 2838 # 2839 # PEP 574 tests below 2840 # 2841 2842 def buffer_like_objects(self): 2843 # Yield buffer-like objects with the bytestring "abcdef" in them 2844 bytestring = b"abcdefgh" 2845 yield ZeroCopyBytes(bytestring) 2846 yield ZeroCopyBytearray(bytestring) 2847 if _testbuffer is not None: 2848 items = list(bytestring) 2849 value = int.from_bytes(bytestring, byteorder='little') 2850 for flags in (0, _testbuffer.ND_WRITABLE): 2851 # 1-D, contiguous 2852 yield PicklableNDArray(items, format='B', shape=(8,), 2853 flags=flags) 2854 # 2-D, C-contiguous 2855 yield PicklableNDArray(items, format='B', shape=(4, 2), 2856 strides=(2, 1), flags=flags) 2857 # 2-D, Fortran-contiguous 2858 yield PicklableNDArray(items, format='B', 2859 shape=(4, 2), strides=(1, 4), 2860 flags=flags) 2861 2862 def test_in_band_buffers(self): 2863 # Test in-band buffers (PEP 574) 2864 for obj in self.buffer_like_objects(): 2865 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2866 data = self.dumps(obj, proto) 2867 if obj.c_contiguous and proto >= 5: 2868 # The raw memory bytes are serialized in physical order 2869 self.assertIn(b"abcdefgh", data) 2870 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0) 2871 if proto >= 5: 2872 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 2873 1 if obj.readonly else 0) 2874 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 2875 0 if obj.readonly else 1) 2876 # Return a true value from buffer_callback should have 2877 # the same effect 2878 def buffer_callback(obj): 2879 return True 2880 data2 = self.dumps(obj, proto, 2881 buffer_callback=buffer_callback) 2882 self.assertEqual(data2, data) 2883 2884 new = self.loads(data) 2885 # It's a copy 2886 self.assertIsNot(new, obj) 2887 self.assertIs(type(new), type(obj)) 2888 self.assertEqual(new, obj) 2889 2890 # XXX Unfortunately cannot test non-contiguous array 2891 # (see comment in PicklableNDArray.__reduce_ex__) 2892 2893 def test_oob_buffers(self): 2894 # Test out-of-band buffers (PEP 574) 2895 for obj in self.buffer_like_objects(): 2896 for proto in range(0, 5): 2897 # Need protocol >= 5 for buffer_callback 2898 with self.assertRaises(ValueError): 2899 self.dumps(obj, proto, 2900 buffer_callback=[].append) 2901 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2902 buffers = [] 2903 buffer_callback = lambda pb: buffers.append(pb.raw()) 2904 data = self.dumps(obj, proto, 2905 buffer_callback=buffer_callback) 2906 self.assertNotIn(b"abcdefgh", data) 2907 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0) 2908 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0) 2909 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1) 2910 self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data), 2911 1 if obj.readonly else 0) 2912 2913 if obj.c_contiguous: 2914 self.assertEqual(bytes(buffers[0]), b"abcdefgh") 2915 # Need buffers argument to unpickle properly 2916 with self.assertRaises(pickle.UnpicklingError): 2917 self.loads(data) 2918 2919 new = self.loads(data, buffers=buffers) 2920 if obj.zero_copy_reconstruct: 2921 # Zero-copy achieved 2922 self.assertIs(new, obj) 2923 else: 2924 self.assertIs(type(new), type(obj)) 2925 self.assertEqual(new, obj) 2926 # Non-sequence buffers accepted too 2927 new = self.loads(data, buffers=iter(buffers)) 2928 if obj.zero_copy_reconstruct: 2929 # Zero-copy achieved 2930 self.assertIs(new, obj) 2931 else: 2932 self.assertIs(type(new), type(obj)) 2933 self.assertEqual(new, obj) 2934 2935 def test_oob_buffers_writable_to_readonly(self): 2936 # Test reconstructing readonly object from writable buffer 2937 obj = ZeroCopyBytes(b"foobar") 2938 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2939 buffers = [] 2940 buffer_callback = buffers.append 2941 data = self.dumps(obj, proto, buffer_callback=buffer_callback) 2942 2943 buffers = map(bytearray, buffers) 2944 new = self.loads(data, buffers=buffers) 2945 self.assertIs(type(new), type(obj)) 2946 self.assertEqual(new, obj) 2947 2948 def test_picklebuffer_error(self): 2949 # PickleBuffer forbidden with protocol < 5 2950 pb = pickle.PickleBuffer(b"foobar") 2951 for proto in range(0, 5): 2952 with self.assertRaises(pickle.PickleError): 2953 self.dumps(pb, proto) 2954 2955 def test_buffer_callback_error(self): 2956 def buffer_callback(buffers): 2957 1/0 2958 pb = pickle.PickleBuffer(b"foobar") 2959 with self.assertRaises(ZeroDivisionError): 2960 self.dumps(pb, 5, buffer_callback=buffer_callback) 2961 2962 def test_buffers_error(self): 2963 pb = pickle.PickleBuffer(b"foobar") 2964 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2965 data = self.dumps(pb, proto, buffer_callback=[].append) 2966 # Non iterable buffers 2967 with self.assertRaises(TypeError): 2968 self.loads(data, buffers=object()) 2969 # Buffer iterable exhausts too early 2970 with self.assertRaises(pickle.UnpicklingError): 2971 self.loads(data, buffers=[]) 2972 2973 def test_inband_accept_default_buffers_argument(self): 2974 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2975 data_pickled = self.dumps(1, proto, buffer_callback=None) 2976 data = self.loads(data_pickled, buffers=None) 2977 2978 @unittest.skipIf(np is None, "Test needs Numpy") 2979 def test_buffers_numpy(self): 2980 def check_no_copy(x, y): 2981 np.testing.assert_equal(x, y) 2982 self.assertEqual(x.ctypes.data, y.ctypes.data) 2983 2984 def check_copy(x, y): 2985 np.testing.assert_equal(x, y) 2986 self.assertNotEqual(x.ctypes.data, y.ctypes.data) 2987 2988 def check_array(arr): 2989 # In-band 2990 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2991 data = self.dumps(arr, proto) 2992 new = self.loads(data) 2993 check_copy(arr, new) 2994 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2995 buffer_callback = lambda _: True 2996 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 2997 new = self.loads(data) 2998 check_copy(arr, new) 2999 # Out-of-band 3000 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3001 buffers = [] 3002 buffer_callback = buffers.append 3003 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 3004 new = self.loads(data, buffers=buffers) 3005 if arr.flags.c_contiguous or arr.flags.f_contiguous: 3006 check_no_copy(arr, new) 3007 else: 3008 check_copy(arr, new) 3009 3010 # 1-D 3011 arr = np.arange(6) 3012 check_array(arr) 3013 # 1-D, non-contiguous 3014 check_array(arr[::2]) 3015 # 2-D, C-contiguous 3016 arr = np.arange(12).reshape((3, 4)) 3017 check_array(arr) 3018 # 2-D, F-contiguous 3019 check_array(arr.T) 3020 # 2-D, non-contiguous 3021 check_array(arr[::2]) 3022 3023 3024class BigmemPickleTests: 3025 3026 # Binary protocols can serialize longs of up to 2 GiB-1 3027 3028 @bigmemtest(size=_2G, memuse=3.6, dry_run=False) 3029 def test_huge_long_32b(self, size): 3030 data = 1 << (8 * size) 3031 try: 3032 for proto in protocols: 3033 if proto < 2: 3034 continue 3035 with self.subTest(proto=proto): 3036 with self.assertRaises((ValueError, OverflowError)): 3037 self.dumps(data, protocol=proto) 3038 finally: 3039 data = None 3040 3041 # Protocol 3 can serialize up to 4 GiB-1 as a bytes object 3042 # (older protocols don't have a dedicated opcode for bytes and are 3043 # too inefficient) 3044 3045 @bigmemtest(size=_2G, memuse=2.5, dry_run=False) 3046 def test_huge_bytes_32b(self, size): 3047 data = b"abcd" * (size // 4) 3048 try: 3049 for proto in protocols: 3050 if proto < 3: 3051 continue 3052 with self.subTest(proto=proto): 3053 try: 3054 pickled = self.dumps(data, protocol=proto) 3055 header = (pickle.BINBYTES + 3056 struct.pack("<I", len(data))) 3057 data_start = pickled.index(data) 3058 self.assertEqual( 3059 header, 3060 pickled[data_start-len(header):data_start]) 3061 finally: 3062 pickled = None 3063 finally: 3064 data = None 3065 3066 @bigmemtest(size=_4G, memuse=2.5, dry_run=False) 3067 def test_huge_bytes_64b(self, size): 3068 data = b"acbd" * (size // 4) 3069 try: 3070 for proto in protocols: 3071 if proto < 3: 3072 continue 3073 with self.subTest(proto=proto): 3074 if proto == 3: 3075 # Protocol 3 does not support large bytes objects. 3076 # Verify that we do not crash when processing one. 3077 with self.assertRaises((ValueError, OverflowError)): 3078 self.dumps(data, protocol=proto) 3079 continue 3080 try: 3081 pickled = self.dumps(data, protocol=proto) 3082 header = (pickle.BINBYTES8 + 3083 struct.pack("<Q", len(data))) 3084 data_start = pickled.index(data) 3085 self.assertEqual( 3086 header, 3087 pickled[data_start-len(header):data_start]) 3088 finally: 3089 pickled = None 3090 finally: 3091 data = None 3092 3093 # All protocols use 1-byte per printable ASCII character; we add another 3094 # byte because the encoded form has to be copied into the internal buffer. 3095 3096 @bigmemtest(size=_2G, memuse=8, dry_run=False) 3097 def test_huge_str_32b(self, size): 3098 data = "abcd" * (size // 4) 3099 try: 3100 for proto in protocols: 3101 if proto == 0: 3102 continue 3103 with self.subTest(proto=proto): 3104 try: 3105 pickled = self.dumps(data, protocol=proto) 3106 header = (pickle.BINUNICODE + 3107 struct.pack("<I", len(data))) 3108 data_start = pickled.index(b'abcd') 3109 self.assertEqual( 3110 header, 3111 pickled[data_start-len(header):data_start]) 3112 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3113 pickled.index(b"abcd")), len(data)) 3114 finally: 3115 pickled = None 3116 finally: 3117 data = None 3118 3119 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes 3120 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge 3121 # unicode strings however. 3122 3123 @bigmemtest(size=_4G, memuse=8, dry_run=False) 3124 def test_huge_str_64b(self, size): 3125 data = "abcd" * (size // 4) 3126 try: 3127 for proto in protocols: 3128 if proto == 0: 3129 continue 3130 with self.subTest(proto=proto): 3131 if proto < 4: 3132 with self.assertRaises((ValueError, OverflowError)): 3133 self.dumps(data, protocol=proto) 3134 continue 3135 try: 3136 pickled = self.dumps(data, protocol=proto) 3137 header = (pickle.BINUNICODE8 + 3138 struct.pack("<Q", len(data))) 3139 data_start = pickled.index(b'abcd') 3140 self.assertEqual( 3141 header, 3142 pickled[data_start-len(header):data_start]) 3143 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3144 pickled.index(b"abcd")), len(data)) 3145 finally: 3146 pickled = None 3147 finally: 3148 data = None 3149 3150 3151# Test classes for reduce_ex 3152 3153class REX_one(object): 3154 """No __reduce_ex__ here, but inheriting it from object""" 3155 _reduce_called = 0 3156 def __reduce__(self): 3157 self._reduce_called = 1 3158 return REX_one, () 3159 3160class REX_two(object): 3161 """No __reduce__ here, but inheriting it from object""" 3162 _proto = None 3163 def __reduce_ex__(self, proto): 3164 self._proto = proto 3165 return REX_two, () 3166 3167class REX_three(object): 3168 _proto = None 3169 def __reduce_ex__(self, proto): 3170 self._proto = proto 3171 return REX_two, () 3172 def __reduce__(self): 3173 raise TestFailed("This __reduce__ shouldn't be called") 3174 3175class REX_four(object): 3176 """Calling base class method should succeed""" 3177 _proto = None 3178 def __reduce_ex__(self, proto): 3179 self._proto = proto 3180 return object.__reduce_ex__(self, proto) 3181 3182class REX_five(object): 3183 """This one used to fail with infinite recursion""" 3184 _reduce_called = 0 3185 def __reduce__(self): 3186 self._reduce_called = 1 3187 return object.__reduce__(self) 3188 3189class REX_six(object): 3190 """This class is used to check the 4th argument (list iterator) of 3191 the reduce protocol. 3192 """ 3193 def __init__(self, items=None): 3194 self.items = items if items is not None else [] 3195 def __eq__(self, other): 3196 return type(self) is type(other) and self.items == other.items 3197 def append(self, item): 3198 self.items.append(item) 3199 def __reduce__(self): 3200 return type(self), (), None, iter(self.items), None 3201 3202class REX_seven(object): 3203 """This class is used to check the 5th argument (dict iterator) of 3204 the reduce protocol. 3205 """ 3206 def __init__(self, table=None): 3207 self.table = table if table is not None else {} 3208 def __eq__(self, other): 3209 return type(self) is type(other) and self.table == other.table 3210 def __setitem__(self, key, value): 3211 self.table[key] = value 3212 def __reduce__(self): 3213 return type(self), (), None, None, iter(self.table.items()) 3214 3215class REX_state(object): 3216 """This class is used to check the 3th argument (state) of 3217 the reduce protocol. 3218 """ 3219 def __init__(self, state=None): 3220 self.state = state 3221 def __eq__(self, other): 3222 return type(self) is type(other) and self.state == other.state 3223 def __setstate__(self, state): 3224 self.state = state 3225 def __reduce__(self): 3226 return type(self), (), self.state 3227 3228 3229# Test classes for newobj 3230 3231class MyInt(int): 3232 sample = 1 3233 3234class MyFloat(float): 3235 sample = 1.0 3236 3237class MyComplex(complex): 3238 sample = 1.0 + 0.0j 3239 3240class MyStr(str): 3241 sample = "hello" 3242 3243class MyUnicode(str): 3244 sample = "hello \u1234" 3245 3246class MyTuple(tuple): 3247 sample = (1, 2, 3) 3248 3249class MyList(list): 3250 sample = [1, 2, 3] 3251 3252class MyDict(dict): 3253 sample = {"a": 1, "b": 2} 3254 3255class MySet(set): 3256 sample = {"a", "b"} 3257 3258class MyFrozenSet(frozenset): 3259 sample = frozenset({"a", "b"}) 3260 3261myclasses = [MyInt, MyFloat, 3262 MyComplex, 3263 MyStr, MyUnicode, 3264 MyTuple, MyList, MyDict, MySet, MyFrozenSet] 3265 3266class MyIntWithNew(int): 3267 def __new__(cls, value): 3268 raise AssertionError 3269 3270class MyIntWithNew2(MyIntWithNew): 3271 __new__ = int.__new__ 3272 3273 3274class SlotList(MyList): 3275 __slots__ = ["foo"] 3276 3277class SimpleNewObj(int): 3278 def __init__(self, *args, **kwargs): 3279 # raise an error, to make sure this isn't called 3280 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 3281 def __eq__(self, other): 3282 return int(self) == int(other) and self.__dict__ == other.__dict__ 3283 3284class ComplexNewObj(SimpleNewObj): 3285 def __getnewargs__(self): 3286 return ('%X' % self, 16) 3287 3288class ComplexNewObjEx(SimpleNewObj): 3289 def __getnewargs_ex__(self): 3290 return ('%X' % self,), {'base': 16} 3291 3292class BadGetattr: 3293 def __getattr__(self, key): 3294 self.foo 3295 3296 3297class AbstractPickleModuleTests: 3298 3299 def test_dump_closed_file(self): 3300 f = open(TESTFN, "wb") 3301 try: 3302 f.close() 3303 self.assertRaises(ValueError, self.dump, 123, f) 3304 finally: 3305 support.unlink(TESTFN) 3306 3307 def test_load_closed_file(self): 3308 f = open(TESTFN, "wb") 3309 try: 3310 f.close() 3311 self.assertRaises(ValueError, self.dump, 123, f) 3312 finally: 3313 support.unlink(TESTFN) 3314 3315 def test_load_from_and_dump_to_file(self): 3316 stream = io.BytesIO() 3317 data = [123, {}, 124] 3318 self.dump(data, stream) 3319 stream.seek(0) 3320 unpickled = self.load(stream) 3321 self.assertEqual(unpickled, data) 3322 3323 def test_highest_protocol(self): 3324 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 3325 self.assertEqual(pickle.HIGHEST_PROTOCOL, 5) 3326 3327 def test_callapi(self): 3328 f = io.BytesIO() 3329 # With and without keyword arguments 3330 self.dump(123, f, -1) 3331 self.dump(123, file=f, protocol=-1) 3332 self.dumps(123, -1) 3333 self.dumps(123, protocol=-1) 3334 self.Pickler(f, -1) 3335 self.Pickler(f, protocol=-1) 3336 3337 def test_dump_text_file(self): 3338 f = open(TESTFN, "w") 3339 try: 3340 for proto in protocols: 3341 self.assertRaises(TypeError, self.dump, 123, f, proto) 3342 finally: 3343 f.close() 3344 support.unlink(TESTFN) 3345 3346 def test_incomplete_input(self): 3347 s = io.BytesIO(b"X''.") 3348 self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s) 3349 3350 def test_bad_init(self): 3351 # Test issue3664 (pickle can segfault from a badly initialized Pickler). 3352 # Override initialization without calling __init__() of the superclass. 3353 class BadPickler(self.Pickler): 3354 def __init__(self): pass 3355 3356 class BadUnpickler(self.Unpickler): 3357 def __init__(self): pass 3358 3359 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) 3360 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) 3361 3362 def check_dumps_loads_oob_buffers(self, dumps, loads): 3363 # No need to do the full gamut of tests here, just enough to 3364 # check that dumps() and loads() redirect their arguments 3365 # to the underlying Pickler and Unpickler, respectively. 3366 obj = ZeroCopyBytes(b"foo") 3367 3368 for proto in range(0, 5): 3369 # Need protocol >= 5 for buffer_callback 3370 with self.assertRaises(ValueError): 3371 dumps(obj, protocol=proto, 3372 buffer_callback=[].append) 3373 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3374 buffers = [] 3375 buffer_callback = buffers.append 3376 data = dumps(obj, protocol=proto, 3377 buffer_callback=buffer_callback) 3378 self.assertNotIn(b"foo", data) 3379 self.assertEqual(bytes(buffers[0]), b"foo") 3380 # Need buffers argument to unpickle properly 3381 with self.assertRaises(pickle.UnpicklingError): 3382 loads(data) 3383 new = loads(data, buffers=buffers) 3384 self.assertIs(new, obj) 3385 3386 def test_dumps_loads_oob_buffers(self): 3387 # Test out-of-band buffers (PEP 574) with top-level dumps() and loads() 3388 self.check_dumps_loads_oob_buffers(self.dumps, self.loads) 3389 3390 def test_dump_load_oob_buffers(self): 3391 # Test out-of-band buffers (PEP 574) with top-level dump() and load() 3392 def dumps(obj, **kwargs): 3393 f = io.BytesIO() 3394 self.dump(obj, f, **kwargs) 3395 return f.getvalue() 3396 3397 def loads(data, **kwargs): 3398 f = io.BytesIO(data) 3399 return self.load(f, **kwargs) 3400 3401 self.check_dumps_loads_oob_buffers(dumps, loads) 3402 3403 3404class AbstractPersistentPicklerTests: 3405 3406 # This class defines persistent_id() and persistent_load() 3407 # functions that should be used by the pickler. All even integers 3408 # are pickled using persistent ids. 3409 3410 def persistent_id(self, object): 3411 if isinstance(object, int) and object % 2 == 0: 3412 self.id_count += 1 3413 return str(object) 3414 elif object == "test_false_value": 3415 self.false_count += 1 3416 return "" 3417 else: 3418 return None 3419 3420 def persistent_load(self, oid): 3421 if not oid: 3422 self.load_false_count += 1 3423 return "test_false_value" 3424 else: 3425 self.load_count += 1 3426 object = int(oid) 3427 assert object % 2 == 0 3428 return object 3429 3430 def test_persistence(self): 3431 L = list(range(10)) + ["test_false_value"] 3432 for proto in protocols: 3433 self.id_count = 0 3434 self.false_count = 0 3435 self.load_false_count = 0 3436 self.load_count = 0 3437 self.assertEqual(self.loads(self.dumps(L, proto)), L) 3438 self.assertEqual(self.id_count, 5) 3439 self.assertEqual(self.false_count, 1) 3440 self.assertEqual(self.load_count, 5) 3441 self.assertEqual(self.load_false_count, 1) 3442 3443 3444class AbstractIdentityPersistentPicklerTests: 3445 3446 def persistent_id(self, obj): 3447 return obj 3448 3449 def persistent_load(self, pid): 3450 return pid 3451 3452 def _check_return_correct_type(self, obj, proto): 3453 unpickled = self.loads(self.dumps(obj, proto)) 3454 self.assertIsInstance(unpickled, type(obj)) 3455 self.assertEqual(unpickled, obj) 3456 3457 def test_return_correct_type(self): 3458 for proto in protocols: 3459 # Protocol 0 supports only ASCII strings. 3460 if proto == 0: 3461 self._check_return_correct_type("abc", 0) 3462 else: 3463 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]: 3464 self._check_return_correct_type(obj, proto) 3465 3466 def test_protocol0_is_ascii_only(self): 3467 non_ascii_str = "\N{EMPTY SET}" 3468 self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) 3469 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' 3470 self.assertRaises(pickle.UnpicklingError, self.loads, pickled) 3471 3472 3473class AbstractPicklerUnpicklerObjectTests: 3474 3475 pickler_class = None 3476 unpickler_class = None 3477 3478 def setUp(self): 3479 assert self.pickler_class 3480 assert self.unpickler_class 3481 3482 def test_clear_pickler_memo(self): 3483 # To test whether clear_memo() has any effect, we pickle an object, 3484 # then pickle it again without clearing the memo; the two serialized 3485 # forms should be different. If we clear_memo() and then pickle the 3486 # object again, the third serialized form should be identical to the 3487 # first one we obtained. 3488 data = ["abcdefg", "abcdefg", 44] 3489 for proto in protocols: 3490 f = io.BytesIO() 3491 pickler = self.pickler_class(f, proto) 3492 3493 pickler.dump(data) 3494 first_pickled = f.getvalue() 3495 3496 # Reset BytesIO object. 3497 f.seek(0) 3498 f.truncate() 3499 3500 pickler.dump(data) 3501 second_pickled = f.getvalue() 3502 3503 # Reset the Pickler and BytesIO objects. 3504 pickler.clear_memo() 3505 f.seek(0) 3506 f.truncate() 3507 3508 pickler.dump(data) 3509 third_pickled = f.getvalue() 3510 3511 self.assertNotEqual(first_pickled, second_pickled) 3512 self.assertEqual(first_pickled, third_pickled) 3513 3514 def test_priming_pickler_memo(self): 3515 # Verify that we can set the Pickler's memo attribute. 3516 data = ["abcdefg", "abcdefg", 44] 3517 f = io.BytesIO() 3518 pickler = self.pickler_class(f) 3519 3520 pickler.dump(data) 3521 first_pickled = f.getvalue() 3522 3523 f = io.BytesIO() 3524 primed = self.pickler_class(f) 3525 primed.memo = pickler.memo 3526 3527 primed.dump(data) 3528 primed_pickled = f.getvalue() 3529 3530 self.assertNotEqual(first_pickled, primed_pickled) 3531 3532 def test_priming_unpickler_memo(self): 3533 # Verify that we can set the Unpickler's memo attribute. 3534 data = ["abcdefg", "abcdefg", 44] 3535 f = io.BytesIO() 3536 pickler = self.pickler_class(f) 3537 3538 pickler.dump(data) 3539 first_pickled = f.getvalue() 3540 3541 f = io.BytesIO() 3542 primed = self.pickler_class(f) 3543 primed.memo = pickler.memo 3544 3545 primed.dump(data) 3546 primed_pickled = f.getvalue() 3547 3548 unpickler = self.unpickler_class(io.BytesIO(first_pickled)) 3549 unpickled_data1 = unpickler.load() 3550 3551 self.assertEqual(unpickled_data1, data) 3552 3553 primed = self.unpickler_class(io.BytesIO(primed_pickled)) 3554 primed.memo = unpickler.memo 3555 unpickled_data2 = primed.load() 3556 3557 primed.memo.clear() 3558 3559 self.assertEqual(unpickled_data2, data) 3560 self.assertTrue(unpickled_data2 is unpickled_data1) 3561 3562 def test_reusing_unpickler_objects(self): 3563 data1 = ["abcdefg", "abcdefg", 44] 3564 f = io.BytesIO() 3565 pickler = self.pickler_class(f) 3566 pickler.dump(data1) 3567 pickled1 = f.getvalue() 3568 3569 data2 = ["abcdefg", 44, 44] 3570 f = io.BytesIO() 3571 pickler = self.pickler_class(f) 3572 pickler.dump(data2) 3573 pickled2 = f.getvalue() 3574 3575 f = io.BytesIO() 3576 f.write(pickled1) 3577 f.seek(0) 3578 unpickler = self.unpickler_class(f) 3579 self.assertEqual(unpickler.load(), data1) 3580 3581 f.seek(0) 3582 f.truncate() 3583 f.write(pickled2) 3584 f.seek(0) 3585 self.assertEqual(unpickler.load(), data2) 3586 3587 def _check_multiple_unpicklings(self, ioclass, *, seekable=True): 3588 for proto in protocols: 3589 with self.subTest(proto=proto): 3590 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len] 3591 f = ioclass() 3592 pickler = self.pickler_class(f, protocol=proto) 3593 pickler.dump(data1) 3594 pickled = f.getvalue() 3595 3596 N = 5 3597 f = ioclass(pickled * N) 3598 unpickler = self.unpickler_class(f) 3599 for i in range(N): 3600 if seekable: 3601 pos = f.tell() 3602 self.assertEqual(unpickler.load(), data1) 3603 if seekable: 3604 self.assertEqual(f.tell(), pos + len(pickled)) 3605 self.assertRaises(EOFError, unpickler.load) 3606 3607 def test_multiple_unpicklings_seekable(self): 3608 self._check_multiple_unpicklings(io.BytesIO) 3609 3610 def test_multiple_unpicklings_unseekable(self): 3611 self._check_multiple_unpicklings(UnseekableIO, seekable=False) 3612 3613 def test_multiple_unpicklings_minimal(self): 3614 # File-like object that doesn't support peek() and readinto() 3615 # (bpo-39681) 3616 self._check_multiple_unpicklings(MinimalIO, seekable=False) 3617 3618 def test_unpickling_buffering_readline(self): 3619 # Issue #12687: the unpickler's buffering logic could fail with 3620 # text mode opcodes. 3621 data = list(range(10)) 3622 for proto in protocols: 3623 for buf_size in range(1, 11): 3624 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size) 3625 pickler = self.pickler_class(f, protocol=proto) 3626 pickler.dump(data) 3627 f.seek(0) 3628 unpickler = self.unpickler_class(f) 3629 self.assertEqual(unpickler.load(), data) 3630 3631 3632# Tests for dispatch_table attribute 3633 3634REDUCE_A = 'reduce_A' 3635 3636class AAA(object): 3637 def __reduce__(self): 3638 return str, (REDUCE_A,) 3639 3640class BBB(object): 3641 def __init__(self): 3642 # Add an instance attribute to enable state-saving routines at pickling 3643 # time. 3644 self.a = "some attribute" 3645 3646 def __setstate__(self, state): 3647 self.a = "BBB.__setstate__" 3648 3649 3650def setstate_bbb(obj, state): 3651 """Custom state setter for BBB objects 3652 3653 Such callable may be created by other persons than the ones who created the 3654 BBB class. If passed as the state_setter item of a custom reducer, this 3655 allows for custom state setting behavior of BBB objects. One can think of 3656 it as the analogous of list_setitems or dict_setitems but for foreign 3657 classes/functions. 3658 """ 3659 obj.a = "custom state_setter" 3660 3661 3662 3663class AbstractCustomPicklerClass: 3664 """Pickler implementing a reducing hook using reducer_override.""" 3665 def reducer_override(self, obj): 3666 obj_name = getattr(obj, "__name__", None) 3667 3668 if obj_name == 'f': 3669 # asking the pickler to save f as 5 3670 return int, (5, ) 3671 3672 if obj_name == 'MyClass': 3673 return str, ('some str',) 3674 3675 elif obj_name == 'g': 3676 # in this case, the callback returns an invalid result (not a 2-5 3677 # tuple or a string), the pickler should raise a proper error. 3678 return False 3679 3680 elif obj_name == 'h': 3681 # Simulate a case when the reducer fails. The error should 3682 # be propagated to the original ``dump`` call. 3683 raise ValueError('The reducer just failed') 3684 3685 return NotImplemented 3686 3687class AbstractHookTests: 3688 def test_pickler_hook(self): 3689 # test the ability of a custom, user-defined CPickler subclass to 3690 # override the default reducing routines of any type using the method 3691 # reducer_override 3692 3693 def f(): 3694 pass 3695 3696 def g(): 3697 pass 3698 3699 def h(): 3700 pass 3701 3702 class MyClass: 3703 pass 3704 3705 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3706 with self.subTest(proto=proto): 3707 bio = io.BytesIO() 3708 p = self.pickler_class(bio, proto) 3709 3710 p.dump([f, MyClass, math.log]) 3711 new_f, some_str, math_log = pickle.loads(bio.getvalue()) 3712 3713 self.assertEqual(new_f, 5) 3714 self.assertEqual(some_str, 'some str') 3715 # math.log does not have its usual reducer overridden, so the 3716 # custom reduction callback should silently direct the pickler 3717 # to the default pickling by attribute, by returning 3718 # NotImplemented 3719 self.assertIs(math_log, math.log) 3720 3721 with self.assertRaises(pickle.PicklingError): 3722 p.dump(g) 3723 3724 with self.assertRaisesRegex( 3725 ValueError, 'The reducer just failed'): 3726 p.dump(h) 3727 3728 @support.cpython_only 3729 def test_reducer_override_no_reference_cycle(self): 3730 # bpo-39492: reducer_override used to induce a spurious reference cycle 3731 # inside the Pickler object, that could prevent all serialized objects 3732 # from being garbage-collected without explicitly invoking gc.collect. 3733 3734 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3735 with self.subTest(proto=proto): 3736 def f(): 3737 pass 3738 3739 wr = weakref.ref(f) 3740 3741 bio = io.BytesIO() 3742 p = self.pickler_class(bio, proto) 3743 p.dump(f) 3744 new_f = pickle.loads(bio.getvalue()) 3745 assert new_f == 5 3746 3747 del p 3748 del f 3749 3750 self.assertIsNone(wr()) 3751 3752 3753class AbstractDispatchTableTests: 3754 3755 def test_default_dispatch_table(self): 3756 # No dispatch_table attribute by default 3757 f = io.BytesIO() 3758 p = self.pickler_class(f, 0) 3759 with self.assertRaises(AttributeError): 3760 p.dispatch_table 3761 self.assertFalse(hasattr(p, 'dispatch_table')) 3762 3763 def test_class_dispatch_table(self): 3764 # A dispatch_table attribute can be specified class-wide 3765 dt = self.get_dispatch_table() 3766 3767 class MyPickler(self.pickler_class): 3768 dispatch_table = dt 3769 3770 def dumps(obj, protocol=None): 3771 f = io.BytesIO() 3772 p = MyPickler(f, protocol) 3773 self.assertEqual(p.dispatch_table, dt) 3774 p.dump(obj) 3775 return f.getvalue() 3776 3777 self._test_dispatch_table(dumps, dt) 3778 3779 def test_instance_dispatch_table(self): 3780 # A dispatch_table attribute can also be specified instance-wide 3781 dt = self.get_dispatch_table() 3782 3783 def dumps(obj, protocol=None): 3784 f = io.BytesIO() 3785 p = self.pickler_class(f, protocol) 3786 p.dispatch_table = dt 3787 self.assertEqual(p.dispatch_table, dt) 3788 p.dump(obj) 3789 return f.getvalue() 3790 3791 self._test_dispatch_table(dumps, dt) 3792 3793 def _test_dispatch_table(self, dumps, dispatch_table): 3794 def custom_load_dump(obj): 3795 return pickle.loads(dumps(obj, 0)) 3796 3797 def default_load_dump(obj): 3798 return pickle.loads(pickle.dumps(obj, 0)) 3799 3800 # pickling complex numbers using protocol 0 relies on copyreg 3801 # so check pickling a complex number still works 3802 z = 1 + 2j 3803 self.assertEqual(custom_load_dump(z), z) 3804 self.assertEqual(default_load_dump(z), z) 3805 3806 # modify pickling of complex 3807 REDUCE_1 = 'reduce_1' 3808 def reduce_1(obj): 3809 return str, (REDUCE_1,) 3810 dispatch_table[complex] = reduce_1 3811 self.assertEqual(custom_load_dump(z), REDUCE_1) 3812 self.assertEqual(default_load_dump(z), z) 3813 3814 # check picklability of AAA and BBB 3815 a = AAA() 3816 b = BBB() 3817 self.assertEqual(custom_load_dump(a), REDUCE_A) 3818 self.assertIsInstance(custom_load_dump(b), BBB) 3819 self.assertEqual(default_load_dump(a), REDUCE_A) 3820 self.assertIsInstance(default_load_dump(b), BBB) 3821 3822 # modify pickling of BBB 3823 dispatch_table[BBB] = reduce_1 3824 self.assertEqual(custom_load_dump(a), REDUCE_A) 3825 self.assertEqual(custom_load_dump(b), REDUCE_1) 3826 self.assertEqual(default_load_dump(a), REDUCE_A) 3827 self.assertIsInstance(default_load_dump(b), BBB) 3828 3829 # revert pickling of BBB and modify pickling of AAA 3830 REDUCE_2 = 'reduce_2' 3831 def reduce_2(obj): 3832 return str, (REDUCE_2,) 3833 dispatch_table[AAA] = reduce_2 3834 del dispatch_table[BBB] 3835 self.assertEqual(custom_load_dump(a), REDUCE_2) 3836 self.assertIsInstance(custom_load_dump(b), BBB) 3837 self.assertEqual(default_load_dump(a), REDUCE_A) 3838 self.assertIsInstance(default_load_dump(b), BBB) 3839 3840 # End-to-end testing of save_reduce with the state_setter keyword 3841 # argument. This is a dispatch_table test as the primary goal of 3842 # state_setter is to tweak objects reduction behavior. 3843 # In particular, state_setter is useful when the default __setstate__ 3844 # behavior is not flexible enough. 3845 3846 # No custom reducer for b has been registered for now, so 3847 # BBB.__setstate__ should be used at unpickling time 3848 self.assertEqual(default_load_dump(b).a, "BBB.__setstate__") 3849 3850 def reduce_bbb(obj): 3851 return BBB, (), obj.__dict__, None, None, setstate_bbb 3852 3853 dispatch_table[BBB] = reduce_bbb 3854 3855 # The custom reducer reduce_bbb includes a state setter, that should 3856 # have priority over BBB.__setstate__ 3857 self.assertEqual(custom_load_dump(b).a, "custom state_setter") 3858 3859 3860if __name__ == "__main__": 3861 # Print some stuff that can be used to rewrite DATA{0,1,2} 3862 from pickletools import dis 3863 x = create_data() 3864 for i in range(pickle.HIGHEST_PROTOCOL+1): 3865 p = pickle.dumps(x, i) 3866 print("DATA{0} = (".format(i)) 3867 for j in range(0, len(p), 20): 3868 b = bytes(p[j:j+20]) 3869 print(" {0!r}".format(b)) 3870 print(")") 3871 print() 3872 print("# Disassembly of DATA{0}".format(i)) 3873 print("DATA{0}_DIS = \"\"\"\\".format(i)) 3874 dis(p) 3875 print("\"\"\"") 3876 print() 3877