1# -*- test-case-name: twisted.spread.test.test_jelly -*- 2# Copyright (c) Twisted Matrix Laboratories. 3# See LICENSE for details. 4 5""" 6S-expression-based persistence of python objects. 7 8It does something very much like L{Pickle<pickle>}; however, pickle's main goal 9seems to be efficiency (both in space and time); jelly's main goals are 10security, human readability, and portability to other environments. 11 12This is how Jelly converts various objects to s-expressions. 13 14Boolean:: 15 True --> ['boolean', 'true'] 16 17Integer:: 18 1 --> 1 19 20List:: 21 [1, 2] --> ['list', 1, 2] 22 23String:: 24 \"hello\" --> \"hello\" 25 26Float:: 27 2.3 --> 2.3 28 29Dictionary:: 30 {'a': 1, 'b': 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]] 31 32Module:: 33 UserString --> ['module', 'UserString'] 34 35Class:: 36 UserString.UserString --> ['class', ['module', 'UserString'], 'UserString'] 37 38Function:: 39 string.join --> ['function', 'join', ['module', 'string']] 40 41Instance: s is an instance of UserString.UserString, with a __dict__ 42{'data': 'hello'}:: 43 [\"UserString.UserString\", ['dictionary', ['data', 'hello']]] 44 45Class Method: UserString.UserString.center:: 46 ['method', 'center', ['None'], ['class', ['module', 'UserString'], 47 'UserString']] 48 49Instance Method: s.center, where s is an instance of UserString.UserString:: 50 ['method', 'center', ['instance', ['reference', 1, ['class', 51 ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]], 52 ['dereference', 1]] 53 54The Python 2.x C{sets.Set} and C{sets.ImmutableSet} classes are 55serialized to the same thing as the builtin C{set} and C{frozenset} 56classes. (This is only relevant if you are communicating with a 57version of jelly running on an older version of Python.) 58 59@author: Glyph Lefkowitz 60 61""" 62 63import copy 64import datetime 65import decimal 66 67# System Imports 68import types 69import warnings 70from functools import reduce 71 72from zope.interface import implementer 73 74from incremental import Version 75 76from twisted.persisted.crefutil import ( 77 NotKnown, 78 _Container, 79 _Dereference, 80 _DictKeyAndValue, 81 _InstanceMethod, 82 _Tuple, 83) 84 85# Twisted Imports 86from twisted.python.compat import nativeString 87from twisted.python.deprecate import deprecatedModuleAttribute 88from twisted.python.reflect import namedAny, namedObject, qual 89from twisted.spread.interfaces import IJellyable, IUnjellyable 90 91DictTypes = (dict,) 92 93None_atom = b"None" # N 94# code 95class_atom = b"class" # c 96module_atom = b"module" # m 97function_atom = b"function" # f 98 99# references 100dereference_atom = b"dereference" # D 101persistent_atom = b"persistent" # p 102reference_atom = b"reference" # r 103 104# mutable collections 105dictionary_atom = b"dictionary" # d 106list_atom = b"list" # l 107set_atom = b"set" 108 109# immutable collections 110# (assignment to __dict__ and __class__ still might go away!) 111tuple_atom = b"tuple" # t 112instance_atom = b"instance" # i 113frozenset_atom = b"frozenset" 114 115 116deprecatedModuleAttribute( 117 Version("Twisted", 15, 0, 0), 118 "instance_atom is unused within Twisted.", 119 "twisted.spread.jelly", 120 "instance_atom", 121) 122 123# errors 124unpersistable_atom = b"unpersistable" # u 125unjellyableRegistry = {} 126unjellyableFactoryRegistry = {} 127 128 129def _createBlank(cls): 130 """ 131 Given an object, if that object is a type, return a new, blank instance 132 of that type which has not had C{__init__} called on it. If the object 133 is not a type, return L{None}. 134 135 @param cls: The type (or class) to create an instance of. 136 @type cls: L{type} or something else that cannot be 137 instantiated. 138 139 @return: a new blank instance or L{None} if C{cls} is not a class or type. 140 """ 141 if isinstance(cls, type): 142 return cls.__new__(cls) 143 144 145def _newInstance(cls, state): 146 """ 147 Make a new instance of a class without calling its __init__ method. 148 149 @param state: A C{dict} used to update C{inst.__dict__} either directly or 150 via C{__setstate__}, if available. 151 152 @return: A new instance of C{cls}. 153 """ 154 instance = _createBlank(cls) 155 156 def defaultSetter(state): 157 instance.__dict__ = state 158 159 setter = getattr(instance, "__setstate__", defaultSetter) 160 setter(state) 161 return instance 162 163 164def _maybeClass(classnamep): 165 isObject = isinstance(classnamep, type) 166 167 if isObject: 168 classnamep = qual(classnamep) 169 170 if not isinstance(classnamep, bytes): 171 classnamep = classnamep.encode("utf-8") 172 173 return classnamep 174 175 176def setUnjellyableForClass(classname, unjellyable): 177 """ 178 Set which local class will represent a remote type. 179 180 If you have written a Copyable class that you expect your client to be 181 receiving, write a local "copy" class to represent it, then call:: 182 183 jellier.setUnjellyableForClass('module.package.Class', MyCopier). 184 185 Call this at the module level immediately after its class 186 definition. MyCopier should be a subclass of RemoteCopy. 187 188 The classname may be a special tag returned by 189 'Copyable.getTypeToCopyFor' rather than an actual classname. 190 191 This call is also for cached classes, since there will be no 192 overlap. The rules are the same. 193 """ 194 195 global unjellyableRegistry 196 classname = _maybeClass(classname) 197 unjellyableRegistry[classname] = unjellyable 198 globalSecurity.allowTypes(classname) 199 200 201def setUnjellyableFactoryForClass(classname, copyFactory): 202 """ 203 Set the factory to construct a remote instance of a type:: 204 205 jellier.setUnjellyableFactoryForClass('module.package.Class', MyFactory) 206 207 Call this at the module level immediately after its class definition. 208 C{copyFactory} should return an instance or subclass of 209 L{RemoteCopy<pb.RemoteCopy>}. 210 211 Similar to L{setUnjellyableForClass} except it uses a factory instead 212 of creating an instance. 213 """ 214 215 global unjellyableFactoryRegistry 216 classname = _maybeClass(classname) 217 unjellyableFactoryRegistry[classname] = copyFactory 218 globalSecurity.allowTypes(classname) 219 220 221def setUnjellyableForClassTree(module, baseClass, prefix=None): 222 """ 223 Set all classes in a module derived from C{baseClass} as copiers for 224 a corresponding remote class. 225 226 When you have a hierarchy of Copyable (or Cacheable) classes on one 227 side, and a mirror structure of Copied (or RemoteCache) classes on the 228 other, use this to setUnjellyableForClass all your Copieds for the 229 Copyables. 230 231 Each copyTag (the \"classname\" argument to getTypeToCopyFor, and 232 what the Copyable's getTypeToCopyFor returns) is formed from 233 adding a prefix to the Copied's class name. The prefix defaults 234 to module.__name__. If you wish the copy tag to consist of solely 235 the classname, pass the empty string \'\'. 236 237 @param module: a module object from which to pull the Copied classes. 238 (passing sys.modules[__name__] might be useful) 239 240 @param baseClass: the base class from which all your Copied classes derive. 241 242 @param prefix: the string prefixed to classnames to form the 243 unjellyableRegistry. 244 """ 245 if prefix is None: 246 prefix = module.__name__ 247 248 if prefix: 249 prefix = "%s." % prefix 250 251 for name in dir(module): 252 loaded = getattr(module, name) 253 try: 254 yes = issubclass(loaded, baseClass) 255 except TypeError: 256 "It's not a class." 257 else: 258 if yes: 259 setUnjellyableForClass(f"{prefix}{name}", loaded) 260 261 262def getInstanceState(inst, jellier): 263 """ 264 Utility method to default to 'normal' state rules in serialization. 265 """ 266 if hasattr(inst, "__getstate__"): 267 state = inst.__getstate__() 268 else: 269 state = inst.__dict__ 270 sxp = jellier.prepare(inst) 271 sxp.extend([qual(inst.__class__).encode("utf-8"), jellier.jelly(state)]) 272 return jellier.preserve(inst, sxp) 273 274 275def setInstanceState(inst, unjellier, jellyList): 276 """ 277 Utility method to default to 'normal' state rules in unserialization. 278 """ 279 state = unjellier.unjelly(jellyList[1]) 280 if hasattr(inst, "__setstate__"): 281 inst.__setstate__(state) 282 else: 283 inst.__dict__ = state 284 return inst 285 286 287class Unpersistable: 288 """ 289 This is an instance of a class that comes back when something couldn't be 290 unpersisted. 291 """ 292 293 def __init__(self, reason): 294 """ 295 Initialize an unpersistable object with a descriptive C{reason} string. 296 """ 297 self.reason = reason 298 299 def __repr__(self) -> str: 300 return "Unpersistable(%s)" % repr(self.reason) 301 302 303@implementer(IJellyable) 304class Jellyable: 305 """ 306 Inherit from me to Jelly yourself directly with the `getStateFor' 307 convenience method. 308 """ 309 310 def getStateFor(self, jellier): 311 return self.__dict__ 312 313 def jellyFor(self, jellier): 314 """ 315 @see: L{twisted.spread.interfaces.IJellyable.jellyFor} 316 """ 317 sxp = jellier.prepare(self) 318 sxp.extend( 319 [ 320 qual(self.__class__).encode("utf-8"), 321 jellier.jelly(self.getStateFor(jellier)), 322 ] 323 ) 324 return jellier.preserve(self, sxp) 325 326 327@implementer(IUnjellyable) 328class Unjellyable: 329 """ 330 Inherit from me to Unjelly yourself directly with the 331 C{setStateFor} convenience method. 332 """ 333 334 def setStateFor(self, unjellier, state): 335 self.__dict__ = state 336 337 def unjellyFor(self, unjellier, jellyList): 338 """ 339 Perform the inverse operation of L{Jellyable.jellyFor}. 340 341 @see: L{twisted.spread.interfaces.IUnjellyable.unjellyFor} 342 """ 343 state = unjellier.unjelly(jellyList[1]) 344 self.setStateFor(unjellier, state) 345 return self 346 347 348class _Jellier: 349 """ 350 (Internal) This class manages state for a call to jelly() 351 """ 352 353 def __init__(self, taster, persistentStore, invoker): 354 """ 355 Initialize. 356 """ 357 self.taster = taster 358 # `preserved' is a dict of previously seen instances. 359 self.preserved = {} 360 # `cooked' is a dict of previously backreferenced instances to their 361 # `ref' lists. 362 self.cooked = {} 363 self.cooker = {} 364 self._ref_id = 1 365 self.persistentStore = persistentStore 366 self.invoker = invoker 367 368 def _cook(self, object): 369 """ 370 (internal) Backreference an object. 371 372 Notes on this method for the hapless future maintainer: If I've already 373 gone through the prepare/preserve cycle on the specified object (it is 374 being referenced after the serializer is \"done with\" it, e.g. this 375 reference is NOT circular), the copy-in-place of aList is relevant, 376 since the list being modified is the actual, pre-existing jelly 377 expression that was returned for that object. If not, it's technically 378 superfluous, since the value in self.preserved didn't need to be set, 379 but the invariant that self.preserved[id(object)] is a list is 380 convenient because that means we don't have to test and create it or 381 not create it here, creating fewer code-paths. that's why 382 self.preserved is always set to a list. 383 384 Sorry that this code is so hard to follow, but Python objects are 385 tricky to persist correctly. -glyph 386 """ 387 aList = self.preserved[id(object)] 388 newList = copy.copy(aList) 389 # make a new reference ID 390 refid = self._ref_id 391 self._ref_id = self._ref_id + 1 392 # replace the old list in-place, so that we don't have to track the 393 # previous reference to it. 394 aList[:] = [reference_atom, refid, newList] 395 self.cooked[id(object)] = [dereference_atom, refid] 396 return aList 397 398 def prepare(self, object): 399 """ 400 (internal) Create a list for persisting an object to. This will allow 401 backreferences to be made internal to the object. (circular 402 references). 403 404 The reason this needs to happen is that we don't generate an ID for 405 every object, so we won't necessarily know which ID the object will 406 have in the future. When it is 'cooked' ( see _cook ), it will be 407 assigned an ID, and the temporary placeholder list created here will be 408 modified in-place to create an expression that gives this object an ID: 409 [reference id# [object-jelly]]. 410 """ 411 412 # create a placeholder list to be preserved 413 self.preserved[id(object)] = [] 414 # keep a reference to this object around, so it doesn't disappear! 415 # (This isn't always necessary, but for cases where the objects are 416 # dynamically generated by __getstate__ or getStateToCopyFor calls, it 417 # is; id() will return the same value for a different object if it gets 418 # garbage collected. This may be optimized later.) 419 self.cooker[id(object)] = object 420 return [] 421 422 def preserve(self, object, sexp): 423 """ 424 (internal) Mark an object's persistent list for later referral. 425 """ 426 # if I've been cooked in the meanwhile, 427 if id(object) in self.cooked: 428 # replace the placeholder empty list with the real one 429 self.preserved[id(object)][2] = sexp 430 # but give this one back. 431 sexp = self.preserved[id(object)] 432 else: 433 self.preserved[id(object)] = sexp 434 return sexp 435 436 def _checkMutable(self, obj): 437 objId = id(obj) 438 if objId in self.cooked: 439 return self.cooked[objId] 440 if objId in self.preserved: 441 self._cook(obj) 442 return self.cooked[objId] 443 444 def jelly(self, obj): 445 if isinstance(obj, Jellyable): 446 preRef = self._checkMutable(obj) 447 if preRef: 448 return preRef 449 return obj.jellyFor(self) 450 objType = type(obj) 451 if self.taster.isTypeAllowed(qual(objType).encode("utf-8")): 452 # "Immutable" Types 453 if objType in (bytes, int, float): 454 return obj 455 elif isinstance(obj, types.MethodType): 456 aSelf = obj.__self__ 457 aFunc = obj.__func__ 458 aClass = aSelf.__class__ 459 return [ 460 b"method", 461 aFunc.__name__, 462 self.jelly(aSelf), 463 self.jelly(aClass), 464 ] 465 elif objType is str: 466 return [b"unicode", obj.encode("UTF-8")] 467 elif isinstance(obj, type(None)): 468 return [b"None"] 469 elif isinstance(obj, types.FunctionType): 470 return [b"function", obj.__module__ + "." + obj.__qualname__] 471 elif isinstance(obj, types.ModuleType): 472 return [b"module", obj.__name__] 473 elif objType is bool: 474 return [b"boolean", obj and b"true" or b"false"] 475 elif objType is datetime.datetime: 476 if obj.tzinfo: 477 raise NotImplementedError( 478 "Currently can't jelly datetime objects with tzinfo" 479 ) 480 return [ 481 b"datetime", 482 " ".join( 483 [ 484 str(x) 485 for x in ( 486 obj.year, 487 obj.month, 488 obj.day, 489 obj.hour, 490 obj.minute, 491 obj.second, 492 obj.microsecond, 493 ) 494 ] 495 ).encode("utf-8"), 496 ] 497 elif objType is datetime.time: 498 if obj.tzinfo: 499 raise NotImplementedError( 500 "Currently can't jelly datetime objects with tzinfo" 501 ) 502 return [ 503 b"time", 504 " ".join( 505 [ 506 str(x) 507 for x in (obj.hour, obj.minute, obj.second, obj.microsecond) 508 ] 509 ).encode("utf-8"), 510 ] 511 elif objType is datetime.date: 512 return [ 513 b"date", 514 " ".join([str(x) for x in (obj.year, obj.month, obj.day)]).encode( 515 "utf-8" 516 ), 517 ] 518 elif objType is datetime.timedelta: 519 return [ 520 b"timedelta", 521 " ".join( 522 [str(x) for x in (obj.days, obj.seconds, obj.microseconds)] 523 ).encode("utf-8"), 524 ] 525 elif issubclass(objType, type): 526 return [b"class", qual(obj).encode("utf-8")] 527 elif objType is decimal.Decimal: 528 return self.jelly_decimal(obj) 529 else: 530 preRef = self._checkMutable(obj) 531 if preRef: 532 return preRef 533 # "Mutable" Types 534 sxp = self.prepare(obj) 535 if objType is list: 536 sxp.extend(self._jellyIterable(list_atom, obj)) 537 elif objType is tuple: 538 sxp.extend(self._jellyIterable(tuple_atom, obj)) 539 elif objType in DictTypes: 540 sxp.append(dictionary_atom) 541 for key, val in obj.items(): 542 sxp.append([self.jelly(key), self.jelly(val)]) 543 elif objType is set: 544 sxp.extend(self._jellyIterable(set_atom, obj)) 545 elif objType is frozenset: 546 sxp.extend(self._jellyIterable(frozenset_atom, obj)) 547 else: 548 className = qual(obj.__class__).encode("utf-8") 549 persistent = None 550 if self.persistentStore: 551 persistent = self.persistentStore(obj, self) 552 if persistent is not None: 553 sxp.append(persistent_atom) 554 sxp.append(persistent) 555 elif self.taster.isClassAllowed(obj.__class__): 556 sxp.append(className) 557 if hasattr(obj, "__getstate__"): 558 state = obj.__getstate__() 559 else: 560 state = obj.__dict__ 561 sxp.append(self.jelly(state)) 562 else: 563 self.unpersistable( 564 "instance of class %s deemed insecure" 565 % qual(obj.__class__), 566 sxp, 567 ) 568 return self.preserve(obj, sxp) 569 else: 570 raise InsecureJelly(f"Type not allowed for object: {objType} {obj}") 571 572 def _jellyIterable(self, atom, obj): 573 """ 574 Jelly an iterable object. 575 576 @param atom: the identifier atom of the object. 577 @type atom: C{str} 578 579 @param obj: any iterable object. 580 @type obj: C{iterable} 581 582 @return: a generator of jellied data. 583 @rtype: C{generator} 584 """ 585 yield atom 586 for item in obj: 587 yield self.jelly(item) 588 589 def jelly_decimal(self, d): 590 """ 591 Jelly a decimal object. 592 593 @param d: a decimal object to serialize. 594 @type d: C{decimal.Decimal} 595 596 @return: jelly for the decimal object. 597 @rtype: C{list} 598 """ 599 sign, guts, exponent = d.as_tuple() 600 value = reduce(lambda left, right: left * 10 + right, guts) 601 if sign: 602 value = -value 603 return [b"decimal", value, exponent] 604 605 def unpersistable(self, reason, sxp=None): 606 """ 607 (internal) Returns an sexp: (unpersistable "reason"). Utility method 608 for making note that a particular object could not be serialized. 609 """ 610 if sxp is None: 611 sxp = [] 612 sxp.append(unpersistable_atom) 613 if isinstance(reason, str): 614 reason = reason.encode("utf-8") 615 sxp.append(reason) 616 return sxp 617 618 619class _Unjellier: 620 def __init__(self, taster, persistentLoad, invoker): 621 self.taster = taster 622 self.persistentLoad = persistentLoad 623 self.references = {} 624 self.postCallbacks = [] 625 self.invoker = invoker 626 627 def unjellyFull(self, obj): 628 o = self.unjelly(obj) 629 for m in self.postCallbacks: 630 m() 631 return o 632 633 def _maybePostUnjelly(self, unjellied): 634 """ 635 If the given object has support for the C{postUnjelly} hook, set it up 636 to be called at the end of deserialization. 637 638 @param unjellied: an object that has already been unjellied. 639 640 @return: C{unjellied} 641 """ 642 if hasattr(unjellied, "postUnjelly"): 643 self.postCallbacks.append(unjellied.postUnjelly) 644 return unjellied 645 646 def unjelly(self, obj): 647 if type(obj) is not list: 648 return obj 649 jelTypeBytes = obj[0] 650 if not self.taster.isTypeAllowed(jelTypeBytes): 651 raise InsecureJelly(jelTypeBytes) 652 regClass = unjellyableRegistry.get(jelTypeBytes) 653 if regClass is not None: 654 method = getattr(_createBlank(regClass), "unjellyFor", regClass) 655 return self._maybePostUnjelly(method(self, obj)) 656 regFactory = unjellyableFactoryRegistry.get(jelTypeBytes) 657 if regFactory is not None: 658 return self._maybePostUnjelly(regFactory(self.unjelly(obj[1]))) 659 660 jelTypeText = nativeString(jelTypeBytes) 661 thunk = getattr(self, "_unjelly_%s" % jelTypeText, None) 662 if thunk is not None: 663 return thunk(obj[1:]) 664 else: 665 nameSplit = jelTypeText.split(".") 666 modName = ".".join(nameSplit[:-1]) 667 if not self.taster.isModuleAllowed(modName): 668 raise InsecureJelly( 669 f"Module {modName} not allowed (in type {jelTypeText})." 670 ) 671 clz = namedObject(jelTypeText) 672 if not self.taster.isClassAllowed(clz): 673 raise InsecureJelly("Class %s not allowed." % jelTypeText) 674 return self._genericUnjelly(clz, obj[1]) 675 676 def _genericUnjelly(self, cls, state): 677 """ 678 Unjelly a type for which no specific unjellier is registered, but which 679 is nonetheless allowed. 680 681 @param cls: the class of the instance we are unjellying. 682 @type cls: L{type} 683 684 @param state: The jellied representation of the object's state; its 685 C{__dict__} unless it has a C{__setstate__} that takes something 686 else. 687 @type state: L{list} 688 689 @return: the new, unjellied instance. 690 """ 691 return self._maybePostUnjelly(_newInstance(cls, self.unjelly(state))) 692 693 def _unjelly_None(self, exp): 694 return None 695 696 def _unjelly_unicode(self, exp): 697 return str(exp[0], "UTF-8") 698 699 def _unjelly_decimal(self, exp): 700 """ 701 Unjelly decimal objects. 702 """ 703 value = exp[0] 704 exponent = exp[1] 705 if value < 0: 706 sign = 1 707 else: 708 sign = 0 709 guts = decimal.Decimal(value).as_tuple()[1] 710 return decimal.Decimal((sign, guts, exponent)) 711 712 def _unjelly_boolean(self, exp): 713 assert exp[0] in (b"true", b"false") 714 return exp[0] == b"true" 715 716 def _unjelly_datetime(self, exp): 717 return datetime.datetime(*map(int, exp[0].split())) 718 719 def _unjelly_date(self, exp): 720 return datetime.date(*map(int, exp[0].split())) 721 722 def _unjelly_time(self, exp): 723 return datetime.time(*map(int, exp[0].split())) 724 725 def _unjelly_timedelta(self, exp): 726 days, seconds, microseconds = map(int, exp[0].split()) 727 return datetime.timedelta(days=days, seconds=seconds, microseconds=microseconds) 728 729 def unjellyInto(self, obj, loc, jel): 730 o = self.unjelly(jel) 731 if isinstance(o, NotKnown): 732 o.addDependant(obj, loc) 733 obj[loc] = o 734 return o 735 736 def _unjelly_dereference(self, lst): 737 refid = lst[0] 738 x = self.references.get(refid) 739 if x is not None: 740 return x 741 der = _Dereference(refid) 742 self.references[refid] = der 743 return der 744 745 def _unjelly_reference(self, lst): 746 refid = lst[0] 747 exp = lst[1] 748 o = self.unjelly(exp) 749 ref = self.references.get(refid) 750 if ref is None: 751 self.references[refid] = o 752 elif isinstance(ref, NotKnown): 753 ref.resolveDependants(o) 754 self.references[refid] = o 755 else: 756 assert 0, "Multiple references with same ID!" 757 return o 758 759 def _unjelly_tuple(self, lst): 760 l = list(range(len(lst))) 761 finished = 1 762 for elem in l: 763 if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown): 764 finished = 0 765 if finished: 766 return tuple(l) 767 else: 768 return _Tuple(l) 769 770 def _unjelly_list(self, lst): 771 l = list(range(len(lst))) 772 for elem in l: 773 self.unjellyInto(l, elem, lst[elem]) 774 return l 775 776 def _unjellySetOrFrozenset(self, lst, containerType): 777 """ 778 Helper method to unjelly set or frozenset. 779 780 @param lst: the content of the set. 781 @type lst: C{list} 782 783 @param containerType: the type of C{set} to use. 784 """ 785 l = list(range(len(lst))) 786 finished = True 787 for elem in l: 788 data = self.unjellyInto(l, elem, lst[elem]) 789 if isinstance(data, NotKnown): 790 finished = False 791 if not finished: 792 return _Container(l, containerType) 793 else: 794 return containerType(l) 795 796 def _unjelly_set(self, lst): 797 """ 798 Unjelly set using the C{set} builtin. 799 """ 800 return self._unjellySetOrFrozenset(lst, set) 801 802 def _unjelly_frozenset(self, lst): 803 """ 804 Unjelly frozenset using the C{frozenset} builtin. 805 """ 806 return self._unjellySetOrFrozenset(lst, frozenset) 807 808 def _unjelly_dictionary(self, lst): 809 d = {} 810 for k, v in lst: 811 kvd = _DictKeyAndValue(d) 812 self.unjellyInto(kvd, 0, k) 813 self.unjellyInto(kvd, 1, v) 814 return d 815 816 def _unjelly_module(self, rest): 817 moduleName = nativeString(rest[0]) 818 if type(moduleName) != str: 819 raise InsecureJelly("Attempted to unjelly a module with a non-string name.") 820 if not self.taster.isModuleAllowed(moduleName): 821 raise InsecureJelly(f"Attempted to unjelly module named {moduleName!r}") 822 mod = __import__(moduleName, {}, {}, "x") 823 return mod 824 825 def _unjelly_class(self, rest): 826 cname = nativeString(rest[0]) 827 clist = cname.split(nativeString(".")) 828 modName = nativeString(".").join(clist[:-1]) 829 if not self.taster.isModuleAllowed(modName): 830 raise InsecureJelly("module %s not allowed" % modName) 831 klaus = namedObject(cname) 832 objType = type(klaus) 833 if objType is not type: 834 raise InsecureJelly( 835 "class %r unjellied to something that isn't a class: %r" 836 % (cname, klaus) 837 ) 838 if not self.taster.isClassAllowed(klaus): 839 raise InsecureJelly("class not allowed: %s" % qual(klaus)) 840 return klaus 841 842 def _unjelly_function(self, rest): 843 fname = nativeString(rest[0]) 844 modSplit = fname.split(nativeString(".")) 845 modName = nativeString(".").join(modSplit[:-1]) 846 if not self.taster.isModuleAllowed(modName): 847 raise InsecureJelly("Module not allowed: %s" % modName) 848 # XXX do I need an isFunctionAllowed? 849 function = namedAny(fname) 850 return function 851 852 def _unjelly_persistent(self, rest): 853 if self.persistentLoad: 854 pload = self.persistentLoad(rest[0], self) 855 return pload 856 else: 857 return Unpersistable("Persistent callback not found") 858 859 def _unjelly_instance(self, rest): 860 """ 861 (internal) Unjelly an instance. 862 863 Called to handle the deprecated I{instance} token. 864 865 @param rest: The s-expression representing the instance. 866 867 @return: The unjellied instance. 868 """ 869 warnings.warn_explicit( 870 "Unjelly support for the instance atom is deprecated since " 871 "Twisted 15.0.0. Upgrade peer for modern instance support.", 872 category=DeprecationWarning, 873 filename="", 874 lineno=0, 875 ) 876 877 clz = self.unjelly(rest[0]) 878 return self._genericUnjelly(clz, rest[1]) 879 880 def _unjelly_unpersistable(self, rest): 881 return Unpersistable(f"Unpersistable data: {rest[0]}") 882 883 def _unjelly_method(self, rest): 884 """ 885 (internal) Unjelly a method. 886 """ 887 im_name = rest[0] 888 im_self = self.unjelly(rest[1]) 889 im_class = self.unjelly(rest[2]) 890 if not isinstance(im_class, type): 891 raise InsecureJelly("Method found with non-class class.") 892 if im_name in im_class.__dict__: 893 if im_self is None: 894 im = getattr(im_class, im_name) 895 elif isinstance(im_self, NotKnown): 896 im = _InstanceMethod(im_name, im_self, im_class) 897 else: 898 im = types.MethodType( 899 im_class.__dict__[im_name], im_self, *([im_class] * (False)) 900 ) 901 else: 902 raise TypeError("instance method changed") 903 return im 904 905 906#### Published Interface. 907 908 909class InsecureJelly(Exception): 910 """ 911 This exception will be raised when a jelly is deemed `insecure'; e.g. it 912 contains a type, class, or module disallowed by the specified `taster' 913 """ 914 915 916class DummySecurityOptions: 917 """ 918 DummySecurityOptions() -> insecure security options 919 Dummy security options -- this class will allow anything. 920 """ 921 922 def isModuleAllowed(self, moduleName): 923 """ 924 DummySecurityOptions.isModuleAllowed(moduleName) -> boolean 925 returns 1 if a module by that name is allowed, 0 otherwise 926 """ 927 return 1 928 929 def isClassAllowed(self, klass): 930 """ 931 DummySecurityOptions.isClassAllowed(class) -> boolean 932 Assumes the module has already been allowed. Returns 1 if the given 933 class is allowed, 0 otherwise. 934 """ 935 return 1 936 937 def isTypeAllowed(self, typeName): 938 """ 939 DummySecurityOptions.isTypeAllowed(typeName) -> boolean 940 Returns 1 if the given type is allowed, 0 otherwise. 941 """ 942 return 1 943 944 945class SecurityOptions: 946 """ 947 This will by default disallow everything, except for 'none'. 948 """ 949 950 basicTypes = [ 951 "dictionary", 952 "list", 953 "tuple", 954 "reference", 955 "dereference", 956 "unpersistable", 957 "persistent", 958 "long_int", 959 "long", 960 "dict", 961 ] 962 963 def __init__(self): 964 """ 965 SecurityOptions() initialize. 966 """ 967 # I don't believe any of these types can ever pose a security hazard, 968 # except perhaps "reference"... 969 self.allowedTypes = { 970 b"None": 1, 971 b"bool": 1, 972 b"boolean": 1, 973 b"string": 1, 974 b"str": 1, 975 b"int": 1, 976 b"float": 1, 977 b"datetime": 1, 978 b"time": 1, 979 b"date": 1, 980 b"timedelta": 1, 981 b"NoneType": 1, 982 b"unicode": 1, 983 b"decimal": 1, 984 b"set": 1, 985 b"frozenset": 1, 986 } 987 self.allowedModules = {} 988 self.allowedClasses = {} 989 990 def allowBasicTypes(self): 991 """ 992 Allow all `basic' types. (Dictionary and list. Int, string, and float 993 are implicitly allowed.) 994 """ 995 self.allowTypes(*self.basicTypes) 996 997 def allowTypes(self, *types): 998 """ 999 SecurityOptions.allowTypes(typeString): Allow a particular type, by its 1000 name. 1001 """ 1002 for typ in types: 1003 if isinstance(typ, str): 1004 typ = typ.encode("utf-8") 1005 if not isinstance(typ, bytes): 1006 typ = qual(typ) 1007 self.allowedTypes[typ] = 1 1008 1009 def allowInstancesOf(self, *classes): 1010 """ 1011 SecurityOptions.allowInstances(klass, klass, ...): allow instances 1012 of the specified classes 1013 1014 This will also allow the 'instance', 'class' (renamed 'classobj' in 1015 Python 2.3), and 'module' types, as well as basic types. 1016 """ 1017 self.allowBasicTypes() 1018 self.allowTypes("instance", "class", "classobj", "module") 1019 for klass in classes: 1020 self.allowTypes(qual(klass)) 1021 self.allowModules(klass.__module__) 1022 self.allowedClasses[klass] = 1 1023 1024 def allowModules(self, *modules): 1025 """ 1026 SecurityOptions.allowModules(module, module, ...): allow modules by 1027 name. This will also allow the 'module' type. 1028 """ 1029 for module in modules: 1030 if type(module) == types.ModuleType: 1031 module = module.__name__ 1032 1033 if not isinstance(module, bytes): 1034 module = module.encode("utf-8") 1035 1036 self.allowedModules[module] = 1 1037 1038 def isModuleAllowed(self, moduleName): 1039 """ 1040 SecurityOptions.isModuleAllowed(moduleName) -> boolean 1041 returns 1 if a module by that name is allowed, 0 otherwise 1042 """ 1043 if not isinstance(moduleName, bytes): 1044 moduleName = moduleName.encode("utf-8") 1045 1046 return moduleName in self.allowedModules 1047 1048 def isClassAllowed(self, klass): 1049 """ 1050 SecurityOptions.isClassAllowed(class) -> boolean 1051 Assumes the module has already been allowed. Returns 1 if the given 1052 class is allowed, 0 otherwise. 1053 """ 1054 return klass in self.allowedClasses 1055 1056 def isTypeAllowed(self, typeName): 1057 """ 1058 SecurityOptions.isTypeAllowed(typeName) -> boolean 1059 Returns 1 if the given type is allowed, 0 otherwise. 1060 """ 1061 if not isinstance(typeName, bytes): 1062 typeName = typeName.encode("utf-8") 1063 1064 return typeName in self.allowedTypes or b"." in typeName 1065 1066 1067globalSecurity = SecurityOptions() 1068globalSecurity.allowBasicTypes() 1069 1070 1071def jelly(object, taster=DummySecurityOptions(), persistentStore=None, invoker=None): 1072 """ 1073 Serialize to s-expression. 1074 1075 Returns a list which is the serialized representation of an object. An 1076 optional 'taster' argument takes a SecurityOptions and will mark any 1077 insecure objects as unpersistable rather than serializing them. 1078 """ 1079 return _Jellier(taster, persistentStore, invoker).jelly(object) 1080 1081 1082def unjelly(sexp, taster=DummySecurityOptions(), persistentLoad=None, invoker=None): 1083 """ 1084 Unserialize from s-expression. 1085 1086 Takes a list that was the result from a call to jelly() and unserializes 1087 an arbitrary object from it. The optional 'taster' argument, an instance 1088 of SecurityOptions, will cause an InsecureJelly exception to be raised if a 1089 disallowed type, module, or class attempted to unserialize. 1090 """ 1091 return _Unjellier(taster, persistentLoad, invoker).unjellyFull(sexp) 1092