1# Copyright 2013 IBM Corp. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may 4# not use this file except in compliance with the License. You may obtain 5# a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations 13# under the License. 14 15"""Common internal object model""" 16 17import abc 18import collections 19from collections import abc as collections_abc 20import copy 21import functools 22import logging 23import warnings 24 25import oslo_messaging as messaging 26from oslo_utils import excutils 27from oslo_utils import versionutils as vutils 28 29from oslo_versionedobjects._i18n import _ 30from oslo_versionedobjects import exception 31from oslo_versionedobjects import fields as obj_fields 32 33 34LOG = logging.getLogger('object') 35 36 37class _NotSpecifiedSentinel(object): 38 pass 39 40 41def _get_attrname(name): 42 """Return the mangled name of the attribute's underlying storage.""" 43 return '_obj_' + name 44 45 46def _make_class_properties(cls): 47 # NOTE(danms/comstud): Inherit fields from super classes. 48 # mro() returns the current class first and returns 'object' last, so 49 # those can be skipped. Also be careful to not overwrite any fields 50 # that already exist. And make sure each cls has its own copy of 51 # fields and that it is not sharing the dict with a super class. 52 cls.fields = dict(cls.fields) 53 for supercls in cls.mro()[1:-1]: 54 if not hasattr(supercls, 'fields'): 55 continue 56 for name, field in supercls.fields.items(): 57 if name not in cls.fields: 58 cls.fields[name] = field 59 for name, field in cls.fields.items(): 60 if not isinstance(field, obj_fields.Field): 61 raise exception.ObjectFieldInvalid( 62 field=name, objname=cls.obj_name()) 63 64 def getter(self, name=name): 65 attrname = _get_attrname(name) 66 if not hasattr(self, attrname): 67 self.obj_load_attr(name) 68 return getattr(self, attrname) 69 70 def setter(self, value, name=name, field=field): 71 attrname = _get_attrname(name) 72 field_value = field.coerce(self, name, value) 73 if field.read_only and hasattr(self, attrname): 74 # Note(yjiang5): _from_db_object() may iterate 75 # every field and write, no exception in such situation. 76 if getattr(self, attrname) != field_value: 77 raise exception.ReadOnlyFieldError(field=name) 78 else: 79 return 80 81 self._changed_fields.add(name) 82 try: 83 return setattr(self, attrname, field_value) 84 except Exception: 85 with excutils.save_and_reraise_exception(): 86 attr = "%s.%s" % (self.obj_name(), name) 87 LOG.exception('Error setting %(attr)s', 88 {'attr': attr}) 89 90 def deleter(self, name=name): 91 attrname = _get_attrname(name) 92 if not hasattr(self, attrname): 93 raise AttributeError("No such attribute `%s'" % name) 94 delattr(self, attrname) 95 96 setattr(cls, name, property(getter, setter, deleter)) 97 98 99class VersionedObjectRegistry(object): 100 _registry = None 101 102 def __new__(cls, *args, **kwargs): 103 if not VersionedObjectRegistry._registry: 104 VersionedObjectRegistry._registry = object.__new__( 105 VersionedObjectRegistry, *args, **kwargs) 106 VersionedObjectRegistry._registry._obj_classes = \ 107 collections.defaultdict(list) 108 self = object.__new__(cls, *args, **kwargs) 109 self._obj_classes = VersionedObjectRegistry._registry._obj_classes 110 return self 111 112 def registration_hook(self, cls, index): 113 pass 114 115 def _register_class(self, cls): 116 def _vers_tuple(obj): 117 return vutils.convert_version_to_tuple(obj.VERSION) 118 119 _make_class_properties(cls) 120 obj_name = cls.obj_name() 121 for i, obj in enumerate(self._obj_classes[obj_name]): 122 self.registration_hook(cls, i) 123 if cls.VERSION == obj.VERSION: 124 self._obj_classes[obj_name][i] = cls 125 break 126 if _vers_tuple(cls) > _vers_tuple(obj): 127 # Insert before. 128 self._obj_classes[obj_name].insert(i, cls) 129 break 130 else: 131 # Either this is the first time we've seen the object or it's 132 # an older version than anything we'e seen. 133 self._obj_classes[obj_name].append(cls) 134 self.registration_hook(cls, 0) 135 136 @classmethod 137 def register(cls, obj_cls): 138 registry = cls() 139 registry._register_class(obj_cls) 140 return obj_cls 141 142 @classmethod 143 def register_if(cls, condition): 144 def wraps(obj_cls): 145 if condition: 146 obj_cls = cls.register(obj_cls) 147 else: 148 _make_class_properties(obj_cls) 149 return obj_cls 150 return wraps 151 152 @classmethod 153 def objectify(cls, obj_cls): 154 return cls.register_if(False)(obj_cls) 155 156 @classmethod 157 def obj_classes(cls): 158 registry = cls() 159 return registry._obj_classes 160 161 162# These are decorators that mark an object's method as remotable. 163# If the metaclass is configured to forward object methods to an 164# indirection service, these will result in making an RPC call 165# instead of directly calling the implementation in the object. Instead, 166# the object implementation on the remote end will perform the 167# requested action and the result will be returned here. 168def remotable_classmethod(fn): 169 """Decorator for remotable classmethods.""" 170 @functools.wraps(fn) 171 def wrapper(cls, context, *args, **kwargs): 172 if cls.indirection_api: 173 version_manifest = obj_tree_get_versions(cls.obj_name()) 174 try: 175 result = cls.indirection_api.object_class_action_versions( 176 context, cls.obj_name(), fn.__name__, version_manifest, 177 args, kwargs) 178 except NotImplementedError: 179 # FIXME(danms): Maybe start to warn here about deprecation? 180 result = cls.indirection_api.object_class_action( 181 context, cls.obj_name(), fn.__name__, cls.VERSION, 182 args, kwargs) 183 else: 184 result = fn(cls, context, *args, **kwargs) 185 if isinstance(result, VersionedObject): 186 result._context = context 187 return result 188 189 # NOTE(danms): Make this discoverable 190 wrapper.remotable = True 191 wrapper.original_fn = fn 192 return classmethod(wrapper) 193 194 195# See comment above for remotable_classmethod() 196# 197# Note that this will use either the provided context, or the one 198# stashed in the object. If neither are present, the object is 199# "orphaned" and remotable methods cannot be called. 200def remotable(fn): 201 """Decorator for remotable object methods.""" 202 @functools.wraps(fn) 203 def wrapper(self, *args, **kwargs): 204 ctxt = self._context 205 if ctxt is None: 206 raise exception.OrphanedObjectError(method=fn.__name__, 207 objtype=self.obj_name()) 208 if self.indirection_api: 209 updates, result = self.indirection_api.object_action( 210 ctxt, self, fn.__name__, args, kwargs) 211 for key, value in updates.items(): 212 if key in self.fields: 213 field = self.fields[key] 214 # NOTE(ndipanov): Since VersionedObjectSerializer will have 215 # deserialized any object fields into objects already, 216 # we do not try to deserialize them again here. 217 if isinstance(value, VersionedObject): 218 setattr(self, key, value) 219 else: 220 setattr(self, key, 221 field.from_primitive(self, key, value)) 222 self.obj_reset_changes() 223 self._changed_fields = set(updates.get('obj_what_changed', [])) 224 return result 225 else: 226 return fn(self, *args, **kwargs) 227 228 wrapper.remotable = True 229 wrapper.original_fn = fn 230 return wrapper 231 232 233class VersionedObject(object): 234 """Base class and object factory. 235 236 This forms the base of all objects that can be remoted or instantiated 237 via RPC. Simply defining a class that inherits from this base class 238 will make it remotely instantiatable. Objects should implement the 239 necessary "get" classmethod routines as well as "save" object methods 240 as appropriate. 241 """ 242 243 indirection_api = None 244 245 # Object versioning rules 246 # 247 # Each service has its set of objects, each with a version attached. When 248 # a client attempts to call an object method, the server checks to see if 249 # the version of that object matches (in a compatible way) its object 250 # implementation. If so, cool, and if not, fail. 251 # 252 # This version is allowed to have three parts, X.Y.Z, where the .Z element 253 # is reserved for stable branch backports. The .Z is ignored for the 254 # purposes of triggering a backport, which means anything changed under 255 # a .Z must be additive and non-destructive such that a node that knows 256 # about X.Y can consider X.Y.Z equivalent. 257 VERSION = '1.0' 258 259 # Object namespace for serialization 260 # NB: Generally this should not be changed, but is needed for backwards 261 # compatibility 262 OBJ_SERIAL_NAMESPACE = 'versioned_object' 263 264 # Object project namespace for serialization 265 # This is used to disambiguate owners of objects sharing a common RPC 266 # medium 267 OBJ_PROJECT_NAMESPACE = 'versionedobjects' 268 269 # The fields present in this object as key:field pairs. For example: 270 # 271 # fields = { 'foo': obj_fields.IntegerField(), 272 # 'bar': obj_fields.StringField(), 273 # } 274 fields = {} 275 obj_extra_fields = [] 276 277 # Table of sub-object versioning information 278 # 279 # This contains a list of version mappings, by the field name of 280 # the subobject. The mappings must be in order of oldest to 281 # newest, and are tuples of (my_version, subobject_version). A 282 # request to backport this object to $my_version will cause the 283 # subobject to be backported to $subobject_version. 284 # 285 # obj_relationships = { 286 # 'subobject1': [('1.2', '1.1'), ('1.4', '1.2')], 287 # 'subobject2': [('1.2', '1.0')], 288 # } 289 # 290 # In the above example: 291 # 292 # - If we are asked to backport our object to version 1.3, 293 # subobject1 will be backported to version 1.1, since it was 294 # bumped to version 1.2 when our version was 1.4. 295 # - If we are asked to backport our object to version 1.5, 296 # no changes will be made to subobject1 or subobject2, since 297 # they have not changed since version 1.4. 298 # - If we are asked to backlevel our object to version 1.1, we 299 # will remove both subobject1 and subobject2 from the primitive, 300 # since they were not added until version 1.2. 301 obj_relationships = {} 302 303 def __init__(self, context=None, **kwargs): 304 self._changed_fields = set() 305 self._context = context 306 for key in kwargs.keys(): 307 setattr(self, key, kwargs[key]) 308 309 def __repr__(self): 310 repr_str = '%s(%s)' % ( 311 self.obj_name(), 312 ','.join(['%s=%s' % (name, 313 (self.obj_attr_is_set(name) and 314 field.stringify(getattr(self, name)) or 315 '<?>')) 316 for name, field in sorted(self.fields.items())])) 317 return repr_str 318 319 def __contains__(self, name): 320 try: 321 return self.obj_attr_is_set(name) 322 except AttributeError: 323 return False 324 325 @classmethod 326 def to_json_schema(cls): 327 obj_name = cls.obj_name() 328 schema = { 329 '$schema': 'http://json-schema.org/draft-04/schema#', 330 'title': obj_name, 331 } 332 schema.update(obj_fields.Object(obj_name).get_schema()) 333 return schema 334 335 @classmethod 336 def obj_name(cls): 337 """Return the object's name 338 339 Return a canonical name for this object which will be used over 340 the wire for remote hydration. 341 """ 342 return cls.__name__ 343 344 @classmethod 345 def _obj_primitive_key(cls, field): 346 return '%s.%s' % (cls.OBJ_SERIAL_NAMESPACE, field) 347 348 @classmethod 349 def _obj_primitive_field(cls, primitive, field, 350 default=obj_fields.UnspecifiedDefault): 351 key = cls._obj_primitive_key(field) 352 if default == obj_fields.UnspecifiedDefault: 353 return primitive[key] 354 else: 355 return primitive.get(key, default) 356 357 @classmethod 358 def obj_class_from_name(cls, objname, objver): 359 """Returns a class from the registry based on a name and version.""" 360 if objname not in VersionedObjectRegistry.obj_classes(): 361 LOG.error('Unable to instantiate unregistered object type ' 362 '%(objtype)s'), dict(objtype=objname) 363 raise exception.UnsupportedObjectError(objtype=objname) 364 365 # NOTE(comstud): If there's not an exact match, return the highest 366 # compatible version. The objects stored in the class are sorted 367 # such that highest version is first, so only set compatible_match 368 # once below. 369 compatible_match = None 370 371 for objclass in VersionedObjectRegistry.obj_classes()[objname]: 372 if objclass.VERSION == objver: 373 return objclass 374 if (not compatible_match and 375 vutils.is_compatible(objver, objclass.VERSION)): 376 compatible_match = objclass 377 378 if compatible_match: 379 return compatible_match 380 381 # As mentioned above, latest version is always first in the list. 382 latest_ver = VersionedObjectRegistry.obj_classes()[objname][0].VERSION 383 raise exception.IncompatibleObjectVersion(objname=objname, 384 objver=objver, 385 supported=latest_ver) 386 387 @classmethod 388 def _obj_from_primitive(cls, context, objver, primitive): 389 self = cls() 390 self._context = context 391 self.VERSION = objver 392 objdata = cls._obj_primitive_field(primitive, 'data') 393 changes = cls._obj_primitive_field(primitive, 'changes', []) 394 for name, field in self.fields.items(): 395 if name in objdata: 396 setattr(self, name, field.from_primitive(self, name, 397 objdata[name])) 398 self._changed_fields = set([x for x in changes if x in self.fields]) 399 return self 400 401 @classmethod 402 def obj_from_primitive(cls, primitive, context=None): 403 """Object field-by-field hydration.""" 404 objns = cls._obj_primitive_field(primitive, 'namespace') 405 objname = cls._obj_primitive_field(primitive, 'name') 406 objver = cls._obj_primitive_field(primitive, 'version') 407 if objns != cls.OBJ_PROJECT_NAMESPACE: 408 # NOTE(danms): We don't do anything with this now, but it's 409 # there for "the future" 410 raise exception.UnsupportedObjectError( 411 objtype='%s.%s' % (objns, objname)) 412 objclass = cls.obj_class_from_name(objname, objver) 413 return objclass._obj_from_primitive(context, objver, primitive) 414 415 def __deepcopy__(self, memo): 416 """Efficiently make a deep copy of this object.""" 417 418 # NOTE(danms): A naive deepcopy would copy more than we need, 419 # and since we have knowledge of the volatile bits of the 420 # object, we can be smarter here. Also, nested entities within 421 # some objects may be uncopyable, so we can avoid those sorts 422 # of issues by copying only our field data. 423 424 nobj = self.__class__() 425 426 # NOTE(sskripnick): we should save newly created object into mem 427 # to let deepcopy know which branches are already created. 428 # See launchpad bug #1602314 for more details 429 memo[id(self)] = nobj 430 nobj._context = self._context 431 for name in self.fields: 432 if self.obj_attr_is_set(name): 433 nval = copy.deepcopy(getattr(self, name), memo) 434 setattr(nobj, name, nval) 435 nobj._changed_fields = set(self._changed_fields) 436 return nobj 437 438 def obj_clone(self): 439 """Create a copy.""" 440 return copy.deepcopy(self) 441 442 def _obj_relationship_for(self, field, target_version): 443 # NOTE(danms): We need to be graceful about not having the temporary 444 # version manifest if called from obj_make_compatible(). 445 if (not hasattr(self, '_obj_version_manifest') or 446 self._obj_version_manifest is None): 447 try: 448 return self.obj_relationships[field] 449 except KeyError: 450 raise exception.ObjectActionError( 451 action='obj_make_compatible', 452 reason='No rule for %s' % field) 453 454 objname = self.fields[field].objname 455 if objname not in self._obj_version_manifest: 456 return 457 # NOTE(danms): Compute a relationship mapping that looks like 458 # what the caller expects. 459 return [(target_version, self._obj_version_manifest[objname])] 460 461 def _obj_make_obj_compatible(self, primitive, target_version, field): 462 """Backlevel a sub-object based on our versioning rules. 463 464 This is responsible for backporting objects contained within 465 this object's primitive according to a set of rules we 466 maintain about version dependencies between objects. This 467 requires that the obj_relationships table in this object is 468 correct and up-to-date. 469 470 :param:primitive: The primitive version of this object 471 :param:target_version: The version string requested for this object 472 :param:field: The name of the field in this object containing the 473 sub-object to be backported 474 """ 475 relationship_map = self._obj_relationship_for(field, target_version) 476 if not relationship_map: 477 # NOTE(danms): This means the field was not specified in the 478 # version manifest from the client, so it must not want this 479 # field, so skip. 480 return 481 482 try: 483 _get_subobject_version(target_version, 484 relationship_map, 485 lambda ver: _do_subobject_backport( 486 ver, self, field, primitive)) 487 except exception.TargetBeforeSubobjectExistedException: 488 # Subobject did not exist, so delete it from the primitive 489 del primitive[field] 490 491 def obj_make_compatible(self, primitive, target_version): 492 """Make an object representation compatible with a target version. 493 494 This is responsible for taking the primitive representation of 495 an object and making it suitable for the given target_version. 496 This may mean converting the format of object attributes, removing 497 attributes that have been added since the target version, etc. In 498 general: 499 500 - If a new version of an object adds a field, this routine 501 should remove it for older versions. 502 - If a new version changed or restricted the format of a field, this 503 should convert it back to something a client knowing only of the 504 older version will tolerate. 505 - If an object that this object depends on is bumped, then this 506 object should also take a version bump. Then, this routine should 507 backlevel the dependent object (by calling its obj_make_compatible()) 508 if the requested version of this object is older than the version 509 where the new dependent object was added. 510 511 :param primitive: The result of :meth:`obj_to_primitive` 512 :param target_version: The version string requested by the recipient 513 of the object 514 :raises: :exc:`oslo_versionedobjects.exception.UnsupportedObjectError` 515 if conversion is not possible for some reason 516 """ 517 for key, field in self.fields.items(): 518 if not isinstance(field, (obj_fields.ObjectField, 519 obj_fields.ListOfObjectsField)): 520 continue 521 if not self.obj_attr_is_set(key): 522 continue 523 self._obj_make_obj_compatible(primitive, target_version, key) 524 525 def obj_make_compatible_from_manifest(self, primitive, target_version, 526 version_manifest): 527 # NOTE(danms): Stash the manifest on the object so we can use it in 528 # the deeper layers. We do this because obj_make_compatible() is 529 # defined library API at this point, yet we need to get this manifest 530 # to the other bits that get called so we can propagate it to child 531 # calls. It's not pretty, but a tactical solution. Ideally we will 532 # either evolve or deprecate obj_make_compatible() in a major version 533 # bump. 534 self._obj_version_manifest = version_manifest 535 try: 536 return self.obj_make_compatible(primitive, target_version) 537 finally: 538 delattr(self, '_obj_version_manifest') 539 540 def obj_to_primitive(self, target_version=None, version_manifest=None): 541 """Simple base-case dehydration. 542 543 This calls to_primitive() for each item in fields. 544 """ 545 if target_version is None: 546 target_version = self.VERSION 547 if (vutils.convert_version_to_tuple(target_version) > 548 vutils.convert_version_to_tuple(self.VERSION)): 549 raise exception.InvalidTargetVersion(version=target_version) 550 primitive = dict() 551 for name, field in self.fields.items(): 552 if self.obj_attr_is_set(name): 553 primitive[name] = field.to_primitive(self, name, 554 getattr(self, name)) 555 # NOTE(danms): If we know we're being asked for a different version, 556 # then do the compat step. However, even if we think we're not, 557 # we may have sub-objects that need it, so if we have a manifest we 558 # have to traverse this object just in case. Previously, we 559 # required a parent version bump for any child, so the target 560 # check was enough. 561 if target_version != self.VERSION or version_manifest: 562 self.obj_make_compatible_from_manifest(primitive, 563 target_version, 564 version_manifest) 565 obj = {self._obj_primitive_key('name'): self.obj_name(), 566 self._obj_primitive_key('namespace'): ( 567 self.OBJ_PROJECT_NAMESPACE), 568 self._obj_primitive_key('version'): target_version, 569 self._obj_primitive_key('data'): primitive} 570 if self.obj_what_changed(): 571 # NOTE(cfriesen): if we're downgrading to a lower version, then 572 # it's possible that self.obj_what_changed() includes fields that 573 # no longer exist in the lower version. If so, filter them out. 574 what_changed = self.obj_what_changed() 575 changes = [field for field in what_changed if field in primitive] 576 if changes: 577 obj[self._obj_primitive_key('changes')] = changes 578 return obj 579 580 def obj_set_defaults(self, *attrs): 581 if not attrs: 582 attrs = [name for name, field in self.fields.items() 583 if field.default != obj_fields.UnspecifiedDefault] 584 585 for attr in attrs: 586 default = copy.deepcopy(self.fields[attr].default) 587 if default is obj_fields.UnspecifiedDefault: 588 raise exception.ObjectActionError( 589 action='set_defaults', 590 reason='No default set for field %s' % attr) 591 if not self.obj_attr_is_set(attr): 592 setattr(self, attr, default) 593 594 def obj_load_attr(self, attrname): 595 """Load an additional attribute from the real object. 596 597 This should load self.$attrname and cache any data that might 598 be useful for future load operations. 599 """ 600 raise NotImplementedError( 601 _("Cannot load '%s' in the base class") % attrname) 602 603 def save(self, context): 604 """Save the changed fields back to the store. 605 606 This is optional for subclasses, but is presented here in the base 607 class for consistency among those that do. 608 """ 609 raise NotImplementedError(_('Cannot save anything in the base class')) 610 611 def obj_what_changed(self): 612 """Returns a set of fields that have been modified.""" 613 changes = set([field for field in self._changed_fields 614 if field in self.fields]) 615 for field in self.fields: 616 if (self.obj_attr_is_set(field) and 617 isinstance(getattr(self, field), VersionedObject) and 618 getattr(self, field).obj_what_changed()): 619 changes.add(field) 620 return changes 621 622 def obj_get_changes(self): 623 """Returns a dict of changed fields and their new values.""" 624 changes = {} 625 for key in self.obj_what_changed(): 626 changes[key] = getattr(self, key) 627 return changes 628 629 def obj_reset_changes(self, fields=None, recursive=False): 630 """Reset the list of fields that have been changed. 631 632 :param fields: List of fields to reset, or "all" if None. 633 :param recursive: Call obj_reset_changes(recursive=True) on 634 any sub-objects within the list of fields 635 being reset. 636 637 This is NOT "revert to previous values". 638 639 Specifying fields on recursive resets will only be honored at the top 640 level. Everything below the top will reset all. 641 """ 642 if recursive: 643 for field in self.obj_get_changes(): 644 645 # Ignore fields not in requested set (if applicable) 646 if fields and field not in fields: 647 continue 648 649 # Skip any fields that are unset 650 if not self.obj_attr_is_set(field): 651 continue 652 653 value = getattr(self, field) 654 655 # Don't reset nulled fields 656 if value is None: 657 continue 658 659 # Reset straight Object and ListOfObjects fields 660 if isinstance(self.fields[field], obj_fields.ObjectField): 661 value.obj_reset_changes(recursive=True) 662 elif isinstance(self.fields[field], 663 obj_fields.ListOfObjectsField): 664 for thing in value: 665 thing.obj_reset_changes(recursive=True) 666 667 if fields: 668 self._changed_fields -= set(fields) 669 else: 670 self._changed_fields.clear() 671 672 def obj_attr_is_set(self, attrname): 673 """Test object to see if attrname is present. 674 675 Returns True if the named attribute has a value set, or 676 False if not. Raises AttributeError if attrname is not 677 a valid attribute for this object. 678 """ 679 if attrname not in self.obj_fields: 680 raise AttributeError( 681 _("%(objname)s object has no attribute '%(attrname)s'") % 682 {'objname': self.obj_name(), 'attrname': attrname}) 683 return hasattr(self, _get_attrname(attrname)) 684 685 @property 686 def obj_fields(self): 687 return list(self.fields.keys()) + self.obj_extra_fields 688 689 @property 690 def obj_context(self): 691 return self._context 692 693 694class ComparableVersionedObject(object): 695 """Mix-in to provide comparison methods 696 697 When objects are to be compared with each other (in tests for example), 698 this mixin can be used. 699 """ 700 def __eq__(self, obj): 701 # FIXME(inc0): this can return incorrect value if we consider partially 702 # loaded objects from db and fields which are dropped out differ 703 if hasattr(obj, 'obj_to_primitive'): 704 return self.obj_to_primitive() == obj.obj_to_primitive() 705 return NotImplemented 706 707 def __hash__(self): 708 return super(ComparableVersionedObject, self).__hash__() 709 710 def __ne__(self, obj): 711 if hasattr(obj, 'obj_to_primitive'): 712 return self.obj_to_primitive() != obj.obj_to_primitive() 713 return NotImplemented 714 715 716class TimestampedObject(object): 717 """Mixin class for db backed objects with timestamp fields. 718 719 Sqlalchemy models that inherit from the oslo_db TimestampMixin will include 720 these fields and the corresponding objects will benefit from this mixin. 721 """ 722 fields = { 723 'created_at': obj_fields.DateTimeField(nullable=True), 724 'updated_at': obj_fields.DateTimeField(nullable=True), 725 } 726 727 728class VersionedObjectDictCompat(object): 729 """Mix-in to provide dictionary key access compatibility 730 731 If an object needs to support attribute access using 732 dictionary items instead of object attributes, inherit 733 from this class. This should only be used as a temporary 734 measure until all callers are converted to use modern 735 attribute access. 736 """ 737 738 def __iter__(self): 739 for name in self.obj_fields: 740 if (self.obj_attr_is_set(name) or 741 name in self.obj_extra_fields): 742 yield name 743 744 keys = __iter__ 745 746 def values(self): 747 for name in self: 748 yield getattr(self, name) 749 750 def items(self): 751 for name in self: 752 yield name, getattr(self, name) 753 754 def __getitem__(self, name): 755 return getattr(self, name) 756 757 def __setitem__(self, name, value): 758 setattr(self, name, value) 759 760 def get(self, key, value=_NotSpecifiedSentinel): 761 if key not in self.obj_fields: 762 raise AttributeError("'%s' object has no attribute '%s'" % ( 763 self.__class__, key)) 764 if value != _NotSpecifiedSentinel and not self.obj_attr_is_set(key): 765 return value 766 else: 767 return getattr(self, key) 768 769 def update(self, updates): 770 for key, value in updates.items(): 771 setattr(self, key, value) 772 773 774class ObjectListBase(collections_abc.Sequence): 775 """Mixin class for lists of objects. 776 777 This mixin class can be added as a base class for an object that 778 is implementing a list of objects. It adds a single field of 'objects', 779 which is the list store, and behaves like a list itself. It supports 780 serialization of the list of objects automatically. 781 """ 782 fields = { 783 'objects': obj_fields.ListOfObjectsField('VersionedObject'), 784 } 785 786 # This is a dictionary of my_version:child_version mappings so that 787 # we can support backleveling our contents based on the version 788 # requested of the list object. 789 child_versions = {} 790 791 def __init__(self, *args, **kwargs): 792 super(ObjectListBase, self).__init__(*args, **kwargs) 793 if 'objects' not in kwargs: 794 self.objects = [] 795 self._changed_fields.discard('objects') 796 797 def __len__(self): 798 """List length.""" 799 return len(self.objects) 800 801 def __getitem__(self, index): 802 """List index access.""" 803 if isinstance(index, slice): 804 new_obj = self.__class__() 805 new_obj.objects = self.objects[index] 806 # NOTE(danms): We must be mixed in with a VersionedObject! 807 new_obj.obj_reset_changes() 808 new_obj._context = self._context 809 return new_obj 810 return self.objects[index] 811 812 def sort(self, key=None, reverse=False): 813 self.objects.sort(key=key, reverse=reverse) 814 815 def obj_make_compatible(self, primitive, target_version): 816 # Give priority to using child_versions, if that isn't set, try 817 # obj_relationships 818 if self.child_versions: 819 relationships = self.child_versions.items() 820 else: 821 try: 822 relationships = self._obj_relationship_for('objects', 823 target_version) 824 except exception.ObjectActionError: 825 # No relationship for this found in manifest or 826 # in obj_relationships 827 relationships = {} 828 829 try: 830 # NOTE(rlrossit): If we have no version information, just 831 # backport to child version 1.0 (maintaining default 832 # behavior) 833 if relationships: 834 _get_subobject_version(target_version, relationships, 835 lambda ver: _do_subobject_backport( 836 ver, self, 'objects', primitive)) 837 else: 838 _do_subobject_backport('1.0', self, 'objects', primitive) 839 except exception.TargetBeforeSubobjectExistedException: 840 # Child did not exist, so delete it from the primitive 841 del primitive['objects'] 842 843 def obj_what_changed(self): 844 changes = set(self._changed_fields) 845 for child in self.objects: 846 if child.obj_what_changed(): 847 changes.add('objects') 848 return changes 849 850 def __add__(self, other): 851 # Handling arbitrary fields may not make sense if those fields are not 852 # all concatenatable. Only concatenate if the base 'objects' field is 853 # the only one and the classes match. 854 if (self.__class__ == other.__class__ and 855 list(self.__class__.fields.keys()) == ['objects']): 856 return self.__class__(objects=self.objects + other.objects) 857 else: 858 raise TypeError("List Objects should be of the same type and only " 859 "have an 'objects' field") 860 861 def __radd__(self, other): 862 if (self.__class__ == other.__class__ and 863 list(self.__class__.fields.keys()) == ['objects']): 864 # This should never be run in practice. If the above condition is 865 # met then __add__ would have been run. 866 raise NotImplementedError('__radd__ is not implemented for ' 867 'objects of the same type') 868 else: 869 raise TypeError("List Objects should be of the same type and only " 870 "have an 'objects' field") 871 872 873class VersionedObjectSerializer(messaging.NoOpSerializer): 874 """A VersionedObject-aware Serializer. 875 876 This implements the Oslo Serializer interface and provides the 877 ability to serialize and deserialize VersionedObject entities. Any service 878 that needs to accept or return VersionedObjects as arguments or result 879 values should pass this to its RPCClient and RPCServer objects. 880 """ 881 882 # Base class to use for object hydration 883 OBJ_BASE_CLASS = VersionedObject 884 885 def _do_backport(self, context, objprim, objclass): 886 obj_versions = obj_tree_get_versions(objclass.obj_name()) 887 indirection_api = self.OBJ_BASE_CLASS.indirection_api 888 try: 889 return indirection_api.object_backport_versions( 890 context, objprim, obj_versions) 891 except NotImplementedError: 892 # FIXME(danms): Maybe start to warn here about deprecation? 893 return indirection_api.object_backport(context, objprim, 894 objclass.VERSION) 895 896 def _process_object(self, context, objprim): 897 try: 898 return self.OBJ_BASE_CLASS.obj_from_primitive( 899 objprim, context=context) 900 except exception.IncompatibleObjectVersion: 901 with excutils.save_and_reraise_exception(reraise=False) as ctxt: 902 verkey = \ 903 '%s.version' % self.OBJ_BASE_CLASS.OBJ_SERIAL_NAMESPACE 904 objver = objprim[verkey] 905 if objver.count('.') == 2: 906 # NOTE(danms): For our purposes, the .z part of the version 907 # should be safe to accept without requiring a backport 908 objprim[verkey] = \ 909 '.'.join(objver.split('.')[:2]) 910 return self._process_object(context, objprim) 911 namekey = '%s.name' % self.OBJ_BASE_CLASS.OBJ_SERIAL_NAMESPACE 912 objname = objprim[namekey] 913 supported = VersionedObjectRegistry.obj_classes().get(objname, 914 []) 915 if self.OBJ_BASE_CLASS.indirection_api and supported: 916 return self._do_backport(context, objprim, supported[0]) 917 else: 918 ctxt.reraise = True 919 920 def _process_iterable(self, context, action_fn, values): 921 """Process an iterable, taking an action on each value. 922 923 :param:context: Request context 924 :param:action_fn: Action to take on each item in values 925 :param:values: Iterable container of things to take action on 926 :returns: A new container of the same type (except set) with 927 items from values having had action applied. 928 """ 929 iterable = values.__class__ 930 if issubclass(iterable, dict): 931 return iterable([(k, action_fn(context, v)) 932 for k, v in values.items()]) 933 else: 934 # NOTE(danms, gibi) A set can't have an unhashable value inside, 935 # such as a dict. Convert the set to list, which is fine, since we 936 # can't send them over RPC anyway. We convert it to list as this 937 # way there will be no semantic change between the fake rpc driver 938 # used in functional test and a normal rpc driver. 939 if iterable == set: 940 iterable = list 941 return iterable([action_fn(context, value) for value in values]) 942 943 def serialize_entity(self, context, entity): 944 if isinstance(entity, (tuple, list, set, dict)): 945 entity = self._process_iterable(context, self.serialize_entity, 946 entity) 947 elif (hasattr(entity, 'obj_to_primitive') and 948 callable(entity.obj_to_primitive)): 949 entity = entity.obj_to_primitive() 950 return entity 951 952 def deserialize_entity(self, context, entity): 953 namekey = '%s.name' % self.OBJ_BASE_CLASS.OBJ_SERIAL_NAMESPACE 954 if isinstance(entity, dict) and namekey in entity: 955 entity = self._process_object(context, entity) 956 elif isinstance(entity, (tuple, list, set, dict)): 957 entity = self._process_iterable(context, self.deserialize_entity, 958 entity) 959 return entity 960 961 962class VersionedObjectIndirectionAPI(object, metaclass=abc.ABCMeta): 963 def object_action(self, context, objinst, objmethod, args, kwargs): 964 """Perform an action on a VersionedObject instance. 965 966 When indirection_api is set on a VersionedObject (to a class 967 implementing this interface), method calls on remotable methods 968 will cause this to be executed to actually make the desired 969 call. This often involves performing RPC. 970 971 :param context: The context within which to perform the action 972 :param objinst: The object instance on which to perform the action 973 :param objmethod: The name of the action method to call 974 :param args: The positional arguments to the action method 975 :param kwargs: The keyword arguments to the action method 976 :returns: The result of the action method 977 """ 978 pass 979 980 def object_class_action(self, context, objname, objmethod, objver, 981 args, kwargs): 982 """.. deprecated:: 0.10.0 983 984 Use :func:`object_class_action_versions` instead. 985 986 Perform an action on a VersionedObject class. 987 988 When indirection_api is set on a VersionedObject (to a class 989 implementing this interface), classmethod calls on 990 remotable_classmethod methods will cause this to be executed to 991 actually make the desired call. This usually involves performing 992 RPC. 993 994 :param context: The context within which to perform the action 995 :param objname: The registry name of the object 996 :param objmethod: The name of the action method to call 997 :param objver: The (remote) version of the object on which the 998 action is being taken 999 :param args: The positional arguments to the action method 1000 :param kwargs: The keyword arguments to the action method 1001 :returns: The result of the action method, which may (or may not) 1002 be an instance of the implementing VersionedObject class. 1003 """ 1004 pass 1005 1006 def object_class_action_versions(self, context, objname, objmethod, 1007 object_versions, args, kwargs): 1008 """Perform an action on a VersionedObject class. 1009 1010 When indirection_api is set on a VersionedObject (to a class 1011 implementing this interface), classmethod calls on 1012 remotable_classmethod methods will cause this to be executed to 1013 actually make the desired call. This usually involves performing 1014 RPC. 1015 1016 This differs from object_class_action() in that it is provided 1017 with object_versions, a manifest of client-side object versions 1018 for easier nested backports. The manifest is the result of 1019 calling obj_tree_get_versions(). 1020 1021 NOTE: This was not in the initial spec for this interface, so the 1022 base class raises NotImplementedError if you don't implement it. 1023 For backports, this method will be tried first, and if unimplemented, 1024 will fall back to object_class_action(). New implementations should 1025 provide this method instead of object_class_action() 1026 1027 :param context: The context within which to perform the action 1028 :param objname: The registry name of the object 1029 :param objmethod: The name of the action method to call 1030 :param object_versions: A dict of {objname: version} mappings 1031 :param args: The positional arguments to the action method 1032 :param kwargs: The keyword arguments to the action method 1033 :returns: The result of the action method, which may (or may not) 1034 be an instance of the implementing VersionedObject class. 1035 """ 1036 warnings.warn('object_class_action() is deprecated in favor of ' 1037 'object_class_action_versions() and will be removed ' 1038 'in a later release', DeprecationWarning) 1039 raise NotImplementedError('Multi-version class action not supported') 1040 1041 def object_backport(self, context, objinst, target_version): 1042 """.. deprecated:: 0.10.0 1043 1044 Use :func:`object_backport_versions` instead. 1045 1046 Perform a backport of an object instance to a specified version. 1047 1048 When indirection_api is set on a VersionedObject (to a class 1049 implementing this interface), the default behavior of the base 1050 VersionedObjectSerializer, upon receiving an object with a version 1051 newer than what is in the lcoal registry, is to call this method to 1052 request a backport of the object. In an environment where there is 1053 an RPC-able service on the bus which can gracefully downgrade newer 1054 objects for older services, this method services as a translation 1055 mechanism for older code when receiving objects from newer code. 1056 1057 NOTE: This older/original method is soon to be deprecated. When a 1058 backport is required, the newer object_backport_versions() will be 1059 tried, and if it raises NotImplementedError, then we will fall back 1060 to this (less optimal) method. 1061 1062 :param context: The context within which to perform the backport 1063 :param objinst: An instance of a VersionedObject to be backported 1064 :param target_version: The maximum version of the objinst's class 1065 that is understood by the requesting host. 1066 :returns: The downgraded instance of objinst 1067 """ 1068 pass 1069 1070 def object_backport_versions(self, context, objinst, object_versions): 1071 """Perform a backport of an object instance. 1072 1073 This method is basically just like object_backport() but instead of 1074 providing a specific target version for the toplevel object and 1075 relying on the service-side mapping to handle sub-objects, this sends 1076 a mapping of all the dependent objects and their client-supported 1077 versions. The server will backport objects within the tree starting 1078 at objinst to the versions specified in object_versions, removing 1079 objects that have no entry. Use obj_tree_get_versions() to generate 1080 this mapping. 1081 1082 NOTE: This was not in the initial spec for this interface, so the 1083 base class raises NotImplementedError if you don't implement it. 1084 For backports, this method will be tried first, and if unimplemented, 1085 will fall back to object_backport(). 1086 1087 :param context: The context within which to perform the backport 1088 :param objinst: An instance of a VersionedObject to be backported 1089 :param object_versions: A dict of {objname: version} mappings 1090 """ 1091 warnings.warn('object_backport() is deprecated in favor of ' 1092 'object_backport_versions() and will be removed ' 1093 'in a later release', DeprecationWarning) 1094 raise NotImplementedError('Multi-version backport not supported') 1095 1096 1097def obj_make_list(context, list_obj, item_cls, db_list, **extra_args): 1098 """Construct an object list from a list of primitives. 1099 1100 This calls item_cls._from_db_object() on each item of db_list, and 1101 adds the resulting object to list_obj. 1102 1103 :param:context: Request context 1104 :param:list_obj: An ObjectListBase object 1105 :param:item_cls: The VersionedObject class of the objects within the list 1106 :param:db_list: The list of primitives to convert to objects 1107 :param:extra_args: Extra arguments to pass to _from_db_object() 1108 :returns: list_obj 1109 """ 1110 list_obj.objects = [] 1111 for db_item in db_list: 1112 item = item_cls._from_db_object(context, item_cls(), db_item, 1113 **extra_args) 1114 list_obj.objects.append(item) 1115 list_obj._context = context 1116 list_obj.obj_reset_changes() 1117 return list_obj 1118 1119 1120def obj_tree_get_versions(objname, tree=None): 1121 """Construct a mapping of dependent object versions. 1122 1123 This method builds a list of dependent object versions given a top- 1124 level object with other objects as fields. It walks the tree recursively 1125 to determine all the objects (by symbolic name) that could be contained 1126 within the top-level object, and the maximum versions of each. The result 1127 is a dict like:: 1128 1129 {'MyObject': '1.23', ... } 1130 1131 :param objname: The top-level object at which to start 1132 :param tree: Used internally, pass None here. 1133 :returns: A dictionary of object names and versions 1134 """ 1135 if tree is None: 1136 tree = {} 1137 if objname in tree: 1138 return tree 1139 objclass = VersionedObjectRegistry.obj_classes()[objname][0] 1140 tree[objname] = objclass.VERSION 1141 for field_name in objclass.fields: 1142 field = objclass.fields[field_name] 1143 if isinstance(field, obj_fields.ObjectField): 1144 child_cls = field._type._obj_name 1145 elif isinstance(field, obj_fields.ListOfObjectsField): 1146 child_cls = field._type._element_type._type._obj_name 1147 else: 1148 continue 1149 1150 try: 1151 obj_tree_get_versions(child_cls, tree=tree) 1152 except IndexError: 1153 raise exception.UnregisteredSubobject( 1154 child_objname=child_cls, parent_objname=objname) 1155 return tree 1156 1157 1158def _get_subobject_version(tgt_version, relationships, backport_func): 1159 """Get the version to which we need to convert a subobject. 1160 1161 This uses the relationships between a parent and a subobject, 1162 along with the target parent version, to decide the version we need 1163 to convert a subobject to. If the subobject did not exist in the parent at 1164 the target version, TargetBeforeChildExistedException is raised. If there 1165 is a need to backport, backport_func is called and the subobject version 1166 to backport to is passed in. 1167 1168 :param tgt_version: The version we are converting the parent to 1169 :param relationships: A list of (parent, subobject) version tuples 1170 :param backport_func: A backport function that takes in the subobject 1171 version 1172 :returns: The version we need to convert the subobject to 1173 """ 1174 tgt = vutils.convert_version_to_tuple(tgt_version) 1175 for index, versions in enumerate(relationships): 1176 parent, child = versions 1177 parent = vutils.convert_version_to_tuple(parent) 1178 if tgt < parent: 1179 if index == 0: 1180 # We're backporting to a version of the parent that did 1181 # not contain this subobject 1182 raise exception.TargetBeforeSubobjectExistedException( 1183 target_version=tgt_version) 1184 else: 1185 # We're in a gap between index-1 and index, so set the desired 1186 # version to the previous index's version 1187 child = relationships[index - 1][1] 1188 backport_func(child) 1189 return 1190 elif tgt == parent: 1191 # We found the version we want, so backport to it 1192 backport_func(child) 1193 return 1194 1195 1196def _do_subobject_backport(to_version, parent, field, primitive): 1197 obj = getattr(parent, field) 1198 manifest = (hasattr(parent, '_obj_version_manifest') and 1199 parent._obj_version_manifest or None) 1200 if isinstance(obj, VersionedObject): 1201 obj.obj_make_compatible_from_manifest( 1202 obj._obj_primitive_field(primitive[field], 'data'), 1203 to_version, version_manifest=manifest) 1204 ver_key = obj._obj_primitive_key('version') 1205 primitive[field][ver_key] = to_version 1206 elif isinstance(obj, list): 1207 for i, element in enumerate(obj): 1208 element.obj_make_compatible_from_manifest( 1209 element._obj_primitive_field(primitive[field][i], 'data'), 1210 to_version, version_manifest=manifest) 1211 ver_key = element._obj_primitive_key('version') 1212 primitive[field][i][ver_key] = to_version 1213