1"""The :class:`Schema` class, including its metaclass and options (class Meta).""" 2from collections import defaultdict, OrderedDict 3from collections.abc import Mapping 4from functools import lru_cache 5import datetime as dt 6import uuid 7import decimal 8import copy 9import inspect 10import json 11import typing 12import warnings 13 14from marshmallow import base, fields as ma_fields, class_registry, types 15from marshmallow.error_store import ErrorStore 16from marshmallow.exceptions import ValidationError, StringNotCollectionError 17from marshmallow.orderedset import OrderedSet 18from marshmallow.decorators import ( 19 POST_DUMP, 20 POST_LOAD, 21 PRE_DUMP, 22 PRE_LOAD, 23 VALIDATES, 24 VALIDATES_SCHEMA, 25) 26from marshmallow.utils import ( 27 RAISE, 28 EXCLUDE, 29 INCLUDE, 30 missing, 31 set_value, 32 get_value, 33 is_collection, 34 is_instance_or_subclass, 35) 36from marshmallow.warnings import RemovedInMarshmallow4Warning 37 38_T = typing.TypeVar("_T") 39 40 41def _get_fields(attrs, ordered=False): 42 """Get fields from a class. If ordered=True, fields will sorted by creation index. 43 44 :param attrs: Mapping of class attributes 45 :param bool ordered: Sort fields by creation index 46 """ 47 fields = [ 48 (field_name, field_value) 49 for field_name, field_value in attrs.items() 50 if is_instance_or_subclass(field_value, base.FieldABC) 51 ] 52 if ordered: 53 fields.sort(key=lambda pair: pair[1]._creation_index) 54 return fields 55 56 57# This function allows Schemas to inherit from non-Schema classes and ensures 58# inheritance according to the MRO 59def _get_fields_by_mro(klass, ordered=False): 60 """Collect fields from a class, following its method resolution order. The 61 class itself is excluded from the search; only its parents are checked. Get 62 fields from ``_declared_fields`` if available, else use ``__dict__``. 63 64 :param type klass: Class whose fields to retrieve 65 """ 66 mro = inspect.getmro(klass) 67 # Loop over mro in reverse to maintain correct order of fields 68 return sum( 69 ( 70 _get_fields( 71 getattr(base, "_declared_fields", base.__dict__), 72 ordered=ordered, 73 ) 74 for base in mro[:0:-1] 75 ), 76 [], 77 ) 78 79 80class SchemaMeta(type): 81 """Metaclass for the Schema class. Binds the declared fields to 82 a ``_declared_fields`` attribute, which is a dictionary mapping attribute 83 names to field objects. Also sets the ``opts`` class attribute, which is 84 the Schema class's ``class Meta`` options. 85 """ 86 87 def __new__(mcs, name, bases, attrs): 88 meta = attrs.get("Meta") 89 ordered = getattr(meta, "ordered", False) 90 if not ordered: 91 # Inherit 'ordered' option 92 # Warning: We loop through bases instead of MRO because we don't 93 # yet have access to the class object 94 # (i.e. can't call super before we have fields) 95 for base_ in bases: 96 if hasattr(base_, "Meta") and hasattr(base_.Meta, "ordered"): 97 ordered = base_.Meta.ordered 98 break 99 else: 100 ordered = False 101 cls_fields = _get_fields(attrs, ordered=ordered) 102 # Remove fields from list of class attributes to avoid shadowing 103 # Schema attributes/methods in case of name conflict 104 for field_name, _ in cls_fields: 105 del attrs[field_name] 106 klass = super().__new__(mcs, name, bases, attrs) 107 inherited_fields = _get_fields_by_mro(klass, ordered=ordered) 108 109 meta = klass.Meta 110 # Set klass.opts in __new__ rather than __init__ so that it is accessible in 111 # get_declared_fields 112 klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered) 113 # Add fields specified in the `include` class Meta option 114 cls_fields += list(klass.opts.include.items()) 115 116 dict_cls = OrderedDict if ordered else dict 117 # Assign _declared_fields on class 118 klass._declared_fields = mcs.get_declared_fields( 119 klass=klass, 120 cls_fields=cls_fields, 121 inherited_fields=inherited_fields, 122 dict_cls=dict_cls, 123 ) 124 return klass 125 126 @classmethod 127 def get_declared_fields( 128 mcs, 129 klass: type, 130 cls_fields: typing.List, 131 inherited_fields: typing.List, 132 dict_cls: type, 133 ): 134 """Returns a dictionary of field_name => `Field` pairs declared on the class. 135 This is exposed mainly so that plugins can add additional fields, e.g. fields 136 computed from class Meta options. 137 138 :param klass: The class object. 139 :param cls_fields: The fields declared on the class, including those added 140 by the ``include`` class Meta option. 141 :param inherited_fields: Inherited fields. 142 :param dict_class: Either `dict` or `OrderedDict`, depending on the whether 143 the user specified `ordered=True`. 144 """ 145 return dict_cls(inherited_fields + cls_fields) 146 147 def __init__(cls, name, bases, attrs): 148 super().__init__(name, bases, attrs) 149 if name and cls.opts.register: 150 class_registry.register(name, cls) 151 cls._hooks = cls.resolve_hooks() 152 153 def resolve_hooks(cls) -> typing.Dict[types.Tag, typing.List[str]]: 154 """Add in the decorated processors 155 156 By doing this after constructing the class, we let standard inheritance 157 do all the hard work. 158 """ 159 mro = inspect.getmro(cls) 160 161 hooks = defaultdict(list) # type: typing.Dict[types.Tag, typing.List[str]] 162 163 for attr_name in dir(cls): 164 # Need to look up the actual descriptor, not whatever might be 165 # bound to the class. This needs to come from the __dict__ of the 166 # declaring class. 167 for parent in mro: 168 try: 169 attr = parent.__dict__[attr_name] 170 except KeyError: 171 continue 172 else: 173 break 174 else: 175 # In case we didn't find the attribute and didn't break above. 176 # We should never hit this - it's just here for completeness 177 # to exclude the possibility of attr being undefined. 178 continue 179 180 try: 181 hook_config = attr.__marshmallow_hook__ 182 except AttributeError: 183 pass 184 else: 185 for key in hook_config.keys(): 186 # Use name here so we can get the bound method later, in 187 # case the processor was a descriptor or something. 188 hooks[key].append(attr_name) 189 190 return hooks 191 192 193class SchemaOpts: 194 """class Meta options for the :class:`Schema`. Defines defaults.""" 195 196 def __init__(self, meta, ordered: bool = False): 197 self.fields = getattr(meta, "fields", ()) 198 if not isinstance(self.fields, (list, tuple)): 199 raise ValueError("`fields` option must be a list or tuple.") 200 self.additional = getattr(meta, "additional", ()) 201 if not isinstance(self.additional, (list, tuple)): 202 raise ValueError("`additional` option must be a list or tuple.") 203 if self.fields and self.additional: 204 raise ValueError( 205 "Cannot set both `fields` and `additional` options" 206 " for the same Schema." 207 ) 208 self.exclude = getattr(meta, "exclude", ()) 209 if not isinstance(self.exclude, (list, tuple)): 210 raise ValueError("`exclude` must be a list or tuple.") 211 self.dateformat = getattr(meta, "dateformat", None) 212 self.datetimeformat = getattr(meta, "datetimeformat", None) 213 self.timeformat = getattr(meta, "timeformat", None) 214 if hasattr(meta, "json_module"): 215 warnings.warn( 216 "The json_module class Meta option is deprecated. Use render_module instead.", 217 RemovedInMarshmallow4Warning, 218 ) 219 render_module = getattr(meta, "json_module", json) 220 else: 221 render_module = json 222 self.render_module = getattr(meta, "render_module", render_module) 223 self.ordered = getattr(meta, "ordered", ordered) 224 self.index_errors = getattr(meta, "index_errors", True) 225 self.include = getattr(meta, "include", {}) 226 self.load_only = getattr(meta, "load_only", ()) 227 self.dump_only = getattr(meta, "dump_only", ()) 228 self.unknown = getattr(meta, "unknown", RAISE) 229 self.register = getattr(meta, "register", True) 230 231 232class Schema(base.SchemaABC, metaclass=SchemaMeta): 233 """Base schema class with which to define custom schemas. 234 235 Example usage: 236 237 .. code-block:: python 238 239 import datetime as dt 240 from dataclasses import dataclass 241 242 from marshmallow import Schema, fields 243 244 245 @dataclass 246 class Album: 247 title: str 248 release_date: dt.date 249 250 251 class AlbumSchema(Schema): 252 title = fields.Str() 253 release_date = fields.Date() 254 255 256 album = Album("Beggars Banquet", dt.date(1968, 12, 6)) 257 schema = AlbumSchema() 258 data = schema.dump(album) 259 data # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'} 260 261 :param only: Whitelist of the declared fields to select when 262 instantiating the Schema. If None, all fields are used. Nested fields 263 can be represented with dot delimiters. 264 :param exclude: Blacklist of the declared fields to exclude 265 when instantiating the Schema. If a field appears in both `only` and 266 `exclude`, it is not used. Nested fields can be represented with dot 267 delimiters. 268 :param many: Should be set to `True` if ``obj`` is a collection 269 so that the object will be serialized to a list. 270 :param context: Optional context passed to :class:`fields.Method` and 271 :class:`fields.Function` fields. 272 :param load_only: Fields to skip during serialization (write-only fields) 273 :param dump_only: Fields to skip during deserialization (read-only fields) 274 :param partial: Whether to ignore missing fields and not require 275 any fields declared. Propagates down to ``Nested`` fields as well. If 276 its value is an iterable, only missing fields listed in that iterable 277 will be ignored. Use dot delimiters to specify nested fields. 278 :param unknown: Whether to exclude, include, or raise an error for unknown 279 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 280 281 .. versionchanged:: 3.0.0 282 `prefix` parameter removed. 283 284 .. versionchanged:: 2.0.0 285 `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of 286 `marshmallow.decorators.validates_schema`, 287 `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`. 288 `__accessor__` and `__error_handler__` are deprecated. Implement the 289 `handle_error` and `get_attribute` methods instead. 290 """ 291 292 TYPE_MAPPING = { 293 str: ma_fields.String, 294 bytes: ma_fields.String, 295 dt.datetime: ma_fields.DateTime, 296 float: ma_fields.Float, 297 bool: ma_fields.Boolean, 298 tuple: ma_fields.Raw, 299 list: ma_fields.Raw, 300 set: ma_fields.Raw, 301 int: ma_fields.Integer, 302 uuid.UUID: ma_fields.UUID, 303 dt.time: ma_fields.Time, 304 dt.date: ma_fields.Date, 305 dt.timedelta: ma_fields.TimeDelta, 306 decimal.Decimal: ma_fields.Decimal, 307 } # type: typing.Dict[type, typing.Type[ma_fields.Field]] 308 #: Overrides for default schema-level error messages 309 error_messages = {} # type: typing.Dict[str, str] 310 311 _default_error_messages = { 312 "type": "Invalid input type.", 313 "unknown": "Unknown field.", 314 } # type: typing.Dict[str, str] 315 316 OPTIONS_CLASS = SchemaOpts # type: type 317 318 # These get set by SchemaMeta 319 opts = None # type: SchemaOpts 320 _declared_fields = {} # type: typing.Dict[str, ma_fields.Field] 321 _hooks = {} # type: typing.Dict[types.Tag, typing.List[str]] 322 323 class Meta: 324 """Options object for a Schema. 325 326 Example usage: :: 327 328 class Meta: 329 fields = ("id", "email", "date_created") 330 exclude = ("password", "secret_attribute") 331 332 Available options: 333 334 - ``fields``: Tuple or list of fields to include in the serialized result. 335 - ``additional``: Tuple or list of fields to include *in addition* to the 336 explicitly declared fields. ``additional`` and ``fields`` are 337 mutually-exclusive options. 338 - ``include``: Dictionary of additional fields to include in the schema. It is 339 usually better to define fields as class variables, but you may need to 340 use this option, e.g., if your fields are Python keywords. May be an 341 `OrderedDict`. 342 - ``exclude``: Tuple or list of fields to exclude in the serialized result. 343 Nested fields can be represented with dot delimiters. 344 - ``dateformat``: Default format for `Date <fields.Date>` fields. 345 - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields. 346 - ``timeformat``: Default format for `Time <fields.Time>` fields. 347 - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`. 348 Defaults to `json` from the standard library. 349 - ``ordered``: If `True`, order serialization output according to the 350 order in which fields were declared. Output of `Schema.dump` will be a 351 `collections.OrderedDict`. 352 - ``index_errors``: If `True`, errors dictionaries will include the index 353 of invalid items in a collection. 354 - ``load_only``: Tuple or list of fields to exclude from serialized results. 355 - ``dump_only``: Tuple or list of fields to exclude from deserialization 356 - ``unknown``: Whether to exclude, include, or raise an error for unknown 357 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 358 - ``register``: Whether to register the `Schema` with marshmallow's internal 359 class registry. Must be `True` if you intend to refer to this `Schema` 360 by class name in `Nested` fields. Only set this to `False` when memory 361 usage is critical. Defaults to `True`. 362 """ 363 364 def __init__( 365 self, 366 *, 367 only: typing.Optional[types.StrSequenceOrSet] = None, 368 exclude: types.StrSequenceOrSet = (), 369 many: bool = False, 370 context: typing.Optional[typing.Dict] = None, 371 load_only: types.StrSequenceOrSet = (), 372 dump_only: types.StrSequenceOrSet = (), 373 partial: typing.Union[bool, types.StrSequenceOrSet] = False, 374 unknown: typing.Optional[str] = None, 375 ): 376 # Raise error if only or exclude is passed as string, not list of strings 377 if only is not None and not is_collection(only): 378 raise StringNotCollectionError('"only" should be a list of strings') 379 if not is_collection(exclude): 380 raise StringNotCollectionError('"exclude" should be a list of strings') 381 # copy declared fields from metaclass 382 self.declared_fields = copy.deepcopy(self._declared_fields) 383 self.many = many 384 self.only = only 385 self.exclude = set(self.opts.exclude) | set(exclude) 386 self.ordered = self.opts.ordered 387 self.load_only = set(load_only) or set(self.opts.load_only) 388 self.dump_only = set(dump_only) or set(self.opts.dump_only) 389 self.partial = partial 390 self.unknown = unknown or self.opts.unknown 391 self.context = context or {} 392 self._normalize_nested_options() 393 #: Dictionary mapping field_names -> :class:`Field` objects 394 self.fields = {} # type: typing.Dict[str, ma_fields.Field] 395 self.load_fields = {} # type: typing.Dict[str, ma_fields.Field] 396 self.dump_fields = {} # type: typing.Dict[str, ma_fields.Field] 397 self._init_fields() 398 messages = {} 399 messages.update(self._default_error_messages) 400 for cls in reversed(self.__class__.__mro__): 401 messages.update(getattr(cls, "error_messages", {})) 402 messages.update(self.error_messages or {}) 403 self.error_messages = messages 404 405 def __repr__(self) -> str: 406 return "<{ClassName}(many={self.many})>".format( 407 ClassName=self.__class__.__name__, self=self 408 ) 409 410 @property 411 def dict_class(self) -> type: 412 return OrderedDict if self.ordered else dict 413 414 @property 415 def set_class(self) -> type: 416 return OrderedSet if self.ordered else set 417 418 @classmethod 419 def from_dict( 420 cls, 421 fields: typing.Dict[str, typing.Union[ma_fields.Field, type]], 422 *, 423 name: str = "GeneratedSchema", 424 ) -> type: 425 """Generate a `Schema` class given a dictionary of fields. 426 427 .. code-block:: python 428 429 from marshmallow import Schema, fields 430 431 PersonSchema = Schema.from_dict({"name": fields.Str()}) 432 print(PersonSchema().load({"name": "David"})) # => {'name': 'David'} 433 434 Generated schemas are not added to the class registry and therefore cannot 435 be referred to by name in `Nested` fields. 436 437 :param dict fields: Dictionary mapping field names to field instances. 438 :param str name: Optional name for the class, which will appear in 439 the ``repr`` for the class. 440 441 .. versionadded:: 3.0.0 442 """ 443 attrs = fields.copy() 444 attrs["Meta"] = type( 445 "GeneratedMeta", (getattr(cls, "Meta", object),), {"register": False} 446 ) 447 schema_cls = type(name, (cls,), attrs) 448 return schema_cls 449 450 ##### Override-able methods ##### 451 452 def handle_error( 453 self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs 454 ): 455 """Custom error handler function for the schema. 456 457 :param error: The `ValidationError` raised during (de)serialization. 458 :param data: The original input data. 459 :param many: Value of ``many`` on dump or load. 460 :param partial: Value of ``partial`` on load. 461 462 .. versionadded:: 2.0.0 463 464 .. versionchanged:: 3.0.0rc9 465 Receives `many` and `partial` (on deserialization) as keyword arguments. 466 """ 467 pass 468 469 def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any): 470 """Defines how to pull values from an object to serialize. 471 472 .. versionadded:: 2.0.0 473 474 .. versionchanged:: 3.0.0a1 475 Changed position of ``obj`` and ``attr``. 476 """ 477 return get_value(obj, attr, default) 478 479 ##### Serialization/Deserialization API ##### 480 481 @staticmethod 482 def _call_and_store(getter_func, data, *, field_name, error_store, index=None): 483 """Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`. 484 485 :param callable getter_func: Function for getting the serialized/deserialized 486 value from ``data``. 487 :param data: The data passed to ``getter_func``. 488 :param str field_name: Field name. 489 :param int index: Index of the item being validated, if validating a collection, 490 otherwise `None`. 491 """ 492 try: 493 value = getter_func(data) 494 except ValidationError as error: 495 error_store.store_error(error.messages, field_name, index=index) 496 # When a Nested field fails validation, the marshalled data is stored 497 # on the ValidationError's valid_data attribute 498 return error.valid_data or missing 499 return value 500 501 def _serialize( 502 self, obj: typing.Union[_T, typing.Iterable[_T]], *, many: bool = False 503 ): 504 """Serialize ``obj``. 505 506 :param obj: The object(s) to serialize. 507 :param bool many: `True` if ``data`` should be serialized as a collection. 508 :return: A dictionary of the serialized data 509 510 .. versionchanged:: 1.0.0 511 Renamed from ``marshal``. 512 """ 513 if many and obj is not None: 514 return [ 515 self._serialize(d, many=False) 516 for d in typing.cast(typing.Iterable[_T], obj) 517 ] 518 ret = self.dict_class() 519 for attr_name, field_obj in self.dump_fields.items(): 520 value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute) 521 if value is missing: 522 continue 523 key = field_obj.data_key if field_obj.data_key is not None else attr_name 524 ret[key] = value 525 return ret 526 527 def dump(self, obj: typing.Any, *, many: typing.Optional[bool] = None): 528 """Serialize an object to native Python data types according to this 529 Schema's fields. 530 531 :param obj: The object to serialize. 532 :param many: Whether to serialize `obj` as a collection. If `None`, the value 533 for `self.many` is used. 534 :return: Serialized data 535 536 .. versionadded:: 1.0.0 537 .. versionchanged:: 3.0.0b7 538 This method returns the serialized data rather than a ``(data, errors)`` duple. 539 A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised 540 if ``obj`` is invalid. 541 .. versionchanged:: 3.0.0rc9 542 Validation no longer occurs upon serialization. 543 """ 544 many = self.many if many is None else bool(many) 545 if self._has_processors(PRE_DUMP): 546 processed_obj = self._invoke_dump_processors( 547 PRE_DUMP, obj, many=many, original_data=obj 548 ) 549 else: 550 processed_obj = obj 551 552 result = self._serialize(processed_obj, many=many) 553 554 if self._has_processors(POST_DUMP): 555 result = self._invoke_dump_processors( 556 POST_DUMP, result, many=many, original_data=obj 557 ) 558 559 return result 560 561 def dumps( 562 self, obj: typing.Any, *args, many: typing.Optional[bool] = None, **kwargs 563 ): 564 """Same as :meth:`dump`, except return a JSON-encoded string. 565 566 :param obj: The object to serialize. 567 :param many: Whether to serialize `obj` as a collection. If `None`, the value 568 for `self.many` is used. 569 :return: A ``json`` string 570 571 .. versionadded:: 1.0.0 572 .. versionchanged:: 3.0.0b7 573 This method returns the serialized data rather than a ``(data, errors)`` duple. 574 A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised 575 if ``obj`` is invalid. 576 """ 577 serialized = self.dump(obj, many=many) 578 return self.opts.render_module.dumps(serialized, *args, **kwargs) 579 580 def _deserialize( 581 self, 582 data: typing.Union[ 583 typing.Mapping[str, typing.Any], 584 typing.Iterable[typing.Mapping[str, typing.Any]], 585 ], 586 *, 587 error_store: ErrorStore, 588 many: bool = False, 589 partial=False, 590 unknown=RAISE, 591 index=None, 592 ) -> typing.Union[_T, typing.List[_T]]: 593 """Deserialize ``data``. 594 595 :param dict data: The data to deserialize. 596 :param ErrorStore error_store: Structure to store errors. 597 :param bool many: `True` if ``data`` should be deserialized as a collection. 598 :param bool|tuple partial: Whether to ignore missing fields and not require 599 any fields declared. Propagates down to ``Nested`` fields as well. If 600 its value is an iterable, only missing fields listed in that iterable 601 will be ignored. Use dot delimiters to specify nested fields. 602 :param unknown: Whether to exclude, include, or raise an error for unknown 603 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 604 :param int index: Index of the item being serialized (for storing errors) if 605 serializing a collection, otherwise `None`. 606 :return: A dictionary of the deserialized data. 607 """ 608 index_errors = self.opts.index_errors 609 index = index if index_errors else None 610 if many: 611 if not is_collection(data): 612 error_store.store_error([self.error_messages["type"]], index=index) 613 ret_l = [] # type: typing.List[_T] 614 else: 615 ret_l = [ 616 typing.cast( 617 _T, 618 self._deserialize( 619 typing.cast(typing.Mapping[str, typing.Any], d), 620 error_store=error_store, 621 many=False, 622 partial=partial, 623 unknown=unknown, 624 index=idx, 625 ), 626 ) 627 for idx, d in enumerate(data) 628 ] 629 return ret_l 630 ret_d = self.dict_class() 631 # Check data is a dict 632 if not isinstance(data, Mapping): 633 error_store.store_error([self.error_messages["type"]], index=index) 634 else: 635 partial_is_collection = is_collection(partial) 636 for attr_name, field_obj in self.load_fields.items(): 637 field_name = ( 638 field_obj.data_key if field_obj.data_key is not None else attr_name 639 ) 640 raw_value = data.get(field_name, missing) 641 if raw_value is missing: 642 # Ignore missing field if we're allowed to. 643 if partial is True or ( 644 partial_is_collection and attr_name in partial 645 ): 646 continue 647 d_kwargs = {} 648 # Allow partial loading of nested schemas. 649 if partial_is_collection: 650 prefix = field_name + "." 651 len_prefix = len(prefix) 652 sub_partial = [ 653 f[len_prefix:] for f in partial if f.startswith(prefix) 654 ] 655 d_kwargs["partial"] = sub_partial 656 else: 657 d_kwargs["partial"] = partial 658 getter = lambda val: field_obj.deserialize( 659 val, field_name, data, **d_kwargs 660 ) 661 value = self._call_and_store( 662 getter_func=getter, 663 data=raw_value, 664 field_name=field_name, 665 error_store=error_store, 666 index=index, 667 ) 668 if value is not missing: 669 key = field_obj.attribute or attr_name 670 set_value(ret_d, key, value) 671 if unknown != EXCLUDE: 672 fields = { 673 field_obj.data_key if field_obj.data_key is not None else field_name 674 for field_name, field_obj in self.load_fields.items() 675 } 676 for key in set(data) - fields: 677 value = data[key] 678 if unknown == INCLUDE: 679 ret_d[key] = value 680 elif unknown == RAISE: 681 error_store.store_error( 682 [self.error_messages["unknown"]], 683 key, 684 (index if index_errors else None), 685 ) 686 return ret_d 687 688 def load( 689 self, 690 data: typing.Union[ 691 typing.Mapping[str, typing.Any], 692 typing.Iterable[typing.Mapping[str, typing.Any]], 693 ], 694 *, 695 many: typing.Optional[bool] = None, 696 partial: typing.Optional[typing.Union[bool, types.StrSequenceOrSet]] = None, 697 unknown: typing.Optional[str] = None, 698 ): 699 """Deserialize a data structure to an object defined by this Schema's fields. 700 701 :param data: The data to deserialize. 702 :param many: Whether to deserialize `data` as a collection. If `None`, the 703 value for `self.many` is used. 704 :param partial: Whether to ignore missing fields and not require 705 any fields declared. Propagates down to ``Nested`` fields as well. If 706 its value is an iterable, only missing fields listed in that iterable 707 will be ignored. Use dot delimiters to specify nested fields. 708 :param unknown: Whether to exclude, include, or raise an error for unknown 709 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 710 If `None`, the value for `self.unknown` is used. 711 :return: Deserialized data 712 713 .. versionadded:: 1.0.0 714 .. versionchanged:: 3.0.0b7 715 This method returns the deserialized data rather than a ``(data, errors)`` duple. 716 A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised 717 if invalid data are passed. 718 """ 719 return self._do_load( 720 data, many=many, partial=partial, unknown=unknown, postprocess=True 721 ) 722 723 def loads( 724 self, 725 json_data: str, 726 *, 727 many: typing.Optional[bool] = None, 728 partial: typing.Optional[typing.Union[bool, types.StrSequenceOrSet]] = None, 729 unknown: typing.Optional[str] = None, 730 **kwargs, 731 ): 732 """Same as :meth:`load`, except it takes a JSON string as input. 733 734 :param json_data: A JSON string of the data to deserialize. 735 :param many: Whether to deserialize `obj` as a collection. If `None`, the 736 value for `self.many` is used. 737 :param partial: Whether to ignore missing fields and not require 738 any fields declared. Propagates down to ``Nested`` fields as well. If 739 its value is an iterable, only missing fields listed in that iterable 740 will be ignored. Use dot delimiters to specify nested fields. 741 :param unknown: Whether to exclude, include, or raise an error for unknown 742 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 743 If `None`, the value for `self.unknown` is used. 744 :return: Deserialized data 745 746 .. versionadded:: 1.0.0 747 .. versionchanged:: 3.0.0b7 748 This method returns the deserialized data rather than a ``(data, errors)`` duple. 749 A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised 750 if invalid data are passed. 751 """ 752 data = self.opts.render_module.loads(json_data, **kwargs) 753 return self.load(data, many=many, partial=partial, unknown=unknown) 754 755 def _run_validator( 756 self, 757 validator_func, 758 output, 759 *, 760 original_data, 761 error_store, 762 many, 763 partial, 764 pass_original, 765 index=None, 766 ): 767 try: 768 if pass_original: # Pass original, raw data (before unmarshalling) 769 validator_func(output, original_data, partial=partial, many=many) 770 else: 771 validator_func(output, partial=partial, many=many) 772 except ValidationError as err: 773 error_store.store_error(err.messages, err.field_name, index=index) 774 775 def validate( 776 self, 777 data: typing.Union[ 778 typing.Mapping[str, typing.Any], 779 typing.Iterable[typing.Mapping[str, typing.Any]], 780 ], 781 *, 782 many: typing.Optional[bool] = None, 783 partial: typing.Optional[typing.Union[bool, types.StrSequenceOrSet]] = None, 784 ) -> typing.Dict[str, typing.List[str]]: 785 """Validate `data` against the schema, returning a dictionary of 786 validation errors. 787 788 :param data: The data to validate. 789 :param many: Whether to validate `data` as a collection. If `None`, the 790 value for `self.many` is used. 791 :param partial: Whether to ignore missing fields and not require 792 any fields declared. Propagates down to ``Nested`` fields as well. If 793 its value is an iterable, only missing fields listed in that iterable 794 will be ignored. Use dot delimiters to specify nested fields. 795 :return: A dictionary of validation errors. 796 797 .. versionadded:: 1.1.0 798 """ 799 try: 800 self._do_load(data, many=many, partial=partial, postprocess=False) 801 except ValidationError as exc: 802 return typing.cast(typing.Dict[str, typing.List[str]], exc.messages) 803 return {} 804 805 ##### Private Helpers ##### 806 807 def _do_load( 808 self, 809 data: typing.Union[ 810 typing.Mapping[str, typing.Any], 811 typing.Iterable[typing.Mapping[str, typing.Any]], 812 ], 813 *, 814 many: typing.Optional[bool] = None, 815 partial: typing.Optional[typing.Union[bool, types.StrSequenceOrSet]] = None, 816 unknown: typing.Optional[str] = None, 817 postprocess: bool = True, 818 ): 819 """Deserialize `data`, returning the deserialized result. 820 This method is private API. 821 822 :param data: The data to deserialize. 823 :param many: Whether to deserialize `data` as a collection. If `None`, the 824 value for `self.many` is used. 825 :param partial: Whether to validate required fields. If its 826 value is an iterable, only fields listed in that iterable will be 827 ignored will be allowed missing. If `True`, all fields will be allowed missing. 828 If `None`, the value for `self.partial` is used. 829 :param unknown: Whether to exclude, include, or raise an error for unknown 830 fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. 831 If `None`, the value for `self.unknown` is used. 832 :param postprocess: Whether to run post_load methods.. 833 :return: Deserialized data 834 """ 835 error_store = ErrorStore() 836 errors = {} # type: typing.Dict[str, typing.List[str]] 837 many = self.many if many is None else bool(many) 838 unknown = unknown or self.unknown 839 if partial is None: 840 partial = self.partial 841 # Run preprocessors 842 if self._has_processors(PRE_LOAD): 843 try: 844 processed_data = self._invoke_load_processors( 845 PRE_LOAD, data, many=many, original_data=data, partial=partial 846 ) 847 except ValidationError as err: 848 errors = err.normalized_messages() 849 result = ( 850 None 851 ) # type: typing.Optional[typing.Union[typing.List, typing.Dict]] 852 else: 853 processed_data = data 854 if not errors: 855 # Deserialize data 856 result = self._deserialize( 857 processed_data, 858 error_store=error_store, 859 many=many, 860 partial=partial, 861 unknown=unknown, 862 ) 863 # Run field-level validation 864 self._invoke_field_validators( 865 error_store=error_store, data=result, many=many 866 ) 867 # Run schema-level validation 868 if self._has_processors(VALIDATES_SCHEMA): 869 field_errors = bool(error_store.errors) 870 self._invoke_schema_validators( 871 error_store=error_store, 872 pass_many=True, 873 data=result, 874 original_data=data, 875 many=many, 876 partial=partial, 877 field_errors=field_errors, 878 ) 879 self._invoke_schema_validators( 880 error_store=error_store, 881 pass_many=False, 882 data=result, 883 original_data=data, 884 many=many, 885 partial=partial, 886 field_errors=field_errors, 887 ) 888 errors = error_store.errors 889 # Run post processors 890 if not errors and postprocess and self._has_processors(POST_LOAD): 891 try: 892 result = self._invoke_load_processors( 893 POST_LOAD, 894 result, 895 many=many, 896 original_data=data, 897 partial=partial, 898 ) 899 except ValidationError as err: 900 errors = err.normalized_messages() 901 if errors: 902 exc = ValidationError(errors, data=data, valid_data=result) 903 self.handle_error(exc, data, many=many, partial=partial) 904 raise exc 905 906 return result 907 908 def _normalize_nested_options(self) -> None: 909 """Apply then flatten nested schema options. 910 This method is private API. 911 """ 912 if self.only is not None: 913 # Apply the only option to nested fields. 914 self.__apply_nested_option("only", self.only, "intersection") 915 # Remove the child field names from the only option. 916 self.only = self.set_class([field.split(".", 1)[0] for field in self.only]) 917 if self.exclude: 918 # Apply the exclude option to nested fields. 919 self.__apply_nested_option("exclude", self.exclude, "union") 920 # Remove the parent field names from the exclude option. 921 self.exclude = self.set_class( 922 [field for field in self.exclude if "." not in field] 923 ) 924 925 def __apply_nested_option(self, option_name, field_names, set_operation) -> None: 926 """Apply nested options to nested fields""" 927 # Split nested field names on the first dot. 928 nested_fields = [name.split(".", 1) for name in field_names if "." in name] 929 # Partition the nested field names by parent field. 930 nested_options = defaultdict(list) # type: defaultdict 931 for parent, nested_names in nested_fields: 932 nested_options[parent].append(nested_names) 933 # Apply the nested field options. 934 for key, options in iter(nested_options.items()): 935 new_options = self.set_class(options) 936 original_options = getattr(self.declared_fields[key], option_name, ()) 937 if original_options: 938 if set_operation == "union": 939 new_options |= self.set_class(original_options) 940 if set_operation == "intersection": 941 new_options &= self.set_class(original_options) 942 setattr(self.declared_fields[key], option_name, new_options) 943 944 def _init_fields(self) -> None: 945 """Update self.fields, self.load_fields, and self.dump_fields based on schema options. 946 This method is private API. 947 """ 948 if self.opts.fields: 949 available_field_names = self.set_class(self.opts.fields) 950 else: 951 available_field_names = self.set_class(self.declared_fields.keys()) 952 if self.opts.additional: 953 available_field_names |= self.set_class(self.opts.additional) 954 955 invalid_fields = self.set_class() 956 957 if self.only is not None: 958 # Return only fields specified in only option 959 field_names = self.set_class(self.only) 960 961 invalid_fields |= field_names - available_field_names 962 else: 963 field_names = available_field_names 964 965 # If "exclude" option or param is specified, remove those fields. 966 if self.exclude: 967 # Note that this isn't available_field_names, since we want to 968 # apply "only" for the actual calculation. 969 field_names = field_names - self.exclude 970 invalid_fields |= self.exclude - available_field_names 971 972 if invalid_fields: 973 message = f"Invalid fields for {self}: {invalid_fields}." 974 raise ValueError(message) 975 976 fields_dict = self.dict_class() 977 for field_name in field_names: 978 field_obj = self.declared_fields.get(field_name, ma_fields.Inferred()) 979 self._bind_field(field_name, field_obj) 980 fields_dict[field_name] = field_obj 981 982 load_fields, dump_fields = self.dict_class(), self.dict_class() 983 for field_name, field_obj in fields_dict.items(): 984 if not field_obj.dump_only: 985 load_fields[field_name] = field_obj 986 if not field_obj.load_only: 987 dump_fields[field_name] = field_obj 988 989 dump_data_keys = [ 990 field_obj.data_key if field_obj.data_key is not None else name 991 for name, field_obj in dump_fields.items() 992 ] 993 if len(dump_data_keys) != len(set(dump_data_keys)): 994 data_keys_duplicates = { 995 x for x in dump_data_keys if dump_data_keys.count(x) > 1 996 } 997 raise ValueError( 998 "The data_key argument for one or more fields collides " 999 "with another field's name or data_key argument. " 1000 "Check the following field names and " 1001 "data_key arguments: {}".format(list(data_keys_duplicates)) 1002 ) 1003 load_attributes = [obj.attribute or name for name, obj in load_fields.items()] 1004 if len(load_attributes) != len(set(load_attributes)): 1005 attributes_duplicates = { 1006 x for x in load_attributes if load_attributes.count(x) > 1 1007 } 1008 raise ValueError( 1009 "The attribute argument for one or more fields collides " 1010 "with another field's name or attribute argument. " 1011 "Check the following field names and " 1012 "attribute arguments: {}".format(list(attributes_duplicates)) 1013 ) 1014 1015 self.fields = fields_dict 1016 self.dump_fields = dump_fields 1017 self.load_fields = load_fields 1018 1019 def on_bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None: 1020 """Hook to modify a field when it is bound to the `Schema`. 1021 1022 No-op by default. 1023 """ 1024 return None 1025 1026 def _bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None: 1027 """Bind field to the schema, setting any necessary attributes on the 1028 field (e.g. parent and name). 1029 1030 Also set field load_only and dump_only values if field_name was 1031 specified in ``class Meta``. 1032 """ 1033 if field_name in self.load_only: 1034 field_obj.load_only = True 1035 if field_name in self.dump_only: 1036 field_obj.dump_only = True 1037 try: 1038 field_obj._bind_to_schema(field_name, self) 1039 except TypeError as error: 1040 # Field declared as a class, not an instance. Ignore type checking because 1041 # we handle unsupported arg types, i.e. this is dead code from 1042 # the type checker's perspective. 1043 if isinstance(field_obj, type) and issubclass(field_obj, base.FieldABC): 1044 msg = ( 1045 'Field for "{}" must be declared as a ' 1046 "Field instance, not a class. " 1047 'Did you mean "fields.{}()"?'.format(field_name, field_obj.__name__) 1048 ) 1049 raise TypeError(msg) from error 1050 raise error 1051 self.on_bind_field(field_name, field_obj) 1052 1053 @lru_cache(maxsize=8) 1054 def _has_processors(self, tag) -> bool: 1055 return bool(self._hooks[(tag, True)] or self._hooks[(tag, False)]) 1056 1057 def _invoke_dump_processors( 1058 self, tag: str, data, *, many: bool, original_data=None 1059 ): 1060 # The pass_many post-dump processors may do things like add an envelope, so 1061 # invoke those after invoking the non-pass_many processors which will expect 1062 # to get a list of items. 1063 data = self._invoke_processors( 1064 tag, pass_many=False, data=data, many=many, original_data=original_data 1065 ) 1066 data = self._invoke_processors( 1067 tag, pass_many=True, data=data, many=many, original_data=original_data 1068 ) 1069 return data 1070 1071 def _invoke_load_processors( 1072 self, 1073 tag: str, 1074 data, 1075 *, 1076 many: bool, 1077 original_data, 1078 partial: typing.Union[bool, types.StrSequenceOrSet], 1079 ): 1080 # This has to invert the order of the dump processors, so run the pass_many 1081 # processors first. 1082 data = self._invoke_processors( 1083 tag, 1084 pass_many=True, 1085 data=data, 1086 many=many, 1087 original_data=original_data, 1088 partial=partial, 1089 ) 1090 data = self._invoke_processors( 1091 tag, 1092 pass_many=False, 1093 data=data, 1094 many=many, 1095 original_data=original_data, 1096 partial=partial, 1097 ) 1098 return data 1099 1100 def _invoke_field_validators(self, *, error_store: ErrorStore, data, many: bool): 1101 for attr_name in self._hooks[VALIDATES]: 1102 validator = getattr(self, attr_name) 1103 validator_kwargs = validator.__marshmallow_hook__[VALIDATES] 1104 field_name = validator_kwargs["field_name"] 1105 1106 try: 1107 field_obj = self.fields[field_name] 1108 except KeyError as error: 1109 if field_name in self.declared_fields: 1110 continue 1111 raise ValueError(f'"{field_name}" field does not exist.') from error 1112 1113 data_key = ( 1114 field_obj.data_key if field_obj.data_key is not None else field_name 1115 ) 1116 if many: 1117 for idx, item in enumerate(data): 1118 try: 1119 value = item[field_obj.attribute or field_name] 1120 except KeyError: 1121 pass 1122 else: 1123 validated_value = self._call_and_store( 1124 getter_func=validator, 1125 data=value, 1126 field_name=data_key, 1127 error_store=error_store, 1128 index=(idx if self.opts.index_errors else None), 1129 ) 1130 if validated_value is missing: 1131 data[idx].pop(field_name, None) 1132 else: 1133 try: 1134 value = data[field_obj.attribute or field_name] 1135 except KeyError: 1136 pass 1137 else: 1138 validated_value = self._call_and_store( 1139 getter_func=validator, 1140 data=value, 1141 field_name=data_key, 1142 error_store=error_store, 1143 ) 1144 if validated_value is missing: 1145 data.pop(field_name, None) 1146 1147 def _invoke_schema_validators( 1148 self, 1149 *, 1150 error_store: ErrorStore, 1151 pass_many: bool, 1152 data, 1153 original_data, 1154 many: bool, 1155 partial: typing.Union[bool, types.StrSequenceOrSet], 1156 field_errors: bool = False, 1157 ): 1158 for attr_name in self._hooks[(VALIDATES_SCHEMA, pass_many)]: 1159 validator = getattr(self, attr_name) 1160 validator_kwargs = validator.__marshmallow_hook__[ 1161 (VALIDATES_SCHEMA, pass_many) 1162 ] 1163 if field_errors and validator_kwargs["skip_on_field_errors"]: 1164 continue 1165 pass_original = validator_kwargs.get("pass_original", False) 1166 1167 if many and not pass_many: 1168 for idx, (item, orig) in enumerate(zip(data, original_data)): 1169 self._run_validator( 1170 validator, 1171 item, 1172 original_data=orig, 1173 error_store=error_store, 1174 many=many, 1175 partial=partial, 1176 index=idx, 1177 pass_original=pass_original, 1178 ) 1179 else: 1180 self._run_validator( 1181 validator, 1182 data, 1183 original_data=original_data, 1184 error_store=error_store, 1185 many=many, 1186 pass_original=pass_original, 1187 partial=partial, 1188 ) 1189 1190 def _invoke_processors( 1191 self, 1192 tag: str, 1193 *, 1194 pass_many: bool, 1195 data, 1196 many: bool, 1197 original_data=None, 1198 **kwargs, 1199 ): 1200 key = (tag, pass_many) 1201 for attr_name in self._hooks[key]: 1202 # This will be a bound method. 1203 processor = getattr(self, attr_name) 1204 1205 processor_kwargs = processor.__marshmallow_hook__[key] 1206 pass_original = processor_kwargs.get("pass_original", False) 1207 1208 if many and not pass_many: 1209 if pass_original: 1210 data = [ 1211 processor(item, original, many=many, **kwargs) 1212 for item, original in zip(data, original_data) 1213 ] 1214 else: 1215 data = [processor(item, many=many, **kwargs) for item in data] 1216 else: 1217 if pass_original: 1218 data = processor(data, original_data, many=many, **kwargs) 1219 else: 1220 data = processor(data, many=many, **kwargs) 1221 return data 1222 1223 1224BaseSchema = Schema # for backwards compatibility 1225