1# -------------------------------------------------------------------------- 2# 3# Copyright (c) Microsoft Corporation. All rights reserved. 4# 5# The MIT License (MIT) 6# 7# Permission is hereby granted, free of charge, to any person obtaining a copy 8# of this software and associated documentation files (the ""Software""), to 9# deal in the Software without restriction, including without limitation the 10# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11# sell copies of the Software, and to permit persons to whom the Software is 12# furnished to do so, subject to the following conditions: 13# 14# The above copyright notice and this permission notice shall be included in 15# all copies or substantial portions of the Software. 16# 17# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23# IN THE SOFTWARE. 24# 25# -------------------------------------------------------------------------- 26 27from base64 import b64decode, b64encode 28import calendar 29import datetime 30import decimal 31import email 32from enum import Enum 33import json 34import logging 35import re 36import sys 37try: 38 from urllib import quote # type: ignore 39except ImportError: 40 from urllib.parse import quote # type: ignore 41import xml.etree.ElementTree as ET 42 43import isodate 44 45from typing import Dict, Any 46 47from .exceptions import ( 48 ValidationError, 49 SerializationError, 50 DeserializationError, 51 raise_with_traceback) 52 53try: 54 basestring # type: ignore 55 unicode_str = unicode # type: ignore 56except NameError: 57 basestring = str # type: ignore 58 unicode_str = str # type: ignore 59 60_LOGGER = logging.getLogger(__name__) 61 62try: 63 _long_type = long # type: ignore 64except NameError: 65 _long_type = int 66 67class UTC(datetime.tzinfo): 68 """Time Zone info for handling UTC""" 69 70 def utcoffset(self, dt): 71 """UTF offset for UTC is 0.""" 72 return datetime.timedelta(0) 73 74 def tzname(self, dt): 75 """Timestamp representation.""" 76 return "Z" 77 78 def dst(self, dt): 79 """No daylight saving for UTC.""" 80 return datetime.timedelta(hours=1) 81 82try: 83 from datetime import timezone as _FixedOffset 84except ImportError: # Python 2.7 85 class _FixedOffset(datetime.tzinfo): # type: ignore 86 """Fixed offset in minutes east from UTC. 87 Copy/pasted from Python doc 88 :param datetime.timedelta offset: offset in timedelta format 89 """ 90 91 def __init__(self, offset): 92 self.__offset = offset 93 94 def utcoffset(self, dt): 95 return self.__offset 96 97 def tzname(self, dt): 98 return str(self.__offset.total_seconds()/3600) 99 100 def __repr__(self): 101 return "<FixedOffset {}>".format(self.tzname(None)) 102 103 def dst(self, dt): 104 return datetime.timedelta(0) 105 106 def __getinitargs__(self): 107 return (self.__offset,) 108 109try: 110 from datetime import timezone 111 TZ_UTC = timezone.utc # type: ignore 112except ImportError: 113 TZ_UTC = UTC() # type: ignore 114 115_FLATTEN = re.compile(r"(?<!\\)\.") 116 117def attribute_transformer(key, attr_desc, value): 118 """A key transformer that returns the Python attribute. 119 120 :param str key: The attribute name 121 :param dict attr_desc: The attribute metadata 122 :param object value: The value 123 :returns: A key using attribute name 124 """ 125 return (key, value) 126 127def full_restapi_key_transformer(key, attr_desc, value): 128 """A key transformer that returns the full RestAPI key path. 129 130 :param str _: The attribute name 131 :param dict attr_desc: The attribute metadata 132 :param object value: The value 133 :returns: A list of keys using RestAPI syntax. 134 """ 135 keys = _FLATTEN.split(attr_desc['key']) 136 return ([_decode_attribute_map_key(k) for k in keys], value) 137 138def last_restapi_key_transformer(key, attr_desc, value): 139 """A key transformer that returns the last RestAPI key. 140 141 :param str key: The attribute name 142 :param dict attr_desc: The attribute metadata 143 :param object value: The value 144 :returns: The last RestAPI key. 145 """ 146 key, value = full_restapi_key_transformer(key, attr_desc, value) 147 return (key[-1], value) 148 149def _recursive_validate(attr_name, attr_type, data): 150 result = [] 151 if attr_type.startswith('[') and data is not None: 152 for content in data: 153 result += _recursive_validate(attr_name+" content", attr_type[1:-1], content) 154 elif attr_type.startswith('{') and data is not None: 155 if not hasattr(data, 'values'): 156 raise ValidationError("type", attr_name, attr_type) 157 for content in data.values(): 158 result += _recursive_validate(attr_name+" content", attr_type[1:-1], content) 159 elif hasattr(data, '_validation'): 160 return data.validate() 161 return result 162 163def _create_xml_node(tag, prefix=None, ns=None): 164 """Create a XML node.""" 165 if prefix and ns: 166 ET.register_namespace(prefix, ns) 167 if ns: 168 return ET.Element("{"+ns+"}"+tag) 169 else: 170 return ET.Element(tag) 171 172class Model(object): 173 """Mixin for all client request body/response body models to support 174 serialization and deserialization. 175 """ 176 177 _subtype_map = {} # type: Dict[str, Dict[str, Any]] 178 _attribute_map = {} # type: Dict[str, Dict[str, Any]] 179 _validation = {} # type: Dict[str, Dict[str, Any]] 180 181 def __init__(self, **kwargs): 182 self.additional_properties = {} 183 for k in kwargs: 184 if k not in self._attribute_map: 185 _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) 186 elif k in self._validation and self._validation[k].get("readonly", False): 187 _LOGGER.warning("Readonly attribute %s will be ignored in class %s", k, self.__class__) 188 else: 189 setattr(self, k, kwargs[k]) 190 191 def __eq__(self, other): 192 """Compare objects by comparing all attributes.""" 193 if isinstance(other, self.__class__): 194 return self.__dict__ == other.__dict__ 195 return False 196 197 def __ne__(self, other): 198 """Compare objects by comparing all attributes.""" 199 return not self.__eq__(other) 200 201 def __str__(self): 202 return str(self.__dict__) 203 204 @classmethod 205 def enable_additional_properties_sending(cls): 206 cls._attribute_map['additional_properties'] = {'key': '', 'type': '{object}'} 207 208 @classmethod 209 def is_xml_model(cls): 210 try: 211 cls._xml_map 212 except AttributeError: 213 return False 214 return True 215 216 @classmethod 217 def _create_xml_node(cls): 218 """Create XML node. 219 """ 220 try: 221 xml_map = cls._xml_map 222 except AttributeError: 223 xml_map = {} 224 225 return _create_xml_node( 226 xml_map.get('name', cls.__name__), 227 xml_map.get("prefix", None), 228 xml_map.get("ns", None) 229 ) 230 231 def validate(self): 232 """Validate this model recursively and return a list of ValidationError. 233 234 :returns: A list of validation error 235 :rtype: list 236 """ 237 validation_result = [] 238 for attr_name, value in [(attr, getattr(self, attr)) for attr in self._attribute_map]: 239 attr_desc = self._attribute_map[attr_name] 240 if attr_name == "additional_properties" and attr_desc["key"] == '': 241 # Do NOT validate additional_properties 242 continue 243 attr_type = attr_desc['type'] 244 245 try: 246 debug_name = "{}.{}".format(self.__class__.__name__, attr_name) 247 # https://github.com/Azure/msrest-for-python/issues/85 248 if value is not None and attr_type in Serializer.basic_types.values(): 249 value = Serializer.serialize_basic(value, attr_type) 250 Serializer.validate(value, debug_name, **self._validation.get(attr_name, {})) 251 except ValidationError as validation_error: 252 validation_result.append(validation_error) 253 254 validation_result += _recursive_validate(attr_name, attr_type, value) 255 return validation_result 256 257 def serialize(self, keep_readonly=False, **kwargs): 258 """Return the JSON that would be sent to azure from this model. 259 260 This is an alias to `as_dict(full_restapi_key_transformer, keep_readonly=False)`. 261 262 If you want XML serialization, you can pass the kwargs is_xml=True. 263 264 :param bool keep_readonly: If you want to serialize the readonly attributes 265 :returns: A dict JSON compatible object 266 :rtype: dict 267 """ 268 serializer = Serializer(self._infer_class_models()) 269 return serializer._serialize(self, keep_readonly=keep_readonly, **kwargs) 270 271 def as_dict(self, keep_readonly=True, key_transformer=attribute_transformer, **kwargs): 272 """Return a dict that can be JSONify using json.dump. 273 274 Advanced usage might optionaly use a callback as parameter: 275 276 .. code::python 277 278 def my_key_transformer(key, attr_desc, value): 279 return key 280 281 Key is the attribute name used in Python. Attr_desc 282 is a dict of metadata. Currently contains 'type' with the 283 msrest type and 'key' with the RestAPI encoded key. 284 Value is the current value in this object. 285 286 The string returned will be used to serialize the key. 287 If the return type is a list, this is considered hierarchical 288 result dict. 289 290 See the three examples in this file: 291 292 - attribute_transformer 293 - full_restapi_key_transformer 294 - last_restapi_key_transformer 295 296 If you want XML serialization, you can pass the kwargs is_xml=True. 297 298 :param function key_transformer: A key transformer function. 299 :returns: A dict JSON compatible object 300 :rtype: dict 301 """ 302 serializer = Serializer(self._infer_class_models()) 303 return serializer._serialize(self, key_transformer=key_transformer, keep_readonly=keep_readonly, **kwargs) 304 305 @classmethod 306 def _infer_class_models(cls): 307 try: 308 str_models = cls.__module__.rsplit('.', 1)[0] 309 models = sys.modules[str_models] 310 client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} 311 if cls.__name__ not in client_models: 312 raise ValueError("Not Autorest generated code") 313 except Exception: 314 # Assume it's not Autorest generated (tests?). Add ourselves as dependencies. 315 client_models = {cls.__name__: cls} 316 return client_models 317 318 @classmethod 319 def deserialize(cls, data, content_type=None): 320 """Parse a str using the RestAPI syntax and return a model. 321 322 :param str data: A str using RestAPI structure. JSON by default. 323 :param str content_type: JSON by default, set application/xml if XML. 324 :returns: An instance of this model 325 :raises: DeserializationError if something went wrong 326 """ 327 deserializer = Deserializer(cls._infer_class_models()) 328 return deserializer(cls.__name__, data, content_type=content_type) 329 330 @classmethod 331 def from_dict(cls, data, key_extractors=None, content_type=None): 332 """Parse a dict using given key extractor return a model. 333 334 By default consider key 335 extractors (rest_key_case_insensitive_extractor, attribute_key_case_insensitive_extractor 336 and last_rest_key_case_insensitive_extractor) 337 338 :param dict data: A dict using RestAPI structure 339 :param str content_type: JSON by default, set application/xml if XML. 340 :returns: An instance of this model 341 :raises: DeserializationError if something went wrong 342 """ 343 deserializer = Deserializer(cls._infer_class_models()) 344 deserializer.key_extractors = [ 345 attribute_key_case_insensitive_extractor, 346 rest_key_case_insensitive_extractor, 347 last_rest_key_case_insensitive_extractor 348 ] if key_extractors is None else key_extractors 349 return deserializer(cls.__name__, data, content_type=content_type) 350 351 @classmethod 352 def _flatten_subtype(cls, key, objects): 353 if '_subtype_map' not in cls.__dict__: 354 return {} 355 result = dict(cls._subtype_map[key]) 356 for valuetype in cls._subtype_map[key].values(): 357 result.update(objects[valuetype]._flatten_subtype(key, objects)) 358 return result 359 360 @classmethod 361 def _classify(cls, response, objects): 362 """Check the class _subtype_map for any child classes. 363 We want to ignore any inherited _subtype_maps. 364 Remove the polymorphic key from the initial data. 365 """ 366 for subtype_key in cls.__dict__.get('_subtype_map', {}).keys(): 367 subtype_value = None 368 369 if not isinstance(response, ET.Element): 370 rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1] 371 subtype_value = response.pop(rest_api_response_key, None) or response.pop(subtype_key, None) 372 else: 373 subtype_value = xml_key_extractor( 374 subtype_key, 375 cls._attribute_map[subtype_key], 376 response 377 ) 378 if subtype_value: 379 # Try to match base class. Can be class name only 380 # (bug to fix in Autorest to support x-ms-discriminator-name) 381 if cls.__name__ == subtype_value: 382 return cls 383 flatten_mapping_type = cls._flatten_subtype(subtype_key, objects) 384 try: 385 return objects[flatten_mapping_type[subtype_value]] 386 except KeyError: 387 _LOGGER.warning( 388 "Subtype value %s has no mapping, use base class %s.", 389 subtype_value, 390 cls.__name__, 391 ) 392 break 393 else: 394 _LOGGER.warning( 395 "Discriminator %s is absent or null, use base class %s.", 396 subtype_key, 397 cls.__name__ 398 ) 399 break 400 return cls 401 402 @classmethod 403 def _get_rest_key_parts(cls, attr_key): 404 """Get the RestAPI key of this attr, split it and decode part 405 :param str attr_key: Attribute key must be in attribute_map. 406 :returns: A list of RestAPI part 407 :rtype: list 408 """ 409 rest_split_key = _FLATTEN.split(cls._attribute_map[attr_key]['key']) 410 return [_decode_attribute_map_key(key_part) for key_part in rest_split_key] 411 412 413def _decode_attribute_map_key(key): 414 """This decode a key in an _attribute_map to the actual key we want to look at 415 inside the received data. 416 417 :param str key: A key string from the generated code 418 """ 419 return key.replace('\\.', '.') 420 421 422class Serializer(object): 423 """Request object model serializer.""" 424 425 basic_types = {str: 'str', int: 'int', bool: 'bool', float: 'float'} 426 427 _xml_basic_types_serializers = {'bool': lambda x:str(x).lower()} 428 days = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 429 4: "Fri", 5: "Sat", 6: "Sun"} 430 months = {1: "Jan", 2: "Feb", 3: "Mar", 4: "Apr", 5: "May", 6: "Jun", 431 7: "Jul", 8: "Aug", 9: "Sep", 10: "Oct", 11: "Nov", 12: "Dec"} 432 validation = { 433 "min_length": lambda x, y: len(x) < y, 434 "max_length": lambda x, y: len(x) > y, 435 "minimum": lambda x, y: x < y, 436 "maximum": lambda x, y: x > y, 437 "minimum_ex": lambda x, y: x <= y, 438 "maximum_ex": lambda x, y: x >= y, 439 "min_items": lambda x, y: len(x) < y, 440 "max_items": lambda x, y: len(x) > y, 441 "pattern": lambda x, y: not re.match(y, x, re.UNICODE), 442 "unique": lambda x, y: len(x) != len(set(x)), 443 "multiple": lambda x, y: x % y != 0 444 } 445 446 def __init__(self, classes=None): 447 self.serialize_type = { 448 'iso-8601': Serializer.serialize_iso, 449 'rfc-1123': Serializer.serialize_rfc, 450 'unix-time': Serializer.serialize_unix, 451 'duration': Serializer.serialize_duration, 452 'date': Serializer.serialize_date, 453 'time': Serializer.serialize_time, 454 'decimal': Serializer.serialize_decimal, 455 'long': Serializer.serialize_long, 456 'bytearray': Serializer.serialize_bytearray, 457 'base64': Serializer.serialize_base64, 458 'object': self.serialize_object, 459 '[]': self.serialize_iter, 460 '{}': self.serialize_dict 461 } 462 self.dependencies = dict(classes) if classes else {} 463 self.key_transformer = full_restapi_key_transformer 464 self.client_side_validation = True 465 466 def _serialize(self, target_obj, data_type=None, **kwargs): 467 """Serialize data into a string according to type. 468 469 :param target_obj: The data to be serialized. 470 :param str data_type: The type to be serialized from. 471 :rtype: str, dict 472 :raises: SerializationError if serialization fails. 473 """ 474 key_transformer = kwargs.get("key_transformer", self.key_transformer) 475 keep_readonly = kwargs.get("keep_readonly", False) 476 if target_obj is None: 477 return None 478 479 attr_name = None 480 class_name = target_obj.__class__.__name__ 481 482 if data_type: 483 return self.serialize_data( 484 target_obj, data_type, **kwargs) 485 486 if not hasattr(target_obj, "_attribute_map"): 487 data_type = type(target_obj).__name__ 488 if data_type in self.basic_types.values(): 489 return self.serialize_data( 490 target_obj, data_type, **kwargs) 491 492 # Force "is_xml" kwargs if we detect a XML model 493 try: 494 is_xml_model_serialization = kwargs["is_xml"] 495 except KeyError: 496 is_xml_model_serialization = kwargs.setdefault("is_xml", target_obj.is_xml_model()) 497 498 serialized = {} 499 if is_xml_model_serialization: 500 serialized = target_obj._create_xml_node() 501 try: 502 attributes = target_obj._attribute_map 503 for attr, attr_desc in attributes.items(): 504 attr_name = attr 505 if not keep_readonly and target_obj._validation.get(attr_name, {}).get('readonly', False): 506 continue 507 508 if attr_name == "additional_properties" and attr_desc["key"] == '': 509 if target_obj.additional_properties is not None: 510 serialized.update(target_obj.additional_properties) 511 continue 512 try: 513 ### Extract sub-data to serialize from model ### 514 orig_attr = getattr(target_obj, attr) 515 if is_xml_model_serialization: 516 pass # Don't provide "transformer" for XML for now. Keep "orig_attr" 517 else: # JSON 518 keys, orig_attr = key_transformer(attr, attr_desc.copy(), orig_attr) 519 keys = keys if isinstance(keys, list) else [keys] 520 521 ### Serialize this data ### 522 kwargs["serialization_ctxt"] = attr_desc 523 new_attr = self.serialize_data(orig_attr, attr_desc['type'], **kwargs) 524 525 ### Incorporate this data in the right place ### 526 if is_xml_model_serialization: 527 xml_desc = attr_desc.get('xml', {}) 528 xml_name = xml_desc.get('name', attr_desc['key']) 529 xml_prefix = xml_desc.get('prefix', None) 530 xml_ns = xml_desc.get('ns', None) 531 if xml_desc.get("attr", False): 532 if xml_ns: 533 ET.register_namespace(xml_prefix, xml_ns) 534 xml_name = "{{{}}}{}".format(xml_ns, xml_name) 535 serialized.set(xml_name, new_attr) 536 continue 537 if xml_desc.get("text", False): 538 serialized.text = new_attr 539 continue 540 if isinstance(new_attr, list): 541 serialized.extend(new_attr) 542 elif isinstance(new_attr, ET.Element): 543 # If the down XML has no XML/Name, we MUST replace the tag with the local tag. But keeping the namespaces. 544 if 'name' not in getattr(orig_attr, '_xml_map', {}): 545 splitted_tag = new_attr.tag.split("}") 546 if len(splitted_tag) == 2: # Namespace 547 new_attr.tag = "}".join([splitted_tag[0], xml_name]) 548 else: 549 new_attr.tag = xml_name 550 serialized.append(new_attr) 551 else: # That's a basic type 552 # Integrate namespace if necessary 553 local_node = _create_xml_node( 554 xml_name, 555 xml_prefix, 556 xml_ns 557 ) 558 local_node.text = unicode_str(new_attr) 559 serialized.append(local_node) 560 else: # JSON 561 for k in reversed(keys): 562 unflattened = {k: new_attr} 563 new_attr = unflattened 564 565 _new_attr = new_attr 566 _serialized = serialized 567 for k in keys: 568 if k not in _serialized: 569 _serialized.update(_new_attr) 570 _new_attr = _new_attr[k] 571 _serialized = _serialized[k] 572 except ValueError: 573 continue 574 575 except (AttributeError, KeyError, TypeError) as err: 576 msg = "Attribute {} in object {} cannot be serialized.\n{}".format( 577 attr_name, class_name, str(target_obj)) 578 raise_with_traceback(SerializationError, msg, err) 579 else: 580 return serialized 581 582 def body(self, data, data_type, **kwargs): 583 """Serialize data intended for a request body. 584 585 :param data: The data to be serialized. 586 :param str data_type: The type to be serialized from. 587 :rtype: dict 588 :raises: SerializationError if serialization fails. 589 :raises: ValueError if data is None 590 """ 591 if data is None: 592 raise ValidationError("required", "body", True) 593 594 # Just in case this is a dict 595 internal_data_type = data_type.strip('[]{}') 596 internal_data_type = self.dependencies.get(internal_data_type, None) 597 try: 598 is_xml_model_serialization = kwargs["is_xml"] 599 except KeyError: 600 if internal_data_type and issubclass(internal_data_type, Model): 601 is_xml_model_serialization = kwargs.setdefault("is_xml", internal_data_type.is_xml_model()) 602 else: 603 is_xml_model_serialization = False 604 if internal_data_type and not isinstance(internal_data_type, Enum): 605 try: 606 deserializer = Deserializer(self.dependencies) 607 # Since it's on serialization, it's almost sure that format is not JSON REST 608 # We're not able to deal with additional properties for now. 609 deserializer.additional_properties_detection = False 610 if is_xml_model_serialization: 611 deserializer.key_extractors = [ 612 attribute_key_case_insensitive_extractor, 613 ] 614 else: 615 deserializer.key_extractors = [ 616 rest_key_case_insensitive_extractor, 617 attribute_key_case_insensitive_extractor, 618 last_rest_key_case_insensitive_extractor 619 ] 620 data = deserializer._deserialize(data_type, data) 621 except DeserializationError as err: 622 raise_with_traceback( 623 SerializationError, "Unable to build a model: "+str(err), err) 624 625 if self.client_side_validation: 626 errors = _recursive_validate(data_type, data_type, data) 627 if errors: 628 raise errors[0] 629 return self._serialize(data, data_type, **kwargs) 630 631 def _http_component_validation(self, data, data_type, name, **kwargs): 632 if self.client_side_validation: 633 # https://github.com/Azure/msrest-for-python/issues/85 634 if data is not None and data_type in self.basic_types.values(): 635 data = self.serialize_basic(data, data_type, **kwargs) 636 data = self.validate(data, name, required=True, **kwargs) 637 return data 638 639 def url(self, name, data, data_type, **kwargs): 640 """Serialize data intended for a URL path. 641 642 :param data: The data to be serialized. 643 :param str data_type: The type to be serialized from. 644 :rtype: str 645 :raises: TypeError if serialization fails. 646 :raises: ValueError if data is None 647 """ 648 data = self._http_component_validation(data, data_type, name, **kwargs) 649 try: 650 output = self.serialize_data(data, data_type, **kwargs) 651 if data_type == 'bool': 652 output = json.dumps(output) 653 654 if kwargs.get('skip_quote') is True: 655 output = str(output) 656 else: 657 output = quote(str(output), safe='') 658 except SerializationError: 659 raise TypeError("{} must be type {}.".format(name, data_type)) 660 else: 661 return output 662 663 def query(self, name, data, data_type, **kwargs): 664 """Serialize data intended for a URL query. 665 666 :param data: The data to be serialized. 667 :param str data_type: The type to be serialized from. 668 :rtype: str 669 :raises: TypeError if serialization fails. 670 :raises: ValueError if data is None 671 """ 672 data = self._http_component_validation(data, data_type, name, **kwargs) 673 try: 674 # Treat the list aside, since we don't want to encode the div separator 675 if data_type.startswith("["): 676 internal_data_type = data_type[1:-1] 677 data = [ 678 self.serialize_data(d, internal_data_type, **kwargs) if d is not None else "" 679 for d 680 in data 681 ] 682 if not kwargs.get('skip_quote', False): 683 data = [ 684 quote(str(d), safe='') 685 for d 686 in data 687 ] 688 return str(self.serialize_iter(data, internal_data_type, **kwargs)) 689 690 # Not a list, regular serialization 691 output = self.serialize_data(data, data_type, **kwargs) 692 if data_type == 'bool': 693 output = json.dumps(output) 694 if kwargs.get('skip_quote') is True: 695 output = str(output) 696 else: 697 output = quote(str(output), safe='') 698 except SerializationError: 699 raise TypeError("{} must be type {}.".format(name, data_type)) 700 else: 701 return str(output) 702 703 def header(self, name, data, data_type, **kwargs): 704 """Serialize data intended for a request header. 705 706 :param data: The data to be serialized. 707 :param str data_type: The type to be serialized from. 708 :rtype: str 709 :raises: TypeError if serialization fails. 710 :raises: ValueError if data is None 711 """ 712 data = self._http_component_validation(data, data_type, name, **kwargs) 713 try: 714 if data_type in ['[str]']: 715 data = ["" if d is None else d for d in data] 716 717 output = self.serialize_data(data, data_type, **kwargs) 718 if data_type == 'bool': 719 output = json.dumps(output) 720 except SerializationError: 721 raise TypeError("{} must be type {}.".format(name, data_type)) 722 else: 723 return str(output) 724 725 @classmethod 726 def validate(cls, data, name, **kwargs): 727 """Validate that a piece of data meets certain conditions""" 728 required = kwargs.get('required', False) 729 if required and data is None: 730 raise ValidationError("required", name, True) 731 elif data is None: 732 return 733 elif kwargs.get('readonly'): 734 return 735 736 try: 737 for key, value in kwargs.items(): 738 validator = cls.validation.get(key, lambda x, y: False) 739 if validator(data, value): 740 raise ValidationError(key, name, value) 741 except TypeError: 742 raise ValidationError("unknown", name, "unknown") 743 else: 744 return data 745 746 def serialize_data(self, data, data_type, **kwargs): 747 """Serialize generic data according to supplied data type. 748 749 :param data: The data to be serialized. 750 :param str data_type: The type to be serialized from. 751 :param bool required: Whether it's essential that the data not be 752 empty or None 753 :raises: AttributeError if required data is None. 754 :raises: ValueError if data is None 755 :raises: SerializationError if serialization fails. 756 """ 757 if data is None: 758 raise ValueError("No value for given attribute") 759 760 try: 761 if data_type in self.basic_types.values(): 762 return self.serialize_basic(data, data_type, **kwargs) 763 764 elif data_type in self.serialize_type: 765 return self.serialize_type[data_type](data, **kwargs) 766 767 # If dependencies is empty, try with current data class 768 # It has to be a subclass of Enum anyway 769 enum_type = self.dependencies.get(data_type, data.__class__) 770 if issubclass(enum_type, Enum): 771 return Serializer.serialize_enum(data, enum_obj=enum_type) 772 773 iter_type = data_type[0] + data_type[-1] 774 if iter_type in self.serialize_type: 775 return self.serialize_type[iter_type]( 776 data, data_type[1:-1], **kwargs) 777 778 except (ValueError, TypeError) as err: 779 msg = "Unable to serialize value: {!r} as type: {!r}." 780 raise_with_traceback( 781 SerializationError, msg.format(data, data_type), err) 782 else: 783 return self._serialize(data, **kwargs) 784 785 @classmethod 786 def _get_custom_serializers(cls, data_type, **kwargs): 787 custom_serializer = kwargs.get("basic_types_serializers", {}).get(data_type) 788 if custom_serializer: 789 return custom_serializer 790 if kwargs.get("is_xml", False): 791 return cls._xml_basic_types_serializers.get(data_type) 792 793 @classmethod 794 def serialize_basic(cls, data, data_type, **kwargs): 795 """Serialize basic builting data type. 796 Serializes objects to str, int, float or bool. 797 798 Possible kwargs: 799 - basic_types_serializers dict[str, callable] : If set, use the callable as serializer 800 - is_xml bool : If set, use xml_basic_types_serializers 801 802 :param data: Object to be serialized. 803 :param str data_type: Type of object in the iterable. 804 """ 805 custom_serializer = cls._get_custom_serializers(data_type, **kwargs) 806 if custom_serializer: 807 return custom_serializer(data) 808 if data_type == 'str': 809 return cls.serialize_unicode(data) 810 return eval(data_type)(data) 811 812 @classmethod 813 def serialize_unicode(cls, data): 814 """Special handling for serializing unicode strings in Py2. 815 Encode to UTF-8 if unicode, otherwise handle as a str. 816 817 :param data: Object to be serialized. 818 :rtype: str 819 """ 820 try: # If I received an enum, return its value 821 return data.value 822 except AttributeError: 823 pass 824 825 try: 826 if isinstance(data, unicode): 827 # Don't change it, JSON and XML ElementTree are totally able 828 # to serialize correctly u'' strings 829 return data 830 except NameError: 831 return str(data) 832 else: 833 return str(data) 834 835 def serialize_iter(self, data, iter_type, div=None, **kwargs): 836 """Serialize iterable. 837 838 Supported kwargs: 839 - serialization_ctxt dict : The current entry of _attribute_map, or same format. 840 serialization_ctxt['type'] should be same as data_type. 841 - is_xml bool : If set, serialize as XML 842 843 :param list attr: Object to be serialized. 844 :param str iter_type: Type of object in the iterable. 845 :param bool required: Whether the objects in the iterable must 846 not be None or empty. 847 :param str div: If set, this str will be used to combine the elements 848 in the iterable into a combined string. Default is 'None'. 849 :rtype: list, str 850 """ 851 if isinstance(data, str): 852 raise SerializationError("Refuse str type as a valid iter type.") 853 854 serialization_ctxt = kwargs.get("serialization_ctxt", {}) 855 is_xml = kwargs.get("is_xml", False) 856 857 serialized = [] 858 for d in data: 859 try: 860 serialized.append(self.serialize_data(d, iter_type, **kwargs)) 861 except ValueError: 862 serialized.append(None) 863 864 if div: 865 serialized = ['' if s is None else str(s) for s in serialized] 866 serialized = div.join(serialized) 867 868 if 'xml' in serialization_ctxt or is_xml: 869 # XML serialization is more complicated 870 xml_desc = serialization_ctxt.get('xml', {}) 871 xml_name = xml_desc.get('name') 872 if not xml_name: 873 xml_name = serialization_ctxt['key'] 874 875 # Create a wrap node if necessary (use the fact that Element and list have "append") 876 is_wrapped = xml_desc.get("wrapped", False) 877 node_name = xml_desc.get("itemsName", xml_name) 878 if is_wrapped: 879 final_result = _create_xml_node( 880 xml_name, 881 xml_desc.get('prefix', None), 882 xml_desc.get('ns', None) 883 ) 884 else: 885 final_result = [] 886 # All list elements to "local_node" 887 for el in serialized: 888 if isinstance(el, ET.Element): 889 el_node = el 890 else: 891 el_node = _create_xml_node( 892 node_name, 893 xml_desc.get('prefix', None), 894 xml_desc.get('ns', None) 895 ) 896 if el is not None: # Otherwise it writes "None" :-p 897 el_node.text = str(el) 898 final_result.append(el_node) 899 return final_result 900 return serialized 901 902 def serialize_dict(self, attr, dict_type, **kwargs): 903 """Serialize a dictionary of objects. 904 905 :param dict attr: Object to be serialized. 906 :param str dict_type: Type of object in the dictionary. 907 :param bool required: Whether the objects in the dictionary must 908 not be None or empty. 909 :rtype: dict 910 """ 911 serialization_ctxt = kwargs.get("serialization_ctxt", {}) 912 serialized = {} 913 for key, value in attr.items(): 914 try: 915 serialized[self.serialize_unicode(key)] = self.serialize_data( 916 value, dict_type, **kwargs) 917 except ValueError: 918 serialized[self.serialize_unicode(key)] = None 919 920 if 'xml' in serialization_ctxt: 921 # XML serialization is more complicated 922 xml_desc = serialization_ctxt['xml'] 923 xml_name = xml_desc['name'] 924 925 final_result = _create_xml_node( 926 xml_name, 927 xml_desc.get('prefix', None), 928 xml_desc.get('ns', None) 929 ) 930 for key, value in serialized.items(): 931 ET.SubElement(final_result, key).text = value 932 return final_result 933 934 return serialized 935 936 def serialize_object(self, attr, **kwargs): 937 """Serialize a generic object. 938 This will be handled as a dictionary. If object passed in is not 939 a basic type (str, int, float, dict, list) it will simply be 940 cast to str. 941 942 :param dict attr: Object to be serialized. 943 :rtype: dict or str 944 """ 945 if attr is None: 946 return None 947 if isinstance(attr, ET.Element): 948 return attr 949 obj_type = type(attr) 950 if obj_type in self.basic_types: 951 return self.serialize_basic(attr, self.basic_types[obj_type], **kwargs) 952 if obj_type is _long_type: 953 return self.serialize_long(attr) 954 if obj_type is unicode_str: 955 return self.serialize_unicode(attr) 956 if obj_type is datetime.datetime: 957 return self.serialize_iso(attr) 958 if obj_type is datetime.date: 959 return self.serialize_date(attr) 960 if obj_type is datetime.time: 961 return self.serialize_time(attr) 962 if obj_type is datetime.timedelta: 963 return self.serialize_duration(attr) 964 if obj_type is decimal.Decimal: 965 return self.serialize_decimal(attr) 966 967 # If it's a model or I know this dependency, serialize as a Model 968 elif obj_type in self.dependencies.values() or isinstance(attr, Model): 969 return self._serialize(attr) 970 971 if obj_type == dict: 972 serialized = {} 973 for key, value in attr.items(): 974 try: 975 serialized[self.serialize_unicode(key)] = self.serialize_object( 976 value, **kwargs) 977 except ValueError: 978 serialized[self.serialize_unicode(key)] = None 979 return serialized 980 981 if obj_type == list: 982 serialized = [] 983 for obj in attr: 984 try: 985 serialized.append(self.serialize_object( 986 obj, **kwargs)) 987 except ValueError: 988 pass 989 return serialized 990 return str(attr) 991 992 @staticmethod 993 def serialize_enum(attr, enum_obj=None): 994 try: 995 result = attr.value 996 except AttributeError: 997 result = attr 998 try: 999 enum_obj(result) 1000 return result 1001 except ValueError: 1002 for enum_value in enum_obj: 1003 if enum_value.value.lower() == str(attr).lower(): 1004 return enum_value.value 1005 error = "{!r} is not valid value for enum {!r}" 1006 raise SerializationError(error.format(attr, enum_obj)) 1007 1008 @staticmethod 1009 def serialize_bytearray(attr, **kwargs): 1010 """Serialize bytearray into base-64 string. 1011 1012 :param attr: Object to be serialized. 1013 :rtype: str 1014 """ 1015 return b64encode(attr).decode() 1016 1017 @staticmethod 1018 def serialize_base64(attr, **kwargs): 1019 """Serialize str into base-64 string. 1020 1021 :param attr: Object to be serialized. 1022 :rtype: str 1023 """ 1024 encoded = b64encode(attr).decode('ascii') 1025 return encoded.strip('=').replace('+', '-').replace('/', '_') 1026 1027 @staticmethod 1028 def serialize_decimal(attr, **kwargs): 1029 """Serialize Decimal object to float. 1030 1031 :param attr: Object to be serialized. 1032 :rtype: float 1033 """ 1034 return float(attr) 1035 1036 @staticmethod 1037 def serialize_long(attr, **kwargs): 1038 """Serialize long (Py2) or int (Py3). 1039 1040 :param attr: Object to be serialized. 1041 :rtype: int/long 1042 """ 1043 return _long_type(attr) 1044 1045 @staticmethod 1046 def serialize_date(attr, **kwargs): 1047 """Serialize Date object into ISO-8601 formatted string. 1048 1049 :param Date attr: Object to be serialized. 1050 :rtype: str 1051 """ 1052 if isinstance(attr, str): 1053 attr = isodate.parse_date(attr) 1054 t = "{:04}-{:02}-{:02}".format(attr.year, attr.month, attr.day) 1055 return t 1056 1057 @staticmethod 1058 def serialize_time(attr, **kwargs): 1059 """Serialize Time object into ISO-8601 formatted string. 1060 1061 :param datetime.time attr: Object to be serialized. 1062 :rtype: str 1063 """ 1064 if isinstance(attr, str): 1065 attr = isodate.parse_time(attr) 1066 t = "{:02}:{:02}:{:02}".format(attr.hour, attr.minute, attr.second) 1067 if attr.microsecond: 1068 t += ".{:02}".format(attr.microsecond) 1069 return t 1070 1071 @staticmethod 1072 def serialize_duration(attr, **kwargs): 1073 """Serialize TimeDelta object into ISO-8601 formatted string. 1074 1075 :param TimeDelta attr: Object to be serialized. 1076 :rtype: str 1077 """ 1078 if isinstance(attr, str): 1079 attr = isodate.parse_duration(attr) 1080 return isodate.duration_isoformat(attr) 1081 1082 @staticmethod 1083 def serialize_rfc(attr, **kwargs): 1084 """Serialize Datetime object into RFC-1123 formatted string. 1085 1086 :param Datetime attr: Object to be serialized. 1087 :rtype: str 1088 :raises: TypeError if format invalid. 1089 """ 1090 try: 1091 if not attr.tzinfo: 1092 _LOGGER.warning( 1093 "Datetime with no tzinfo will be considered UTC.") 1094 utc = attr.utctimetuple() 1095 except AttributeError: 1096 raise TypeError("RFC1123 object must be valid Datetime object.") 1097 1098 return "{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT".format( 1099 Serializer.days[utc.tm_wday], utc.tm_mday, 1100 Serializer.months[utc.tm_mon], utc.tm_year, 1101 utc.tm_hour, utc.tm_min, utc.tm_sec) 1102 1103 @staticmethod 1104 def serialize_iso(attr, **kwargs): 1105 """Serialize Datetime object into ISO-8601 formatted string. 1106 1107 :param Datetime attr: Object to be serialized. 1108 :rtype: str 1109 :raises: SerializationError if format invalid. 1110 """ 1111 if isinstance(attr, str): 1112 attr = isodate.parse_datetime(attr) 1113 try: 1114 if not attr.tzinfo: 1115 _LOGGER.warning( 1116 "Datetime with no tzinfo will be considered UTC.") 1117 utc = attr.utctimetuple() 1118 if utc.tm_year > 9999 or utc.tm_year < 1: 1119 raise OverflowError("Hit max or min date") 1120 1121 microseconds = str(attr.microsecond).rjust(6,'0').rstrip('0').ljust(3, '0') 1122 if microseconds: 1123 microseconds = '.'+microseconds 1124 date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( 1125 utc.tm_year, utc.tm_mon, utc.tm_mday, 1126 utc.tm_hour, utc.tm_min, utc.tm_sec) 1127 return date + microseconds + 'Z' 1128 except (ValueError, OverflowError) as err: 1129 msg = "Unable to serialize datetime object." 1130 raise_with_traceback(SerializationError, msg, err) 1131 except AttributeError as err: 1132 msg = "ISO-8601 object must be valid Datetime object." 1133 raise_with_traceback(TypeError, msg, err) 1134 1135 @staticmethod 1136 def serialize_unix(attr, **kwargs): 1137 """Serialize Datetime object into IntTime format. 1138 This is represented as seconds. 1139 1140 :param Datetime attr: Object to be serialized. 1141 :rtype: int 1142 :raises: SerializationError if format invalid 1143 """ 1144 if isinstance(attr, int): 1145 return attr 1146 try: 1147 if not attr.tzinfo: 1148 _LOGGER.warning( 1149 "Datetime with no tzinfo will be considered UTC.") 1150 return int(calendar.timegm(attr.utctimetuple())) 1151 except AttributeError: 1152 raise TypeError("Unix time object must be valid Datetime object.") 1153 1154def rest_key_extractor(attr, attr_desc, data): 1155 key = attr_desc['key'] 1156 working_data = data 1157 1158 while '.' in key: 1159 dict_keys = _FLATTEN.split(key) 1160 if len(dict_keys) == 1: 1161 key = _decode_attribute_map_key(dict_keys[0]) 1162 break 1163 working_key = _decode_attribute_map_key(dict_keys[0]) 1164 working_data = working_data.get(working_key, data) 1165 if working_data is None: 1166 # If at any point while following flatten JSON path see None, it means 1167 # that all properties under are None as well 1168 # https://github.com/Azure/msrest-for-python/issues/197 1169 return None 1170 key = '.'.join(dict_keys[1:]) 1171 1172 return working_data.get(key) 1173 1174def rest_key_case_insensitive_extractor(attr, attr_desc, data): 1175 key = attr_desc['key'] 1176 working_data = data 1177 1178 while '.' in key: 1179 dict_keys = _FLATTEN.split(key) 1180 if len(dict_keys) == 1: 1181 key = _decode_attribute_map_key(dict_keys[0]) 1182 break 1183 working_key = _decode_attribute_map_key(dict_keys[0]) 1184 working_data = attribute_key_case_insensitive_extractor(working_key, None, working_data) 1185 if working_data is None: 1186 # If at any point while following flatten JSON path see None, it means 1187 # that all properties under are None as well 1188 # https://github.com/Azure/msrest-for-python/issues/197 1189 return None 1190 key = '.'.join(dict_keys[1:]) 1191 1192 if working_data: 1193 return attribute_key_case_insensitive_extractor(key, None, working_data) 1194 1195def last_rest_key_extractor(attr, attr_desc, data): 1196 """Extract the attribute in "data" based on the last part of the JSON path key. 1197 """ 1198 key = attr_desc['key'] 1199 dict_keys = _FLATTEN.split(key) 1200 return attribute_key_extractor(dict_keys[-1], None, data) 1201 1202def last_rest_key_case_insensitive_extractor(attr, attr_desc, data): 1203 """Extract the attribute in "data" based on the last part of the JSON path key. 1204 1205 This is the case insensitive version of "last_rest_key_extractor" 1206 """ 1207 key = attr_desc['key'] 1208 dict_keys = _FLATTEN.split(key) 1209 return attribute_key_case_insensitive_extractor(dict_keys[-1], None, data) 1210 1211def attribute_key_extractor(attr, _, data): 1212 return data.get(attr) 1213 1214def attribute_key_case_insensitive_extractor(attr, _, data): 1215 found_key = None 1216 lower_attr = attr.lower() 1217 for key in data: 1218 if lower_attr == key.lower(): 1219 found_key = key 1220 break 1221 1222 return data.get(found_key) 1223 1224def _extract_name_from_internal_type(internal_type): 1225 """Given an internal type XML description, extract correct XML name with namespace. 1226 1227 :param dict internal_type: An model type 1228 :rtype: tuple 1229 :returns: A tuple XML name + namespace dict 1230 """ 1231 internal_type_xml_map = getattr(internal_type, "_xml_map", {}) 1232 xml_name = internal_type_xml_map.get('name', internal_type.__name__) 1233 xml_ns = internal_type_xml_map.get("ns", None) 1234 if xml_ns: 1235 xml_name = "{{{}}}{}".format(xml_ns, xml_name) 1236 return xml_name 1237 1238 1239def xml_key_extractor(attr, attr_desc, data): 1240 if isinstance(data, dict): 1241 return None 1242 1243 # Test if this model is XML ready first 1244 if not isinstance(data, ET.Element): 1245 return None 1246 1247 xml_desc = attr_desc.get('xml', {}) 1248 xml_name = xml_desc.get('name', attr_desc['key']) 1249 1250 # Look for a children 1251 is_iter_type = attr_desc['type'].startswith("[") 1252 is_wrapped = xml_desc.get("wrapped", False) 1253 internal_type = attr_desc.get("internalType", None) 1254 internal_type_xml_map = getattr(internal_type, "_xml_map", {}) 1255 1256 # Integrate namespace if necessary 1257 xml_ns = xml_desc.get('ns', internal_type_xml_map.get("ns", None)) 1258 if xml_ns: 1259 xml_name = "{{{}}}{}".format(xml_ns, xml_name) 1260 1261 # If it's an attribute, that's simple 1262 if xml_desc.get("attr", False): 1263 return data.get(xml_name) 1264 1265 # If it's x-ms-text, that's simple too 1266 if xml_desc.get("text", False): 1267 return data.text 1268 1269 # Scenario where I take the local name: 1270 # - Wrapped node 1271 # - Internal type is an enum (considered basic types) 1272 # - Internal type has no XML/Name node 1273 if is_wrapped or (internal_type and (issubclass(internal_type, Enum) or 'name' not in internal_type_xml_map)): 1274 children = data.findall(xml_name) 1275 # If internal type has a local name and it's not a list, I use that name 1276 elif not is_iter_type and internal_type and 'name' in internal_type_xml_map: 1277 xml_name = _extract_name_from_internal_type(internal_type) 1278 children = data.findall(xml_name) 1279 # That's an array 1280 else: 1281 if internal_type: # Complex type, ignore itemsName and use the complex type name 1282 items_name = _extract_name_from_internal_type(internal_type) 1283 else: 1284 items_name = xml_desc.get("itemsName", xml_name) 1285 children = data.findall(items_name) 1286 1287 if len(children) == 0: 1288 if is_iter_type: 1289 if is_wrapped: 1290 return None # is_wrapped no node, we want None 1291 else: 1292 return [] # not wrapped, assume empty list 1293 return None # Assume it's not there, maybe an optional node. 1294 1295 # If is_iter_type and not wrapped, return all found children 1296 if is_iter_type: 1297 if not is_wrapped: 1298 return children 1299 else: # Iter and wrapped, should have found one node only (the wrap one) 1300 if len(children) != 1: 1301 raise DeserializationError( 1302 "Tried to deserialize an array not wrapped, and found several nodes '{}'. Maybe you should declare this array as wrapped?".format( 1303 xml_name 1304 )) 1305 return list(children[0]) # Might be empty list and that's ok. 1306 1307 # Here it's not a itertype, we should have found one element only or empty 1308 if len(children) > 1: 1309 raise DeserializationError("Find several XML '{}' where it was not expected".format(xml_name)) 1310 return children[0] 1311 1312class Deserializer(object): 1313 """Response object model deserializer. 1314 1315 :param dict classes: Class type dictionary for deserializing complex types. 1316 :ivar list key_extractors: Ordered list of extractors to be used by this deserializer. 1317 """ 1318 1319 basic_types = {str: 'str', int: 'int', bool: 'bool', float: 'float'} 1320 1321 valid_date = re.compile( 1322 r'\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}' 1323 r'\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?') 1324 1325 def __init__(self, classes=None): 1326 self.deserialize_type = { 1327 'iso-8601': Deserializer.deserialize_iso, 1328 'rfc-1123': Deserializer.deserialize_rfc, 1329 'unix-time': Deserializer.deserialize_unix, 1330 'duration': Deserializer.deserialize_duration, 1331 'date': Deserializer.deserialize_date, 1332 'time': Deserializer.deserialize_time, 1333 'decimal': Deserializer.deserialize_decimal, 1334 'long': Deserializer.deserialize_long, 1335 'bytearray': Deserializer.deserialize_bytearray, 1336 'base64': Deserializer.deserialize_base64, 1337 'object': self.deserialize_object, 1338 '[]': self.deserialize_iter, 1339 '{}': self.deserialize_dict 1340 } 1341 self.deserialize_expected_types = { 1342 'duration': (isodate.Duration, datetime.timedelta), 1343 'iso-8601': (datetime.datetime) 1344 } 1345 self.dependencies = dict(classes) if classes else {} 1346 self.key_extractors = [ 1347 rest_key_extractor, 1348 xml_key_extractor 1349 ] 1350 # Additional properties only works if the "rest_key_extractor" is used to 1351 # extract the keys. Making it to work whatever the key extractor is too much 1352 # complicated, with no real scenario for now. 1353 # So adding a flag to disable additional properties detection. This flag should be 1354 # used if your expect the deserialization to NOT come from a JSON REST syntax. 1355 # Otherwise, result are unexpected 1356 self.additional_properties_detection = True 1357 1358 def __call__(self, target_obj, response_data, content_type=None): 1359 """Call the deserializer to process a REST response. 1360 1361 :param str target_obj: Target data type to deserialize to. 1362 :param requests.Response response_data: REST response object. 1363 :param str content_type: Swagger "produces" if available. 1364 :raises: DeserializationError if deserialization fails. 1365 :return: Deserialized object. 1366 """ 1367 data = self._unpack_content(response_data, content_type) 1368 return self._deserialize(target_obj, data) 1369 1370 def _deserialize(self, target_obj, data): 1371 """Call the deserializer on a model. 1372 1373 Data needs to be already deserialized as JSON or XML ElementTree 1374 1375 :param str target_obj: Target data type to deserialize to. 1376 :param object data: Object to deserialize. 1377 :raises: DeserializationError if deserialization fails. 1378 :return: Deserialized object. 1379 """ 1380 # This is already a model, go recursive just in case 1381 if hasattr(data, "_attribute_map"): 1382 constants = [name for name, config in getattr(data, '_validation', {}).items() 1383 if config.get('constant')] 1384 try: 1385 for attr, mapconfig in data._attribute_map.items(): 1386 if attr in constants: 1387 continue 1388 value = getattr(data, attr) 1389 if value is None: 1390 continue 1391 local_type = mapconfig['type'] 1392 internal_data_type = local_type.strip('[]{}') 1393 if internal_data_type not in self.dependencies or isinstance(internal_data_type, Enum): 1394 continue 1395 setattr( 1396 data, 1397 attr, 1398 self._deserialize(local_type, value) 1399 ) 1400 return data 1401 except AttributeError: 1402 return 1403 1404 response, class_name = self._classify_target(target_obj, data) 1405 1406 if isinstance(response, basestring): 1407 return self.deserialize_data(data, response) 1408 elif isinstance(response, type) and issubclass(response, Enum): 1409 return self.deserialize_enum(data, response) 1410 1411 if data is None: 1412 return data 1413 try: 1414 attributes = response._attribute_map 1415 d_attrs = {} 1416 for attr, attr_desc in attributes.items(): 1417 # Check empty string. If it's not empty, someone has a real "additionalProperties"... 1418 if attr == "additional_properties" and attr_desc["key"] == '': 1419 continue 1420 raw_value = None 1421 # Enhance attr_desc with some dynamic data 1422 attr_desc = attr_desc.copy() # Do a copy, do not change the real one 1423 internal_data_type = attr_desc["type"].strip('[]{}') 1424 if internal_data_type in self.dependencies: 1425 attr_desc["internalType"] = self.dependencies[internal_data_type] 1426 1427 for key_extractor in self.key_extractors: 1428 found_value = key_extractor(attr, attr_desc, data) 1429 if found_value is not None: 1430 if raw_value is not None and raw_value != found_value: 1431 msg = ("Ignoring extracted value '%s' from %s for key '%s'" 1432 " (duplicate extraction, follow extractors order)" ) 1433 _LOGGER.warning( 1434 msg, 1435 found_value, 1436 key_extractor, 1437 attr 1438 ) 1439 continue 1440 raw_value = found_value 1441 1442 value = self.deserialize_data(raw_value, attr_desc['type']) 1443 d_attrs[attr] = value 1444 except (AttributeError, TypeError, KeyError) as err: 1445 msg = "Unable to deserialize to object: " + class_name 1446 raise_with_traceback(DeserializationError, msg, err) 1447 else: 1448 additional_properties = self._build_additional_properties(attributes, data) 1449 return self._instantiate_model(response, d_attrs, additional_properties) 1450 1451 def _build_additional_properties(self, attribute_map, data): 1452 if not self.additional_properties_detection: 1453 return None 1454 if "additional_properties" in attribute_map and attribute_map.get("additional_properties", {}).get("key") != '': 1455 # Check empty string. If it's not empty, someone has a real "additionalProperties" 1456 return None 1457 if isinstance(data, ET.Element): 1458 data = {el.tag: el.text for el in data} 1459 1460 known_keys = {_decode_attribute_map_key(_FLATTEN.split(desc['key'])[0]) 1461 for desc in attribute_map.values() if desc['key'] != ''} 1462 present_keys = set(data.keys()) 1463 missing_keys = present_keys - known_keys 1464 return {key: data[key] for key in missing_keys} 1465 1466 def _classify_target(self, target, data): 1467 """Check to see whether the deserialization target object can 1468 be classified into a subclass. 1469 Once classification has been determined, initialize object. 1470 1471 :param str target: The target object type to deserialize to. 1472 :param str/dict data: The response data to deseralize. 1473 """ 1474 if target is None: 1475 return None, None 1476 1477 if isinstance(target, basestring): 1478 try: 1479 target = self.dependencies[target] 1480 except KeyError: 1481 return target, target 1482 1483 try: 1484 target = target._classify(data, self.dependencies) 1485 except AttributeError: 1486 pass # Target is not a Model, no classify 1487 return target, target.__class__.__name__ 1488 1489 def failsafe_deserialize(self, target_obj, data, content_type=None): 1490 """Ignores any errors encountered in deserialization, 1491 and falls back to not deserializing the object. Recommended 1492 for use in error deserialization, as we want to return the 1493 HttpResponseError to users, and not have them deal with 1494 a deserialization error. 1495 1496 :param str target_obj: The target object type to deserialize to. 1497 :param str/dict data: The response data to deseralize. 1498 :param str content_type: Swagger "produces" if available. 1499 """ 1500 try: 1501 return self(target_obj, data, content_type=content_type) 1502 except: 1503 _LOGGER.warning( 1504 "Ran into a deserialization error. Ignoring since this is failsafe deserialization", 1505 exc_info=True 1506 ) 1507 return None 1508 1509 @staticmethod 1510 def _unpack_content(raw_data, content_type=None): 1511 """Extract the correct structure for deserialization. 1512 1513 If raw_data is a PipelineResponse, try to extract the result of RawDeserializer. 1514 if we can't, raise. Your Pipeline should have a RawDeserializer. 1515 1516 If not a pipeline response and raw_data is bytes or string, use content-type 1517 to decode it. If no content-type, try JSON. 1518 1519 If raw_data is something else, bypass all logic and return it directly. 1520 1521 :param raw_data: Data to be processed. 1522 :param content_type: How to parse if raw_data is a string/bytes. 1523 :raises JSONDecodeError: If JSON is requested and parsing is impossible. 1524 :raises UnicodeDecodeError: If bytes is not UTF8 1525 """ 1526 # This avoids a circular dependency. We might want to consider RawDesializer is more generic 1527 # than the pipeline concept, and put it in a toolbox, used both here and in pipeline. TBD. 1528 from .pipeline.universal import RawDeserializer 1529 1530 # Assume this is enough to detect a Pipeline Response without importing it 1531 context = getattr(raw_data, "context", {}) 1532 if context: 1533 if RawDeserializer.CONTEXT_NAME in context: 1534 return context[RawDeserializer.CONTEXT_NAME] 1535 raise ValueError("This pipeline didn't have the RawDeserializer policy; can't deserialize") 1536 1537 #Assume this is enough to recognize universal_http.ClientResponse without importing it 1538 if hasattr(raw_data, "body"): 1539 return RawDeserializer.deserialize_from_http_generics( 1540 raw_data.text(), 1541 raw_data.headers 1542 ) 1543 1544 # Assume this enough to recognize requests.Response without importing it. 1545 if hasattr(raw_data, '_content_consumed'): 1546 return RawDeserializer.deserialize_from_http_generics( 1547 raw_data.text, 1548 raw_data.headers 1549 ) 1550 1551 if isinstance(raw_data, (basestring, bytes)) or hasattr(raw_data, 'read'): 1552 return RawDeserializer.deserialize_from_text(raw_data, content_type) 1553 return raw_data 1554 1555 def _instantiate_model(self, response, attrs, additional_properties=None): 1556 """Instantiate a response model passing in deserialized args. 1557 1558 :param response: The response model class. 1559 :param d_attrs: The deserialized response attributes. 1560 """ 1561 if callable(response): 1562 subtype = getattr(response, '_subtype_map', {}) 1563 try: 1564 readonly = [k for k, v in response._validation.items() 1565 if v.get('readonly')] 1566 const = [k for k, v in response._validation.items() 1567 if v.get('constant')] 1568 kwargs = {k: v for k, v in attrs.items() 1569 if k not in subtype and k not in readonly + const} 1570 response_obj = response(**kwargs) 1571 for attr in readonly: 1572 setattr(response_obj, attr, attrs.get(attr)) 1573 if additional_properties: 1574 response_obj.additional_properties = additional_properties 1575 return response_obj 1576 except TypeError as err: 1577 msg = "Unable to deserialize {} into model {}. ".format( 1578 kwargs, response) 1579 raise DeserializationError(msg + str(err)) 1580 else: 1581 try: 1582 for attr, value in attrs.items(): 1583 setattr(response, attr, value) 1584 return response 1585 except Exception as exp: 1586 msg = "Unable to populate response model. " 1587 msg += "Type: {}, Error: {}".format(type(response), exp) 1588 raise DeserializationError(msg) 1589 1590 def deserialize_data(self, data, data_type): 1591 """Process data for deserialization according to data type. 1592 1593 :param str data: The response string to be deserialized. 1594 :param str data_type: The type to deserialize to. 1595 :raises: DeserializationError if deserialization fails. 1596 :return: Deserialized object. 1597 """ 1598 if data is None: 1599 return data 1600 1601 try: 1602 if not data_type: 1603 return data 1604 if data_type in self.basic_types.values(): 1605 return self.deserialize_basic(data, data_type) 1606 if data_type in self.deserialize_type: 1607 if isinstance(data, self.deserialize_expected_types.get(data_type, tuple())): 1608 return data 1609 1610 is_a_text_parsing_type = lambda x: x not in ["object", "[]", r"{}"] 1611 if isinstance(data, ET.Element) and is_a_text_parsing_type(data_type) and not data.text: 1612 return None 1613 data_val = self.deserialize_type[data_type](data) 1614 return data_val 1615 1616 iter_type = data_type[0] + data_type[-1] 1617 if iter_type in self.deserialize_type: 1618 return self.deserialize_type[iter_type](data, data_type[1:-1]) 1619 1620 obj_type = self.dependencies[data_type] 1621 if issubclass(obj_type, Enum): 1622 if isinstance(data, ET.Element): 1623 data = data.text 1624 return self.deserialize_enum(data, obj_type) 1625 1626 except (ValueError, TypeError, AttributeError) as err: 1627 msg = "Unable to deserialize response data." 1628 msg += " Data: {}, {}".format(data, data_type) 1629 raise_with_traceback(DeserializationError, msg, err) 1630 else: 1631 return self._deserialize(obj_type, data) 1632 1633 def deserialize_iter(self, attr, iter_type): 1634 """Deserialize an iterable. 1635 1636 :param list attr: Iterable to be deserialized. 1637 :param str iter_type: The type of object in the iterable. 1638 :rtype: list 1639 """ 1640 if attr is None: 1641 return None 1642 if isinstance(attr, ET.Element): # If I receive an element here, get the children 1643 attr = list(attr) 1644 if not isinstance(attr, (list, set)): 1645 raise DeserializationError("Cannot deserialize as [{}] an object of type {}".format( 1646 iter_type, 1647 type(attr) 1648 )) 1649 return [self.deserialize_data(a, iter_type) for a in attr] 1650 1651 def deserialize_dict(self, attr, dict_type): 1652 """Deserialize a dictionary. 1653 1654 :param dict/list attr: Dictionary to be deserialized. Also accepts 1655 a list of key, value pairs. 1656 :param str dict_type: The object type of the items in the dictionary. 1657 :rtype: dict 1658 """ 1659 if isinstance(attr, list): 1660 return {x['key']: self.deserialize_data(x['value'], dict_type) for x in attr} 1661 1662 if isinstance(attr, ET.Element): 1663 # Transform <Key>value</Key> into {"Key": "value"} 1664 attr = {el.tag: el.text for el in attr} 1665 return {k: self.deserialize_data(v, dict_type) for k, v in attr.items()} 1666 1667 def deserialize_object(self, attr, **kwargs): 1668 """Deserialize a generic object. 1669 This will be handled as a dictionary. 1670 1671 :param dict attr: Dictionary to be deserialized. 1672 :rtype: dict 1673 :raises: TypeError if non-builtin datatype encountered. 1674 """ 1675 if attr is None: 1676 return None 1677 if isinstance(attr, ET.Element): 1678 # Do no recurse on XML, just return the tree as-is 1679 return attr 1680 if isinstance(attr, basestring): 1681 return self.deserialize_basic(attr, 'str') 1682 obj_type = type(attr) 1683 if obj_type in self.basic_types: 1684 return self.deserialize_basic(attr, self.basic_types[obj_type]) 1685 if obj_type is _long_type: 1686 return self.deserialize_long(attr) 1687 1688 if obj_type == dict: 1689 deserialized = {} 1690 for key, value in attr.items(): 1691 try: 1692 deserialized[key] = self.deserialize_object( 1693 value, **kwargs) 1694 except ValueError: 1695 deserialized[key] = None 1696 return deserialized 1697 1698 if obj_type == list: 1699 deserialized = [] 1700 for obj in attr: 1701 try: 1702 deserialized.append(self.deserialize_object( 1703 obj, **kwargs)) 1704 except ValueError: 1705 pass 1706 return deserialized 1707 1708 else: 1709 error = "Cannot deserialize generic object with type: " 1710 raise TypeError(error + str(obj_type)) 1711 1712 def deserialize_basic(self, attr, data_type): 1713 """Deserialize baisc builtin data type from string. 1714 Will attempt to convert to str, int, float and bool. 1715 This function will also accept '1', '0', 'true' and 'false' as 1716 valid bool values. 1717 1718 :param str attr: response string to be deserialized. 1719 :param str data_type: deserialization data type. 1720 :rtype: str, int, float or bool 1721 :raises: TypeError if string format is not valid. 1722 """ 1723 # If we're here, data is supposed to be a basic type. 1724 # If it's still an XML node, take the text 1725 if isinstance(attr, ET.Element): 1726 attr = attr.text 1727 if not attr: 1728 if data_type == "str": 1729 # None or '', node <a/> is empty string. 1730 return '' 1731 else: 1732 # None or '', node <a/> with a strong type is None. 1733 # Don't try to model "empty bool" or "empty int" 1734 return None 1735 1736 if data_type == 'bool': 1737 if attr in [True, False, 1, 0]: 1738 return bool(attr) 1739 elif isinstance(attr, basestring): 1740 if attr.lower() in ['true', '1']: 1741 return True 1742 elif attr.lower() in ['false', '0']: 1743 return False 1744 raise TypeError("Invalid boolean value: {}".format(attr)) 1745 1746 if data_type == 'str': 1747 return self.deserialize_unicode(attr) 1748 return eval(data_type)(attr) 1749 1750 @staticmethod 1751 def deserialize_unicode(data): 1752 """Preserve unicode objects in Python 2, otherwise return data 1753 as a string. 1754 1755 :param str data: response string to be deserialized. 1756 :rtype: str or unicode 1757 """ 1758 # We might be here because we have an enum modeled as string, 1759 # and we try to deserialize a partial dict with enum inside 1760 if isinstance(data, Enum): 1761 return data 1762 1763 # Consider this is real string 1764 try: 1765 if isinstance(data, unicode): 1766 return data 1767 except NameError: 1768 return str(data) 1769 else: 1770 return str(data) 1771 1772 @staticmethod 1773 def deserialize_enum(data, enum_obj): 1774 """Deserialize string into enum object. 1775 1776 If the string is not a valid enum value it will be returned as-is 1777 and a warning will be logged. 1778 1779 :param str data: Response string to be deserialized. If this value is 1780 None or invalid it will be returned as-is. 1781 :param Enum enum_obj: Enum object to deserialize to. 1782 :rtype: Enum 1783 """ 1784 if isinstance(data, enum_obj) or data is None: 1785 return data 1786 if isinstance(data, Enum): 1787 data = data.value 1788 if isinstance(data, int): 1789 # Workaround. We might consider remove it in the future. 1790 # https://github.com/Azure/azure-rest-api-specs/issues/141 1791 try: 1792 return list(enum_obj.__members__.values())[data] 1793 except IndexError: 1794 error = "{!r} is not a valid index for enum {!r}" 1795 raise DeserializationError(error.format(data, enum_obj)) 1796 try: 1797 return enum_obj(str(data)) 1798 except ValueError: 1799 for enum_value in enum_obj: 1800 if enum_value.value.lower() == str(data).lower(): 1801 return enum_value 1802 # We don't fail anymore for unknown value, we deserialize as a string 1803 _LOGGER.warning("Deserializer is not able to find %s as valid enum in %s", data, enum_obj) 1804 return Deserializer.deserialize_unicode(data) 1805 1806 @staticmethod 1807 def deserialize_bytearray(attr): 1808 """Deserialize string into bytearray. 1809 1810 :param str attr: response string to be deserialized. 1811 :rtype: bytearray 1812 :raises: TypeError if string format invalid. 1813 """ 1814 if isinstance(attr, ET.Element): 1815 attr = attr.text 1816 return bytearray(b64decode(attr)) 1817 1818 @staticmethod 1819 def deserialize_base64(attr): 1820 """Deserialize base64 encoded string into string. 1821 1822 :param str attr: response string to be deserialized. 1823 :rtype: bytearray 1824 :raises: TypeError if string format invalid. 1825 """ 1826 if isinstance(attr, ET.Element): 1827 attr = attr.text 1828 padding = '=' * (3 - (len(attr) + 3) % 4) 1829 attr = attr + padding 1830 encoded = attr.replace('-', '+').replace('_', '/') 1831 return b64decode(encoded) 1832 1833 @staticmethod 1834 def deserialize_decimal(attr): 1835 """Deserialize string into Decimal object. 1836 1837 :param str attr: response string to be deserialized. 1838 :rtype: Decimal 1839 :raises: DeserializationError if string format invalid. 1840 """ 1841 if isinstance(attr, ET.Element): 1842 attr = attr.text 1843 try: 1844 return decimal.Decimal(attr) 1845 except decimal.DecimalException as err: 1846 msg = "Invalid decimal {}".format(attr) 1847 raise_with_traceback(DeserializationError, msg, err) 1848 1849 @staticmethod 1850 def deserialize_long(attr): 1851 """Deserialize string into long (Py2) or int (Py3). 1852 1853 :param str attr: response string to be deserialized. 1854 :rtype: long or int 1855 :raises: ValueError if string format invalid. 1856 """ 1857 if isinstance(attr, ET.Element): 1858 attr = attr.text 1859 return _long_type(attr) 1860 1861 @staticmethod 1862 def deserialize_duration(attr): 1863 """Deserialize ISO-8601 formatted string into TimeDelta object. 1864 1865 :param str attr: response string to be deserialized. 1866 :rtype: TimeDelta 1867 :raises: DeserializationError if string format invalid. 1868 """ 1869 if isinstance(attr, ET.Element): 1870 attr = attr.text 1871 try: 1872 duration = isodate.parse_duration(attr) 1873 except(ValueError, OverflowError, AttributeError) as err: 1874 msg = "Cannot deserialize duration object." 1875 raise_with_traceback(DeserializationError, msg, err) 1876 else: 1877 return duration 1878 1879 @staticmethod 1880 def deserialize_date(attr): 1881 """Deserialize ISO-8601 formatted string into Date object. 1882 1883 :param str attr: response string to be deserialized. 1884 :rtype: Date 1885 :raises: DeserializationError if string format invalid. 1886 """ 1887 if isinstance(attr, ET.Element): 1888 attr = attr.text 1889 if re.search(r"[^\W\d_]", attr, re.I + re.U): 1890 raise DeserializationError("Date must have only digits and -. Received: %s" % attr) 1891 # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. 1892 return isodate.parse_date(attr, defaultmonth=None, defaultday=None) 1893 1894 @staticmethod 1895 def deserialize_time(attr): 1896 """Deserialize ISO-8601 formatted string into time object. 1897 1898 :param str attr: response string to be deserialized. 1899 :rtype: datetime.time 1900 :raises: DeserializationError if string format invalid. 1901 """ 1902 if isinstance(attr, ET.Element): 1903 attr = attr.text 1904 if re.search(r"[^\W\d_]", attr, re.I + re.U): 1905 raise DeserializationError("Date must have only digits and -. Received: %s" % attr) 1906 return isodate.parse_time(attr) 1907 1908 @staticmethod 1909 def deserialize_rfc(attr): 1910 """Deserialize RFC-1123 formatted string into Datetime object. 1911 1912 :param str attr: response string to be deserialized. 1913 :rtype: Datetime 1914 :raises: DeserializationError if string format invalid. 1915 """ 1916 if isinstance(attr, ET.Element): 1917 attr = attr.text 1918 try: 1919 parsed_date = email.utils.parsedate_tz(attr) 1920 date_obj = datetime.datetime( 1921 *parsed_date[:6], 1922 tzinfo=_FixedOffset(datetime.timedelta(minutes=(parsed_date[9] or 0)/60)) 1923 ) 1924 if not date_obj.tzinfo: 1925 date_obj = date_obj.astimezone(tz=TZ_UTC) 1926 except ValueError as err: 1927 msg = "Cannot deserialize to rfc datetime object." 1928 raise_with_traceback(DeserializationError, msg, err) 1929 else: 1930 return date_obj 1931 1932 @staticmethod 1933 def deserialize_iso(attr): 1934 """Deserialize ISO-8601 formatted string into Datetime object. 1935 1936 :param str attr: response string to be deserialized. 1937 :rtype: Datetime 1938 :raises: DeserializationError if string format invalid. 1939 """ 1940 if isinstance(attr, ET.Element): 1941 attr = attr.text 1942 try: 1943 attr = attr.upper() 1944 match = Deserializer.valid_date.match(attr) 1945 if not match: 1946 raise ValueError("Invalid datetime string: " + attr) 1947 1948 check_decimal = attr.split('.') 1949 if len(check_decimal) > 1: 1950 decimal_str = "" 1951 for digit in check_decimal[1]: 1952 if digit.isdigit(): 1953 decimal_str += digit 1954 else: 1955 break 1956 if len(decimal_str) > 6: 1957 attr = attr.replace(decimal_str, decimal_str[0:6]) 1958 1959 date_obj = isodate.parse_datetime(attr) 1960 test_utc = date_obj.utctimetuple() 1961 if test_utc.tm_year > 9999 or test_utc.tm_year < 1: 1962 raise OverflowError("Hit max or min date") 1963 except(ValueError, OverflowError, AttributeError) as err: 1964 msg = "Cannot deserialize datetime object." 1965 raise_with_traceback(DeserializationError, msg, err) 1966 else: 1967 return date_obj 1968 1969 @staticmethod 1970 def deserialize_unix(attr): 1971 """Serialize Datetime object into IntTime format. 1972 This is represented as seconds. 1973 1974 :param int attr: Object to be serialized. 1975 :rtype: Datetime 1976 :raises: DeserializationError if format invalid 1977 """ 1978 if isinstance(attr, ET.Element): 1979 attr = int(attr.text) 1980 try: 1981 date_obj = datetime.datetime.fromtimestamp(attr, TZ_UTC) 1982 except ValueError as err: 1983 msg = "Cannot deserialize to unix datetime object." 1984 raise_with_traceback(DeserializationError, msg, err) 1985 else: 1986 return date_obj 1987