1# coding: utf-8 2# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. 3# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. 4 5from __future__ import absolute_import 6import json 7import logging 8import platform 9 10import circuitbreaker 11import pytz 12import random 13import os 14import re 15import string 16import uuid 17# This was added to address thread safety issues with datetime.strptime 18# See https://bugs.python.org/issue7980. 19import _strptime # noqa: F401 20from datetime import date, datetime 21from timeit import default_timer as timer 22from ._vendor import requests, six, urllib3 23from dateutil.parser import parse 24from dateutil import tz 25 26import functools 27from six.moves.http_client import HTTPResponse 28 29from . import constants, exceptions, regions, retry 30from .auth import signers 31from .config import get_config_value_or_default, validate_config 32from .request import Request 33from .response import Response 34from .circuit_breaker import CircuitBreakerStrategy, NoCircuitBreakerStrategy 35from circuitbreaker import CircuitBreaker, CircuitBreakerMonitor 36from .version import __version__ 37from .util import NONE_SENTINEL, Sentinel, extract_service_endpoint 38missing = Sentinel("Missing") 39APPEND_USER_AGENT_ENV_VAR_NAME = "OCI_SDK_APPEND_USER_AGENT" 40APPEND_USER_AGENT = os.environ.get(APPEND_USER_AGENT_ENV_VAR_NAME) 41USER_INFO = "Oracle-PythonSDK/{}".format(__version__) 42 43DICT_VALUE_TYPE_REGEX = re.compile('dict\(str, (.+?)\)$') # noqa: W605 44LIST_ITEM_TYPE_REGEX = re.compile('list\[(.+?)\]$') # noqa: W605 45 46# Expect header is enabled by default 47enable_expect_header = True 48expect_header_env_var = os.environ.get('OCI_PYSDK_USING_EXPECT_HEADER', True) 49if isinstance(expect_header_env_var, six.string_types) and expect_header_env_var.lower() == "false": 50 enable_expect_header = False 51 52 53def merge_type_mappings(*dictionaries): 54 merged = {} 55 for dictionary in dictionaries: 56 merged.update(dictionary) 57 return merged 58 59 60def build_user_agent(extra=""): 61 agent = "{} (python {}; {}-{}) {}".format( 62 USER_INFO, 63 platform.python_version(), 64 platform.machine(), 65 platform.system(), 66 (extra or "") 67 ) 68 agent = agent.strip() 69 if APPEND_USER_AGENT: 70 agent += " {}".format(APPEND_USER_AGENT) 71 return agent 72 73 74def utc_now(): 75 return " " + str(datetime.utcnow()) + ": " 76 77 78def is_http_log_enabled(is_enabled): 79 if is_enabled: 80 six.moves.http_client.HTTPConnection.debuglevel = 1 81 else: 82 six.moves.http_client.HTTPConnection.debuglevel = 0 83 84 85def _sanitize_headers_for_requests(headers): 86 # Requests does not accept int or float values headers 87 # Convert int, float and bool to string 88 # Bools are automatically handled with this as bool is a subclass of int 89 for header_name, header_value in six.iteritems(headers): 90 if isinstance(header_value, six.integer_types) or isinstance(header_value, float): 91 headers[header_name] = str(header_value) 92 return headers 93 94 95STREAM_RESPONSE_TYPE = 'stream' 96BYTES_RESPONSE_TYPE = 'bytes' 97 98# Default timeout value(second) 99DEFAULT_CONNECTION_TIMEOUT = 10.0 100DEFAULT_READ_TIMEOUT = 60.0 101 102# The keys here correspond to the Swagger collection format values described here: https://swagger.io/docs/specification/2-0/describing-parameters/ 103# and the values represent delimiters we'll use between values of the collection when placing those values in the query string. 104# 105# Note that the 'multi' type has no delimiter since in the query string we'll want to repeat the same query string param key but 106# with different values each time (e.g. myKey=val1&myKey=val2), whereas for the other types we will only pass in a single 107# key=value in the query string, where the "value" is the members of the collecction with a given delimiter. 108VALID_COLLECTION_FORMAT_TYPES = { 109 'multi': None, 110 'csv': ',', 111 'tsv': '\t', 112 'ssv': ' ', 113 'pipes': '|' 114} 115 116 117def _read_all_headers(fp): 118 current = None 119 while current != b'\r\n': 120 current = fp.readline() 121 122 123def _to_bytes(input_buffer): 124 bytes_buffer = [] 125 for chunk in input_buffer: 126 if isinstance(chunk, six.text_type): 127 bytes_buffer.append(chunk.encode('utf-8')) 128 else: 129 bytes_buffer.append(chunk) 130 msg = b"\r\n".join(bytes_buffer) 131 return msg 132 133 134def _is_100_continue(line): 135 parts = line.split(None, 2) 136 return ( 137 len(parts) >= 3 and parts[0].startswith(b'HTTP/') and 138 parts[1] == b'100') 139 140 141class OCIHTTPResponse(HTTPResponse): 142 143 def __init__(self, *args, **kwargs): 144 self._status_tuple = kwargs.pop('status_tuple') 145 HTTPResponse.__init__(self, *args, **kwargs) 146 147 def _read_status(self): 148 if self._status_tuple is not None: 149 status_tuple = self._status_tuple 150 self._status_tuple = None 151 return status_tuple 152 else: 153 return HTTPResponse._read_status(self) 154 155 156class OCIConnection(urllib3.connection.VerifiedHTTPSConnection): 157 """ HTTPConnection with 100 Continue support. """ 158 159 def __init__(self, *args, **kwargs): 160 super(OCIConnection, self).__init__(*args, **kwargs) 161 self._original_response_cls = self.response_class 162 self._response_received = False 163 self._using_expect_header = False 164 self.logger = logging.getLogger("{}.{}".format(__name__, id(self))) 165 166 def _send_request(self, method, url, body, headers, *args, **kwargs): 167 self._response_received = False 168 if headers.get('expect', '') == '100-continue': 169 if self.debuglevel > 0: 170 print('Using Expect header...') 171 self._using_expect_header = True 172 else: 173 if self.debuglevel > 0: 174 print('Not using Expect header...') 175 self._using_expect_header = False 176 self.response_class = self._original_response_cls 177 rval = super(OCIConnection, self)._send_request( 178 method, url, body, headers, *args, **kwargs) 179 self._expect_header_set = False 180 return rval 181 182 def _send_output(self, message_body=None, *args, **kwargs): 183 self._buffer.extend((b"", b"")) 184 msg = _to_bytes(self._buffer) 185 del self._buffer[:] 186 187 if not self._using_expect_header: 188 if isinstance(message_body, bytes): 189 msg += message_body 190 message_body = None 191 192 self.send(msg) 193 194 if self._using_expect_header: 195 if self.debuglevel > 0: 196 print('Waiting 3 seconds for 100-continue response...') 197 if urllib3.util.wait_for_read(self.sock, 3): 198 self._handle_expect_response(message_body) 199 return 200 201 if message_body is not None: 202 if self.debuglevel > 0: 203 print('Timeout waiting for 100-continue response, sending message body...') 204 self.send(message_body) 205 206 def _handle_expect_response(self, message_body): 207 fp = self.sock.makefile('rb', 0) 208 try: 209 line = fp.readline() 210 parts = line.split(None, 2) 211 if self.debuglevel > 0: 212 print(line) 213 if _is_100_continue(line): 214 if self.debuglevel > 0: 215 print('Received 100-continue response, sending message body...') 216 _read_all_headers(fp) 217 self._send_message_body(message_body) 218 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'): 219 if self.debuglevel > 0: 220 print('Received non-100-continue response, abort request...') 221 status_tuple = (parts[0].decode('ascii'), 222 int(parts[1]), parts[2].decode('ascii')) 223 response_class = functools.partial( 224 OCIHTTPResponse, status_tuple=status_tuple) 225 self.response_class = response_class 226 self._response_received = True 227 finally: 228 fp.close() 229 230 def _send_message_body(self, message_body): 231 if message_body is not None: 232 self.send(message_body) 233 234 def send(self, str): 235 if self._response_received: 236 return 237 return super(OCIConnection, self).send(str) 238 239 240class OCIConnectionPool(urllib3.HTTPSConnectionPool): 241 ConnectionCls = OCIConnection 242 """ HTTPConnectionPool with 100 Continue support. """ 243 244 245# Replace the HTTPS connection pool with OCIConnectionPool once the env var `OCI_PYSDK_USING_EXPECT_HEADER` is not set 246# to "FALSE" 247if enable_expect_header: 248 urllib3.poolmanager.pool_classes_by_scheme["https"] = OCIConnectionPool 249 250 251class BaseClient(object): 252 primitive_type_map = { 253 'int': int, 254 'float': float, 255 'str': six.u, 256 'bool': bool, 257 'date': date, 258 'datetime': datetime, 259 "object": object 260 } 261 262 def __init__(self, service, config, signer, type_mapping, **kwargs): 263 validate_config(config, signer=signer) 264 self.signer = signer 265 266 # Default to true (is a regional client) if there is nothing explicitly set. Regional 267 # clients allow us to call set_region and that'll also set the endpoint. For non-regional 268 # clients we require an endpoint 269 self.regional_client = kwargs.get('regional_client', True) 270 271 self._endpoint = None 272 self._base_path = kwargs.get('base_path') 273 self.service_endpoint_template = kwargs.get('service_endpoint_template') 274 self.endpoint_service_name = kwargs.get('endpoint_service_name') 275 276 if self.regional_client: 277 if kwargs.get('service_endpoint'): 278 self.endpoint = kwargs.get('service_endpoint') 279 else: 280 region_to_use = None 281 if 'region' in config and config['region']: 282 region_to_use = config.get('region') 283 elif hasattr(signer, 'region'): 284 region_to_use = signer.region 285 286 self.endpoint = regions.endpoint_for( 287 service, 288 service_endpoint_template=self.service_endpoint_template, 289 region=region_to_use, 290 endpoint=config.get('endpoint'), 291 endpoint_service_name=self.endpoint_service_name) 292 else: 293 if not kwargs.get('service_endpoint'): 294 raise exceptions.MissingEndpointForNonRegionalServiceClientError('An endpoint must be provided for a non-regional service client') 295 self.endpoint = kwargs.get('service_endpoint') 296 297 self.service = service 298 self.complex_type_mappings = type_mapping 299 self.type_mappings = merge_type_mappings(self.primitive_type_map, type_mapping) 300 self.session = requests.Session() 301 302 # If the user doesn't specify timeout explicitly we would use default timeout. 303 self.timeout = kwargs.get('timeout') if 'timeout' in kwargs else (DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT) 304 305 self.user_agent = build_user_agent(get_config_value_or_default(config, "additional_user_agent")) 306 307 self.logger = logging.getLogger("{}.{}".format(__name__, id(self))) 308 self.logger.addHandler(logging.NullHandler()) 309 if get_config_value_or_default(config, "log_requests"): 310 self.logger.disabled = False 311 self.logger.setLevel(logging.DEBUG) 312 is_http_log_enabled(True) 313 else: 314 self.logger.disabled = True 315 is_http_log_enabled(False) 316 317 self.skip_deserialization = kwargs.get('skip_deserialization') 318 319 # Circuit Breaker at client level 320 self.circuit_breaker_strategy = kwargs.get('circuit_breaker_strategy') 321 self.circuit_breaker_name = None 322 # Log if Circuit Breaker Strategy is not enabled or if using Default Circuit breaker Strategy 323 if self.circuit_breaker_strategy is None or isinstance(self.circuit_breaker_strategy, NoCircuitBreakerStrategy): 324 self.logger.debug('No circuit breaker strategy enabled!') 325 else: 326 # Enable Circuit breaker if a valid circuit breaker strategy is available 327 if not isinstance(self.circuit_breaker_strategy, CircuitBreakerStrategy): 328 raise TypeError('Invalid Circuit Breaker Strategy!') 329 self.circuit_breaker_name = str(uuid.uuid4()) if self.circuit_breaker_strategy.name is None else self.circuit_breaker_strategy.name 330 # Re-use Circuit breaker if sharing a Circuit Breaker Strategy. 331 circuit_breaker = CircuitBreakerMonitor.get(self.circuit_breaker_name) 332 if circuit_breaker is None: 333 circuit_breaker = CircuitBreaker( 334 failure_threshold=self.circuit_breaker_strategy.failure_threshold, 335 recovery_timeout=self.circuit_breaker_strategy.recovery_timeout, 336 expected_exception=self.circuit_breaker_strategy.expected_exception, 337 name=self.circuit_breaker_name 338 ) 339 # Equivalent to decorating the request function with Circuit Breaker 340 self.request = circuit_breaker(self.request) 341 self.logger.debug('Endpoint: {}'.format(self._endpoint)) 342 343 @property 344 def endpoint(self): 345 return self._endpoint 346 347 @endpoint.setter 348 def endpoint(self, endpoint): 349 if self._base_path == '/': 350 # If it's just the root path then use the endpoint as-is 351 self._endpoint = endpoint 352 elif self._base_path and not (endpoint.endswith(self._base_path) or endpoint.endswith('{}/'.format(self._base_path))): 353 # Account for formats like https://iaas.us-phoenix-1.oraclecloud.com/20160918 and 354 # https://iaas.us-phoenix-1.oraclecloud.com/20160918/ as they should both be fine 355 self._endpoint = '{}{}'.format(endpoint, self._base_path) 356 else: 357 self._endpoint = endpoint 358 359 def get_endpoint(self): 360 return extract_service_endpoint(self._endpoint) 361 362 def set_region(self, region): 363 if self.regional_client: 364 self.endpoint = regions.endpoint_for(self.service, service_endpoint_template=self.service_endpoint_template, region=region, endpoint_service_name=self.endpoint_service_name) 365 else: 366 raise TypeError('Setting the region is not allowed for non-regional service clients. You must instead set the endpoint') 367 368 def call_api(self, resource_path, method, 369 path_params=None, 370 query_params=None, 371 header_params=None, 372 body=None, 373 response_type=None, 374 enforce_content_headers=True): 375 """ 376 Makes the HTTP request and return the deserialized data. 377 378 :param resource_path: Path to the resource (e.g. /instance) 379 :param method: HTTP method 380 :param path_params: (optional) Path parameters in the url. 381 :param query_params: (optional) Query parameters in the url. 382 :param header_params: (optional) Request header params. 383 :param body: (optional) Request body. 384 :param response_type: (optional) Response data type. 385 :param enforce_content_headers: (optional) Whether content headers should be added for 386 PUT and POST requests when not present. Defaults to True. 387 :return: A Response object, or throw in the case of an error. 388 389 """ 390 391 if header_params: 392 # Remove expect header if user has disabled it, or if the operation is not PUT, POST or PATCH 393 if not enable_expect_header or method.lower() not in ["put", "post", "patch"]: 394 map_lowercase_header_params_keys_to_actual_keys = {k.lower(): k for k in header_params} 395 if "expect" in map_lowercase_header_params_keys_to_actual_keys: 396 header_params.pop(map_lowercase_header_params_keys_to_actual_keys.get("expect"), None) 397 398 header_params = self.sanitize_for_serialization(header_params) 399 400 header_params = header_params or {} 401 402 # All the headers have been prepared for serialization at this point 403 header_params = _sanitize_headers_for_requests(header_params) 404 405 header_params[constants.HEADER_CLIENT_INFO] = USER_INFO 406 header_params[constants.HEADER_USER_AGENT] = self.user_agent 407 408 if header_params.get(constants.HEADER_REQUEST_ID, missing) is missing: 409 header_params[constants.HEADER_REQUEST_ID] = self.build_request_id() 410 411 # This allows for testing with "fake" database resources. 412 opc_host_serial = os.environ.get('OCI_DB_OPC_HOST_SERIAL') 413 if opc_host_serial: 414 header_params['opc-host-serial'] = opc_host_serial 415 416 if path_params: 417 path_params = self.sanitize_for_serialization(path_params) 418 for k, v in path_params.items(): 419 replacement = six.moves.urllib.parse.quote(str(self.to_path_value(v))) 420 resource_path = resource_path.\ 421 replace('{' + k + '}', replacement) 422 423 if query_params: 424 query_params = self.process_query_params(query_params) 425 426 if body is not None and header_params.get('content-type') == 'application/json': 427 body = self.sanitize_for_serialization(body) 428 body = json.dumps(body) 429 430 url = self.endpoint + resource_path 431 432 request = Request( 433 method=method, 434 url=url, 435 query_params=query_params, 436 header_params=header_params, 437 body=body, 438 response_type=response_type, 439 enforce_content_headers=enforce_content_headers 440 ) 441 442 if isinstance(self.signer, signers.InstancePrincipalsSecurityTokenSigner) or \ 443 isinstance(self.signer, signers.ResourcePrincipalsFederationSigner) or \ 444 isinstance(self.signer, signers.EphemeralResourcePrincipalSigner): 445 call_attempts = 0 446 while call_attempts < 2: 447 try: 448 return self.request(request) 449 except exceptions.ServiceError as e: 450 call_attempts += 1 451 if e.status == 401 and call_attempts < 2: 452 self.signer.refresh_security_token() 453 else: 454 raise 455 else: 456 start = timer() 457 response = self.request(request) 458 end = timer() 459 self.logger.debug('time elapsed for request: {}'.format(str(end - start))) 460 return response 461 462 def generate_collection_format_param(self, param_value, collection_format_type): 463 if param_value is missing: 464 return missing 465 466 if collection_format_type not in VALID_COLLECTION_FORMAT_TYPES: 467 raise ValueError('Invalid collection format type {}. Valid types are: {}'.format(collection_format_type, list(VALID_COLLECTION_FORMAT_TYPES.keys()))) 468 469 if collection_format_type == 'multi': 470 return param_value 471 else: 472 return VALID_COLLECTION_FORMAT_TYPES[collection_format_type].join(param_value) 473 474 def process_query_params(self, query_params): 475 query_params = self.sanitize_for_serialization(query_params) 476 477 processed_query_params = {} 478 for k, v in query_params.items(): 479 # First divide our query params into ones where the param value is "simple" (not a dict or list), a list or a dict. Since we're 480 # executing after sanitize_for_serialization has been called it's dicts, lists or primitives all the way down. 481 # 482 # The params where the value is a dict are, for example, tags we need to handle differently for inclusion 483 # in the query string. 484 # 485 # The params where the value is a list are multivalued parameters in the query string. 486 # 487 # An example query_params is: 488 # 489 # { 490 # "stuff": "things", 491 # "collectionFormat": ["val1", "val2", "val3"] 492 # "dictTags": { "tag1": ["val1", "val2", "val3"], "tag2": ["val1"] }, 493 # "dictTagsExists": { "tag3": True, "tag4": True } 494 # } 495 # 496 # And we can categorize the params as: 497 # 498 # Simple: "stuff":"things" 499 # List: "collectionFormat": ["val1", "val2", "val3"] 500 # Dict: "dictTags": { "tag1": ["val1", "val2", "val3"], "tag2": ["val1"] }, "dictTagsExists": { "tag3": True, "tag4": True } 501 if isinstance(v, bool): 502 # Python capitalizes boolean values in the query parameters. 503 processed_query_params[k] = 'true' if v else 'false' 504 elif not isinstance(v, dict) and not isinstance(v, list): 505 processed_query_params[k] = self.to_path_value(v) 506 elif isinstance(v, list): 507 # The requests library supports lists to represent multivalued params natively 508 # (http://docs.python-requests.org/en/master/api/#requests.Session.params) so we just have to assign 509 # the list to the key (where the key is the query string param key) 510 processed_query_params[k] = v 511 else: 512 # If we are here then we either have: 513 # 514 # 1) a dict where the value is an array. The requests library supports lists to represent multivalued params 515 # natively (http://docs.python-requests.org/en/master/api/#requests.Session.params) so we just have to 516 # manipulate things into the right key. In the case of something like: 517 # 518 # "dictTags": { "tag1": ["val1", "val2", "val3"], "tag2": ["val1"] } 519 # 520 # What we want is to end up with: 521 # 522 # "dictTags.tag1": ["val1", "val2", "val3"], "dictTags.tag2": ["val1"] 523 # 524 # 2) a dict where the value is not an array and in this case we just explode out the content. For example if we have: 525 # 526 # "dictTagsExists": { "tag3": True, "tag4": True } 527 # 528 # What we'll end up with is: 529 # 530 # "dictTagsExists.tag3": True, "dictTagsExists.tag4": True 531 for inner_key, inner_val in v.items(): 532 processed_query_params['{}.{}'.format(k, inner_key)] = inner_val 533 534 return processed_query_params 535 536 def request(self, request): 537 self.logger.info(utc_now() + "Request: %s %s" % (str(request.method), request.url)) 538 539 initial_circuit_breaker_state = None 540 if self.circuit_breaker_name: 541 initial_circuit_breaker_state = CircuitBreakerMonitor.get(self.circuit_breaker_name).state 542 if initial_circuit_breaker_state != circuitbreaker.STATE_CLOSED: 543 self.logger.debug("Circuit Breaker State is {}!".format(initial_circuit_breaker_state)) 544 545 signer = self.signer 546 if not request.enforce_content_headers: 547 signer = signer.without_content_headers 548 549 stream = False 550 if request.response_type == STREAM_RESPONSE_TYPE: 551 stream = True 552 553 try: 554 start = timer() 555 response = self.session.request( 556 request.method, 557 request.url, 558 auth=signer, 559 params=request.query_params, 560 headers=request.header_params, 561 data=request.body, 562 stream=stream, 563 timeout=self.timeout) 564 end = timer() 565 if request.header_params[constants.HEADER_REQUEST_ID]: 566 self.logger.debug(utc_now() + 'time elapsed for request {}: {}'.format(request.header_params[constants.HEADER_REQUEST_ID], str(end - start))) 567 if response and hasattr(response, 'elapsed'): 568 self.logger.debug(utc_now() + "time elapsed in response: " + str(response.elapsed)) 569 except requests.exceptions.ConnectTimeout as e: 570 raise exceptions.ConnectTimeout(e) 571 except requests.exceptions.RequestException as e: 572 raise exceptions.RequestException(e) 573 574 response_type = request.response_type 575 self.logger.debug(utc_now() + "Response status: %s" % str(response.status_code)) 576 577 # Raise Service Error or Transient Service Error 578 if not 200 <= response.status_code <= 299: 579 service_code, message = self.get_deserialized_service_code_and_message(response) 580 if isinstance(self.circuit_breaker_strategy, CircuitBreakerStrategy) and self.circuit_breaker_strategy.is_transient_error(response.status_code, service_code): 581 new_circuit_breaker_state = CircuitBreakerMonitor.get(self.circuit_breaker_name).state 582 if initial_circuit_breaker_state != new_circuit_breaker_state: 583 self.logger.warning("Circuit Breaker state changed from {} to {}".format(initial_circuit_breaker_state, new_circuit_breaker_state)) 584 self.raise_transient_service_error(request, response, service_code, message) 585 else: 586 self.raise_service_error(request, response, service_code, message) 587 588 if stream: 589 # Don't unpack a streaming response body 590 deserialized_data = response 591 elif response_type == BYTES_RESPONSE_TYPE: 592 # Don't deserialize data responses. 593 deserialized_data = response.content 594 elif response_type: 595 deserialized_data = self.deserialize_response_data(response.content, response_type) 596 else: 597 deserialized_data = None 598 599 resp = Response(response.status_code, response.headers, deserialized_data, request) 600 self.logger.debug(utc_now() + "Response returned") 601 return resp 602 603 # Builds the client info string to be sent with each request. 604 def build_request_id(self): 605 return str(uuid.uuid4()).replace('-', '').upper() 606 607 def add_opc_retry_token_if_needed(self, header_params, retry_token_length=30): 608 if 'opc-retry-token' not in header_params: 609 header_params['opc-retry-token'] = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(retry_token_length)) 610 611 @staticmethod 612 def add_opc_client_retries_header(header_params): 613 if 'opc-client-retries' not in header_params: 614 header_params['opc-client-retries'] = "true" 615 616 def to_path_value(self, obj): 617 """ 618 Takes value and turn it into a string suitable for inclusion in 619 the path, by url-encoding. 620 621 :param obj: object or string value. 622 623 :return string: quoted value. 624 """ 625 if type(obj) == list: 626 return ','.join(obj) 627 else: 628 return str(obj) 629 630 def sanitize_for_serialization(self, obj, declared_type=None, field_name=None): 631 """ 632 Builds a JSON POST object. 633 634 If obj is str, int, float, bool, None, return directly. 635 If obj is datetime.datetime, datetime.date 636 convert to string in iso8601 format. 637 If obj is list, sanitize each element in the list. 638 If obj is dict, return the dict. 639 If obj is swagger model, return the properties dict. 640 641 :param obj: The data to serialize. 642 :return: The serialized form of data. 643 """ 644 types = (six.string_types, six.integer_types, float, bool, type(None)) 645 646 declared_swagger_type_to_acceptable_python_types = { 647 'str': six.string_types, 648 'bool': bool, 649 'int': (float, six.integer_types), 650 'float': (float, six.integer_types) 651 } 652 653 # if there is a declared type for this obj, then validate that obj is of that type. None types (either None or the NONE_SENTINEL) are not validated but 654 # instead passed through 655 if declared_type and not self.is_none_or_none_sentinel(obj): 656 if declared_type.startswith('dict(') and not isinstance(obj, dict): 657 self.raise_type_error_serializing_model(field_name, obj, declared_type) 658 elif declared_type.startswith('list[') and not (isinstance(obj, list) or isinstance(obj, tuple)): 659 self.raise_type_error_serializing_model(field_name, obj, declared_type) 660 elif declared_type in self.complex_type_mappings: 661 # if its supposed to be one of our models, it can either be an instance of that model OR a dict 662 if not isinstance(obj, dict) and not isinstance(obj, self.complex_type_mappings[declared_type]): 663 self.raise_type_error_serializing_model(field_name, obj, declared_type) 664 elif declared_type in declared_swagger_type_to_acceptable_python_types and not isinstance(obj, declared_swagger_type_to_acceptable_python_types[declared_type]): 665 # if its a primitive with corresponding acceptable python types, validate that obj is an instance of one of those acceptable types 666 self.raise_type_error_serializing_model(field_name, obj, declared_type) 667 668 if isinstance(obj, types): 669 return obj 670 elif obj is NONE_SENTINEL: 671 return None 672 elif isinstance(obj, list) or isinstance(obj, tuple): 673 return [self.sanitize_for_serialization( 674 sub_obj, 675 self.extract_list_item_type_from_swagger_type(declared_type) if declared_type else None, 676 field_name + '[*]' if field_name else None) 677 for sub_obj in obj] 678 elif isinstance(obj, datetime): 679 if not obj.tzinfo: 680 obj = pytz.utc.localize(obj) 681 return obj.astimezone(pytz.utc).isoformat().replace('+00:00', 'Z') 682 elif isinstance(obj, date): 683 return obj.isoformat() 684 else: 685 if isinstance(obj, dict): 686 obj_dict = obj 687 688 keys_to_types_and_field_name = None 689 690 # if there is a declared type, then we can use that to validate the types of values in the dict 691 if declared_type: 692 dict_value_type = self.extract_dict_value_type_from_swagger_type(declared_type) 693 keys_to_types_and_field_name = {k: (dict_value_type, k) for k in obj_dict} 694 else: 695 # at this point we are assuming it is one of our models with swagger_types so explicitly throw if its not to give a better error 696 if not hasattr(obj, 'swagger_types'): 697 raise TypeError('Not able to serialize data: {} of type: {} in field: {}'.format(str(obj), type(obj).__name__, field_name)) 698 699 obj_dict = {obj.attribute_map[attr]: getattr(obj, attr) 700 for attr, _ in obj.swagger_types.items() 701 if getattr(obj, attr) is not None} 702 703 keys_to_types_and_field_name = {obj.attribute_map[attr]: (swagger_type, attr) for attr, swagger_type in six.iteritems(obj.swagger_types)} 704 705 sanitized_dict = {} 706 for key, val in six.iteritems(obj_dict): 707 value_declared_type = None 708 inner_field_name = key 709 if keys_to_types_and_field_name: 710 value_declared_type = keys_to_types_and_field_name[key][0] 711 inner_field_name = keys_to_types_and_field_name[key][1] 712 713 inner_field_name = '{}.{}'.format(field_name, inner_field_name) if field_name else inner_field_name 714 sanitized_dict[key] = self.sanitize_for_serialization(val, value_declared_type, inner_field_name) 715 716 return sanitized_dict 717 718 def is_none_or_none_sentinel(self, obj): 719 return (obj is None) or (obj is NONE_SENTINEL) 720 721 def raise_type_error_serializing_model(self, field_name, obj, declared_type): 722 raise TypeError('Field {} with value {} was expected to be of type {} but was of type {}'.format(field_name, str(obj), declared_type, type(obj).__name__)) 723 724 def extract_dict_value_type_from_swagger_type(self, swagger_type): 725 m = DICT_VALUE_TYPE_REGEX.search(swagger_type) 726 727 result = None 728 if m: 729 result = m.group(1) 730 731 return result 732 733 def extract_list_item_type_from_swagger_type(self, swagger_type): 734 m = LIST_ITEM_TYPE_REGEX.search(swagger_type) 735 736 result = None 737 if m: 738 result = m.group(1) 739 740 return result 741 742 def raise_service_error(self, request, response, service_code, message): 743 raise exceptions.ServiceError( 744 response.status_code, 745 service_code, 746 response.headers, 747 message, 748 original_request=request) 749 750 def raise_transient_service_error(self, request, response, service_code, message): 751 raise exceptions.TransientServiceError( 752 response.status_code, 753 service_code, 754 response.headers, 755 message, 756 original_request=request) 757 758 def get_deserialized_service_code_and_message(self, response): 759 deserialized_data = self.deserialize_response_data(response.content, 'object') 760 service_code = None 761 message = None 762 763 if isinstance(deserialized_data, dict): 764 service_code = deserialized_data.get('code') 765 message = deserialized_data.get('message') 766 else: 767 # Deserialized data should be a string if we couldn't deserialize into a dict (i.e. it failed 768 # json.loads()). There could still be error information of value to the customer, so instead 769 # of black holing the message, just put it in the error 770 message = deserialized_data 771 772 return service_code, message 773 774 def deserialize_response_data(self, response_data, response_type): 775 """ 776 Deserializes response into an object. 777 778 :param response_data: object to be deserialized. 779 :param response_type: class literal for 780 deserialized object, or string of class name. 781 782 :return: deserialized object. 783 """ 784 # response.content is always bytes 785 response_data = response_data.decode('utf8') 786 787 try: 788 json_response = json.loads(response_data) 789 # Load everything as JSON and then verify that the object returned 790 # is a string (six.text_type) if the response type is a string. 791 # This is matches the previous behavior, which happens to strip 792 # the embedded quotes in the get_namespace response. 793 # There is the potential that an API will declare that it returns 794 # a string and the string will be a valid JSON Object. In that case 795 # we do not update the response_data with the json_response. 796 # If we do later steps will fail because they are expecting the 797 # response_data to be a string. 798 if response_type != "str" or type(json_response) == six.text_type: 799 response_data = json_response 800 except ValueError: 801 pass 802 803 if self.skip_deserialization: 804 return response_data 805 else: 806 start = timer() 807 res = self.__deserialize(response_data, response_type) 808 end = timer() 809 self.logger.debug(utc_now() + 'python SDK time elapsed for deserializing: {}'.format(str(end - start))) 810 return res 811 812 def __deserialize(self, data, cls): 813 """ 814 Deserialize a dict, list, or str into an object. 815 816 :param data: dict, list or str 817 :param cls: string of class name 818 819 :return: object. 820 """ 821 if data is None: 822 return None 823 824 if cls.startswith('list['): 825 sub_kls = re.match('list\[(.*)\]', cls).group(1) # noqa: W605 826 return [self.__deserialize(sub_data, sub_kls) 827 for sub_data in data] 828 829 if cls.startswith('dict('): 830 sub_kls = re.match('dict\(([^,]*), (.*)\)', cls).group(2) # noqa: W605 831 return {k: self.__deserialize(v, sub_kls) 832 for k, v in data.items()} 833 834 # Enums are not present in type mappings, and they are strings, so we need to call __deserialize_primitive() 835 if cls in self.type_mappings: 836 cls = self.type_mappings[cls] 837 else: 838 return self.__deserialize_primitive(data, cls) 839 840 if hasattr(cls, 'get_subtype'): 841 # Use the discriminator value to get the correct subtype. 842 cls = cls.get_subtype(data) # get_subtype returns a str 843 cls = self.type_mappings[cls] 844 845 if cls in [int, float, six.u, bool]: 846 return self.__deserialize_primitive(data, cls) 847 elif cls == object: 848 return data 849 elif cls == date: 850 return self.__deserialize_date(data) 851 elif cls == datetime: 852 return self.__deserialize_datetime(data) 853 else: 854 return self.__deserialize_model(data, cls) 855 856 def __deserialize_primitive(self, data, cls): 857 """ 858 Deserializes string to primitive type. 859 860 :param data: str. 861 :param cls: class literal. 862 863 :return: int, float, str, bool. 864 """ 865 try: 866 value = cls(data) 867 except UnicodeEncodeError: 868 value = six.u(data) 869 except TypeError: 870 value = data 871 return value 872 873 def __deserialize_date(self, string): 874 """ 875 Deserializes string to date. 876 877 :param string: str. 878 :return: date. 879 """ 880 try: 881 return parse(string).date() 882 except ImportError: 883 return string 884 except ValueError: 885 raise Exception("Failed to parse `{0}` into a date object".format(string)) 886 887 def __deserialize_datetime(self, string): 888 """ 889 Deserializes string to datetime. 890 891 The string should be in iso8601 datetime format. 892 893 :param string: str. 894 :return: datetime. 895 """ 896 try: 897 # If this parser creates a date without raising an exception 898 # then the time zone is utc and needs to be set. 899 naivedatetime = datetime.strptime(string, "%Y-%m-%dT%H:%M:%S.%fZ") 900 awaredatetime = naivedatetime.replace(tzinfo=tz.tzutc()) 901 return awaredatetime 902 903 except ValueError: 904 try: 905 return parse(string) 906 except ImportError: 907 return string 908 except ValueError: 909 raise Exception("Failed to parse `{0}` into a datetime object".format(string)) 910 except ImportError: 911 return string 912 913 def __deserialize_model(self, data, cls): 914 """ 915 Deserializes list or dict to model. 916 917 :param data: dict, list. 918 :param cls: class literal. 919 :return: model object. 920 """ 921 instance = cls() 922 923 for attr, attr_type in instance.swagger_types.items(): 924 property = instance.attribute_map[attr] 925 if property in data: 926 value = data[property] 927 setattr(instance, attr, self.__deserialize(value, attr_type)) 928 929 return instance 930 931 @staticmethod 932 def get_preferred_retry_strategy(operation_retry_strategy, client_retry_strategy): 933 """ 934 Gets the preferred Retry Strategy for the client 935 :param operation_retry_strategy: (optional) Operation level Retry Strategy 936 :param client_retry_strategy: (optional) Client level Retry Strategy 937 :return: Preferred Retry Strategy. 938 """ 939 retry_strategy = None 940 if operation_retry_strategy: 941 retry_strategy = operation_retry_strategy 942 elif client_retry_strategy: 943 retry_strategy = client_retry_strategy 944 elif retry.GLOBAL_RETRY_STRATEGY: 945 retry_strategy = retry.GLOBAL_RETRY_STRATEGY 946 return retry_strategy 947