1# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ 2# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"). You 5# may not use this file except in compliance with the License. A copy of 6# the License is located at 7# 8# http://aws.amazon.com/apache2.0/ 9# 10# or in the "license" file accompanying this file. This file is 11# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12# ANY KIND, either express or implied. See the License for the specific 13# language governing permissions and limitations under the License. 14 15import random 16import functools 17import logging 18from binascii import crc32 19 20from botocore.exceptions import ( 21 ChecksumError, EndpointConnectionError, ReadTimeoutError, 22 ConnectionError, ConnectionClosedError, 23) 24 25 26logger = logging.getLogger(__name__) 27# The only supported error for now is GENERAL_CONNECTION_ERROR 28# which maps to requests generic ConnectionError. If we're able 29# to get more specific exceptions from requests we can update 30# this mapping with more specific exceptions. 31EXCEPTION_MAP = { 32 'GENERAL_CONNECTION_ERROR': [ 33 ConnectionError, ConnectionClosedError, ReadTimeoutError, 34 EndpointConnectionError 35 ], 36} 37 38 39def delay_exponential(base, growth_factor, attempts): 40 """Calculate time to sleep based on exponential function. 41 42 The format is:: 43 44 base * growth_factor ^ (attempts - 1) 45 46 If ``base`` is set to 'rand' then a random number between 47 0 and 1 will be used as the base. 48 Base must be greater than 0, otherwise a ValueError will be 49 raised. 50 51 """ 52 if base == 'rand': 53 base = random.random() 54 elif base <= 0: 55 raise ValueError("The 'base' param must be greater than 0, " 56 "got: %s" % base) 57 time_to_sleep = base * (growth_factor ** (attempts - 1)) 58 return time_to_sleep 59 60 61def create_exponential_delay_function(base, growth_factor): 62 """Create an exponential delay function based on the attempts. 63 64 This is used so that you only have to pass it the attempts 65 parameter to calculate the delay. 66 67 """ 68 return functools.partial( 69 delay_exponential, base=base, growth_factor=growth_factor) 70 71 72def create_retry_handler(config, operation_name=None): 73 checker = create_checker_from_retry_config( 74 config, operation_name=operation_name) 75 action = create_retry_action_from_config( 76 config, operation_name=operation_name) 77 return RetryHandler(checker=checker, action=action) 78 79 80def create_retry_action_from_config(config, operation_name=None): 81 # The spec has the possibility of supporting per policy 82 # actions, but right now, we assume this comes from the 83 # default section, which means that delay functions apply 84 # for every policy in the retry config (per service). 85 delay_config = config['__default__']['delay'] 86 if delay_config['type'] == 'exponential': 87 return create_exponential_delay_function( 88 base=delay_config['base'], 89 growth_factor=delay_config['growth_factor']) 90 91 92def create_checker_from_retry_config(config, operation_name=None): 93 checkers = [] 94 max_attempts = None 95 retryable_exceptions = [] 96 if '__default__' in config: 97 policies = config['__default__'].get('policies', []) 98 max_attempts = config['__default__']['max_attempts'] 99 for key in policies: 100 current_config = policies[key] 101 checkers.append(_create_single_checker(current_config)) 102 retry_exception = _extract_retryable_exception(current_config) 103 if retry_exception is not None: 104 retryable_exceptions.extend(retry_exception) 105 if operation_name is not None and config.get(operation_name) is not None: 106 operation_policies = config[operation_name]['policies'] 107 for key in operation_policies: 108 checkers.append(_create_single_checker(operation_policies[key])) 109 retry_exception = _extract_retryable_exception( 110 operation_policies[key]) 111 if retry_exception is not None: 112 retryable_exceptions.extend(retry_exception) 113 if len(checkers) == 1: 114 # Don't need to use a MultiChecker 115 return MaxAttemptsDecorator(checkers[0], max_attempts=max_attempts) 116 else: 117 multi_checker = MultiChecker(checkers) 118 return MaxAttemptsDecorator( 119 multi_checker, max_attempts=max_attempts, 120 retryable_exceptions=tuple(retryable_exceptions)) 121 122 123def _create_single_checker(config): 124 if 'response' in config['applies_when']: 125 return _create_single_response_checker( 126 config['applies_when']['response']) 127 elif 'socket_errors' in config['applies_when']: 128 return ExceptionRaiser() 129 130 131def _create_single_response_checker(response): 132 if 'service_error_code' in response: 133 checker = ServiceErrorCodeChecker( 134 status_code=response['http_status_code'], 135 error_code=response['service_error_code']) 136 elif 'http_status_code' in response: 137 checker = HTTPStatusCodeChecker( 138 status_code=response['http_status_code']) 139 elif 'crc32body' in response: 140 checker = CRC32Checker(header=response['crc32body']) 141 else: 142 # TODO: send a signal. 143 raise ValueError("Unknown retry policy") 144 return checker 145 146 147def _extract_retryable_exception(config): 148 applies_when = config['applies_when'] 149 if 'crc32body' in applies_when.get('response', {}): 150 return [ChecksumError] 151 elif 'socket_errors' in applies_when: 152 exceptions = [] 153 for name in applies_when['socket_errors']: 154 exceptions.extend(EXCEPTION_MAP[name]) 155 return exceptions 156 157 158class RetryHandler(object): 159 """Retry handler. 160 161 The retry handler takes two params, ``checker`` object 162 and an ``action`` object. 163 164 The ``checker`` object must be a callable object and based on a response 165 and an attempt number, determines whether or not sufficient criteria for 166 a retry has been met. If this is the case then the ``action`` object 167 (which also is a callable) determines what needs to happen in the event 168 of a retry. 169 170 """ 171 172 def __init__(self, checker, action): 173 self._checker = checker 174 self._action = action 175 176 def __call__(self, attempts, response, caught_exception, **kwargs): 177 """Handler for a retry. 178 179 Intended to be hooked up to an event handler (hence the **kwargs), 180 this will process retries appropriately. 181 182 """ 183 if self._checker(attempts, response, caught_exception): 184 result = self._action(attempts=attempts) 185 logger.debug("Retry needed, action of: %s", result) 186 return result 187 logger.debug("No retry needed.") 188 189 190class BaseChecker(object): 191 """Base class for retry checkers. 192 193 Each class is responsible for checking a single criteria that determines 194 whether or not a retry should not happen. 195 196 """ 197 def __call__(self, attempt_number, response, caught_exception): 198 """Determine if retry criteria matches. 199 200 Note that either ``response`` is not None and ``caught_exception`` is 201 None or ``response`` is None and ``caught_exception`` is not None. 202 203 :type attempt_number: int 204 :param attempt_number: The total number of times we've attempted 205 to send the request. 206 207 :param response: The HTTP response (if one was received). 208 209 :type caught_exception: Exception 210 :param caught_exception: Any exception that was caught while trying to 211 send the HTTP response. 212 213 :return: True, if the retry criteria matches (and therefore a retry 214 should occur. False if the criteria does not match. 215 216 """ 217 # The default implementation allows subclasses to not have to check 218 # whether or not response is None or not. 219 if response is not None: 220 return self._check_response(attempt_number, response) 221 elif caught_exception is not None: 222 return self._check_caught_exception( 223 attempt_number, caught_exception) 224 else: 225 raise ValueError("Both response and caught_exception are None.") 226 227 def _check_response(self, attempt_number, response): 228 pass 229 230 def _check_caught_exception(self, attempt_number, caught_exception): 231 pass 232 233 234class MaxAttemptsDecorator(BaseChecker): 235 """Allow retries up to a maximum number of attempts. 236 237 This will pass through calls to the decorated retry checker, provided 238 that the number of attempts does not exceed max_attempts. It will 239 also catch any retryable_exceptions passed in. Once max_attempts has 240 been exceeded, then False will be returned or the retryable_exceptions 241 that was previously being caught will be raised. 242 243 """ 244 def __init__(self, checker, max_attempts, retryable_exceptions=None): 245 self._checker = checker 246 self._max_attempts = max_attempts 247 self._retryable_exceptions = retryable_exceptions 248 249 def __call__(self, attempt_number, response, caught_exception): 250 should_retry = self._should_retry(attempt_number, response, 251 caught_exception) 252 if should_retry: 253 if attempt_number >= self._max_attempts: 254 # explicitly set MaxAttemptsReached 255 if response is not None and 'ResponseMetadata' in response[1]: 256 response[1]['ResponseMetadata']['MaxAttemptsReached'] = True 257 logger.debug("Reached the maximum number of retry " 258 "attempts: %s", attempt_number) 259 return False 260 else: 261 return should_retry 262 else: 263 return False 264 265 def _should_retry(self, attempt_number, response, caught_exception): 266 if self._retryable_exceptions and \ 267 attempt_number < self._max_attempts: 268 try: 269 return self._checker(attempt_number, response, caught_exception) 270 except self._retryable_exceptions as e: 271 logger.debug("retry needed, retryable exception caught: %s", 272 e, exc_info=True) 273 return True 274 else: 275 # If we've exceeded the max attempts we just let the exception 276 # propogate if one has occurred. 277 return self._checker(attempt_number, response, caught_exception) 278 279 280class HTTPStatusCodeChecker(BaseChecker): 281 def __init__(self, status_code): 282 self._status_code = status_code 283 284 def _check_response(self, attempt_number, response): 285 if response[0].status_code == self._status_code: 286 logger.debug( 287 "retry needed: retryable HTTP status code received: %s", 288 self._status_code) 289 return True 290 else: 291 return False 292 293 294class ServiceErrorCodeChecker(BaseChecker): 295 def __init__(self, status_code, error_code): 296 self._status_code = status_code 297 self._error_code = error_code 298 299 def _check_response(self, attempt_number, response): 300 if response[0].status_code == self._status_code: 301 actual_error_code = response[1].get('Error', {}).get('Code') 302 if actual_error_code == self._error_code: 303 logger.debug( 304 "retry needed: matching HTTP status and error code seen: " 305 "%s, %s", self._status_code, self._error_code) 306 return True 307 return False 308 309 310class MultiChecker(BaseChecker): 311 def __init__(self, checkers): 312 self._checkers = checkers 313 314 def __call__(self, attempt_number, response, caught_exception): 315 for checker in self._checkers: 316 checker_response = checker(attempt_number, response, 317 caught_exception) 318 if checker_response: 319 return checker_response 320 return False 321 322 323class CRC32Checker(BaseChecker): 324 def __init__(self, header): 325 # The header where the expected crc32 is located. 326 self._header_name = header 327 328 def _check_response(self, attempt_number, response): 329 http_response = response[0] 330 expected_crc = http_response.headers.get(self._header_name) 331 if expected_crc is None: 332 logger.debug("crc32 check skipped, the %s header is not " 333 "in the http response.", self._header_name) 334 else: 335 actual_crc32 = crc32(response[0].content) & 0xffffffff 336 if not actual_crc32 == int(expected_crc): 337 logger.debug( 338 "retry needed: crc32 check failed, expected != actual: " 339 "%s != %s", int(expected_crc), actual_crc32) 340 raise ChecksumError(checksum_type='crc32', 341 expected_checksum=int(expected_crc), 342 actual_checksum=actual_crc32) 343 344 345class ExceptionRaiser(BaseChecker): 346 """Raise any caught exceptions. 347 348 This class will raise any non None ``caught_exception``. 349 350 """ 351 def _check_caught_exception(self, attempt_number, caught_exception): 352 # This is implementation specific, but this class is useful by 353 # coordinating with the MaxAttemptsDecorator. 354 # The MaxAttemptsDecorator has a list of exceptions it should catch 355 # and retry, but something needs to come along and actually raise the 356 # caught_exception. That's what this class is being used for. If 357 # the MaxAttemptsDecorator is not interested in retrying the exception 358 # then this exception just propogates out past the retry code. 359 raise caught_exception 360