1# Licensed to the Apache Software Foundation (ASF) under one or more 2# contributor license agreements. See the NOTICE file distributed with 3# this work for additional information regarding copyright ownership. 4# The ASF licenses this file to You under the Apache License, Version 2.0 5# (the "License"); you may not use this file except in compliance with 6# the License. You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import socket 17import ssl 18import time 19from datetime import datetime, timedelta 20from functools import wraps 21import logging 22 23from libcloud.utils.py3 import httplib 24from libcloud.common.exceptions import RateLimitReachedError 25 26__all__ = [ 27 'Retry', 28 'RetryForeverOnRateLimitError', 29] 30 31_logger = logging.getLogger(__name__) 32# Error message which indicates a transient SSL error upon which request 33# can be retried 34TRANSIENT_SSL_ERROR = 'The read operation timed out' 35 36 37class TransientSSLError(ssl.SSLError): 38 """Represent transient SSL errors, e.g. timeouts""" 39 pass 40 41 42# Constants used by the ``retry`` class 43# All the time values (timeout, delay, backoff) are in seconds 44DEFAULT_TIMEOUT = 30 # default retry timeout 45DEFAULT_DELAY = 1 # default sleep delay used in each iterator 46DEFAULT_BACKOFF = 1 # retry backup multiplier 47RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror, 48 httplib.NotConnected, httplib.ImproperConnectionState, 49 TransientSSLError) 50 51 52class MinimalRetry: 53 54 def __init__(self, retry_delay=DEFAULT_DELAY, 55 timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF): 56 """ 57 Wrapper around retrying that helps to handle common transient 58 exceptions. 59 60 This minimalistic version only retries SSL errors and rate limiting. 61 62 :param retry_delay: retry delay between the attempts. 63 :param timeout: maximum time to wait. 64 :param backoff: multiplier added to delay between attempts. 65 66 :Example: 67 68 retry_request = MinimalRetry(timeout=1, retry_delay=1, backoff=1) 69 retry_request(self.connection.request)() 70 """ 71 72 if retry_delay is None: 73 retry_delay = DEFAULT_DELAY 74 if timeout is None: 75 timeout = DEFAULT_TIMEOUT 76 if backoff is None: 77 backoff = DEFAULT_BACKOFF 78 79 timeout = max(timeout, 0) 80 81 self.retry_delay = retry_delay 82 self.timeout = timeout 83 self.backoff = backoff 84 85 def __call__(self, func): 86 def transform_ssl_error(function, *args, **kwargs): 87 try: 88 return function(*args, **kwargs) 89 except ssl.SSLError as exc: 90 if TRANSIENT_SSL_ERROR in str(exc): 91 raise TransientSSLError(*exc.args) 92 93 raise exc 94 95 @wraps(func) 96 def retry_loop(*args, **kwargs): 97 current_delay = self.retry_delay 98 end = datetime.now() + timedelta(seconds=self.timeout) 99 last_exc = None 100 101 while datetime.now() < end: 102 try: 103 return transform_ssl_error(func, *args, **kwargs) 104 except Exception as exc: 105 last_exc = exc 106 107 if isinstance(exc, RateLimitReachedError): 108 _logger.debug("You are being rate limited, backing " 109 "off...") 110 111 # NOTE: Retry after defaults to 0 in the 112 # RateLimitReachedError class so we a use more 113 # reasonable default in case that attribute is not 114 # present. This way we prevent busy waiting, etc. 115 retry_after = exc.retry_after if exc.retry_after else 2 116 time.sleep(retry_after) 117 118 # Reset delay if we're told to wait due to rate 119 # limiting 120 current_delay = self.retry_delay 121 elif self.should_retry(exc): 122 time.sleep(current_delay) 123 current_delay *= self.backoff 124 else: 125 raise 126 127 raise last_exc 128 129 return retry_loop 130 131 def should_retry(self, exception): 132 return False 133 134 135class Retry(MinimalRetry): 136 137 def __init__(self, retry_exceptions=RETRY_EXCEPTIONS, 138 retry_delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT, 139 backoff=DEFAULT_BACKOFF): 140 """ 141 Wrapper around retrying that helps to handle common transient 142 exceptions. 143 144 This version retries the errors that 145 `libcloud.utils.retry:MinimalRetry` retries and all errors of the 146 exception types that are given. 147 148 :param retry_exceptions: types of exceptions to retry on. 149 :param retry_delay: retry delay between the attempts. 150 :param timeout: maximum time to wait. 151 :param backoff: multiplier added to delay between attempts. 152 153 :Example: 154 155 retry_request = Retry(retry_exceptions=(httplib.NotConnected,), 156 timeout=1, retry_delay=1, backoff=1) 157 retry_request(self.connection.request)() 158 """ 159 160 super().__init__(retry_delay=retry_delay, timeout=timeout, 161 backoff=backoff) 162 if retry_exceptions is None: 163 retry_exceptions = RETRY_EXCEPTIONS 164 self.retry_exceptions = retry_exceptions 165 166 def should_retry(self, exception): 167 return isinstance(exception, tuple(self.retry_exceptions)) 168 169 170class RetryForeverOnRateLimitError(Retry): 171 """ 172 This class is only here for backward compatibility reasons with 173 pre-Libcloud v3.3.2. 174 175 If works by ignoring timeout argument and retrying forever until API 176 is returning 429 RateLimitReached errors. 177 178 In most cases using this class is not a good idea since it can cause code 179 to hang and retry for ever in case API continues to return retry limit 180 reached. 181 """ 182 183 def __call__(self, func): 184 def transform_ssl_error(function, *args, **kwargs): 185 try: 186 return function(*args, **kwargs) 187 except ssl.SSLError as exc: 188 if TRANSIENT_SSL_ERROR in str(exc): 189 raise TransientSSLError(*exc.args) 190 191 raise exc 192 193 @wraps(func) 194 def retry_loop(*args, **kwargs): 195 current_delay = self.retry_delay 196 end = datetime.now() + timedelta(seconds=self.timeout) 197 198 while True: 199 try: 200 return transform_ssl_error(func, *args, **kwargs) 201 except Exception as exc: 202 if isinstance(exc, RateLimitReachedError): 203 time.sleep(exc.retry_after) 204 205 # Reset retries if we're told to wait due to rate 206 # limiting 207 current_delay = self.retry_delay 208 end = datetime.now() + timedelta( 209 seconds=exc.retry_after + self.timeout) 210 elif datetime.now() >= end: 211 raise 212 elif self.should_retry(exc): 213 time.sleep(current_delay) 214 current_delay *= self.backoff 215 else: 216 raise 217 218 return retry_loop 219