1# Copyright 2017 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Helpers for retrying functions with exponential back-off. 16 17The :class:`Retry` decorator can be used to retry functions that raise 18exceptions using exponential backoff. Because a exponential sleep algorithm is 19used, the retry is limited by a `deadline`. The deadline is the maxmimum amount 20of time a method can block. This is used instead of total number of retries 21because it is difficult to ascertain the amount of time a function can block 22when using total number of retries and exponential backoff. 23 24By default, this decorator will retry transient 25API errors (see :func:`if_transient_error`). For example: 26 27.. code-block:: python 28 29 @retry.Retry() 30 def call_flaky_rpc(): 31 return client.flaky_rpc() 32 33 # Will retry flaky_rpc() if it raises transient API errors. 34 result = call_flaky_rpc() 35 36You can pass a custom predicate to retry on different exceptions, such as 37waiting for an eventually consistent item to be available: 38 39.. code-block:: python 40 41 @retry.Retry(predicate=if_exception_type(exceptions.NotFound)) 42 def check_if_exists(): 43 return client.does_thing_exist() 44 45 is_available = check_if_exists() 46 47Some client library methods apply retry automatically. These methods can accept 48a ``retry`` parameter that allows you to configure the behavior: 49 50.. code-block:: python 51 52 my_retry = retry.Retry(deadline=60) 53 result = client.some_method(retry=my_retry) 54 55""" 56 57from __future__ import unicode_literals 58 59import datetime 60import functools 61import logging 62import random 63import time 64 65import requests.exceptions 66 67from google.api_core import datetime_helpers 68from google.api_core import exceptions 69from google.auth import exceptions as auth_exceptions 70 71_LOGGER = logging.getLogger(__name__) 72_DEFAULT_INITIAL_DELAY = 1.0 # seconds 73_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds 74_DEFAULT_DELAY_MULTIPLIER = 2.0 75_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds 76 77 78def if_exception_type(*exception_types): 79 """Creates a predicate to check if the exception is of a given type. 80 81 Args: 82 exception_types (Sequence[:func:`type`]): The exception types to check 83 for. 84 85 Returns: 86 Callable[Exception]: A predicate that returns True if the provided 87 exception is of the given type(s). 88 """ 89 90 def if_exception_type_predicate(exception): 91 """Bound predicate for checking an exception type.""" 92 return isinstance(exception, exception_types) 93 94 return if_exception_type_predicate 95 96 97# pylint: disable=invalid-name 98# Pylint sees this as a constant, but it is also an alias that should be 99# considered a function. 100if_transient_error = if_exception_type( 101 exceptions.InternalServerError, 102 exceptions.TooManyRequests, 103 exceptions.ServiceUnavailable, 104 requests.exceptions.ConnectionError, 105 requests.exceptions.ChunkedEncodingError, 106 auth_exceptions.TransportError, 107) 108"""A predicate that checks if an exception is a transient API error. 109 110The following server errors are considered transient: 111 112- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC 113 ``INTERNAL(13)`` and its subclasses. 114- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429 115- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503 116- :class:`requests.exceptions.ConnectionError` 117- :class:`requests.exceptions.ChunkedEncodingError` - The server declared 118 chunked encoding but sent an invalid chunk. 119- :class:`google.auth.exceptions.TransportError` - Used to indicate an 120 error occurred during an HTTP request. 121""" 122# pylint: enable=invalid-name 123 124 125def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER): 126 """Generates sleep intervals based on the exponential back-off algorithm. 127 128 This implements the `Truncated Exponential Back-off`_ algorithm. 129 130 .. _Truncated Exponential Back-off: 131 https://cloud.google.com/storage/docs/exponential-backoff 132 133 Args: 134 initial (float): The minimum amount of time to delay. This must 135 be greater than 0. 136 maximum (float): The maximum amount of time to delay. 137 multiplier (float): The multiplier applied to the delay. 138 139 Yields: 140 float: successive sleep intervals. 141 """ 142 delay = initial 143 while True: 144 # Introduce jitter by yielding a delay that is uniformly distributed 145 # to average out to the delay time. 146 yield min(random.uniform(0.0, delay * 2.0), maximum) 147 delay = delay * multiplier 148 149 150def retry_target(target, predicate, sleep_generator, deadline, on_error=None): 151 """Call a function and retry if it fails. 152 153 This is the lowest-level retry helper. Generally, you'll use the 154 higher-level retry helper :class:`Retry`. 155 156 Args: 157 target(Callable): The function to call and retry. This must be a 158 nullary function - apply arguments with `functools.partial`. 159 predicate (Callable[Exception]): A callable used to determine if an 160 exception raised by the target should be considered retryable. 161 It should return True to retry or False otherwise. 162 sleep_generator (Iterable[float]): An infinite iterator that determines 163 how long to sleep between retries. 164 deadline (float): How long to keep retrying the target. The last sleep 165 period is shortened as necessary, so that the last retry runs at 166 ``deadline`` (and not considerably beyond it). 167 on_error (Callable[Exception]): A function to call while processing a 168 retryable exception. Any error raised by this function will *not* 169 be caught. 170 171 Returns: 172 Any: the return value of the target function. 173 174 Raises: 175 google.api_core.RetryError: If the deadline is exceeded while retrying. 176 ValueError: If the sleep generator stops yielding values. 177 Exception: If the target raises a method that isn't retryable. 178 """ 179 if deadline is not None: 180 deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta( 181 seconds=deadline 182 ) 183 else: 184 deadline_datetime = None 185 186 last_exc = None 187 188 for sleep in sleep_generator: 189 try: 190 return target() 191 192 # pylint: disable=broad-except 193 # This function explicitly must deal with broad exceptions. 194 except Exception as exc: 195 if not predicate(exc): 196 raise 197 last_exc = exc 198 if on_error is not None: 199 on_error(exc) 200 201 now = datetime_helpers.utcnow() 202 203 if deadline_datetime is not None: 204 if deadline_datetime <= now: 205 raise exceptions.RetryError( 206 "Deadline of {:.1f}s exceeded while calling {}".format( 207 deadline, target 208 ), 209 last_exc, 210 ) from last_exc 211 else: 212 time_to_deadline = (deadline_datetime - now).total_seconds() 213 sleep = min(time_to_deadline, sleep) 214 215 _LOGGER.debug( 216 "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep) 217 ) 218 time.sleep(sleep) 219 220 raise ValueError("Sleep generator stopped yielding sleep values.") 221 222 223class Retry(object): 224 """Exponential retry decorator. 225 226 This class is a decorator used to add exponential back-off retry behavior 227 to an RPC call. 228 229 Although the default behavior is to retry transient API errors, a 230 different predicate can be provided to retry other exceptions. 231 232 Args: 233 predicate (Callable[Exception]): A callable that should return ``True`` 234 if the given exception is retryable. 235 initial (float): The minimum amount of time to delay in seconds. This 236 must be greater than 0. 237 maximum (float): The maximum amount of time to delay in seconds. 238 multiplier (float): The multiplier applied to the delay. 239 deadline (float): How long to keep retrying in seconds. The last sleep 240 period is shortened as necessary, so that the last retry runs at 241 ``deadline`` (and not considerably beyond it). 242 """ 243 244 def __init__( 245 self, 246 predicate=if_transient_error, 247 initial=_DEFAULT_INITIAL_DELAY, 248 maximum=_DEFAULT_MAXIMUM_DELAY, 249 multiplier=_DEFAULT_DELAY_MULTIPLIER, 250 deadline=_DEFAULT_DEADLINE, 251 on_error=None, 252 ): 253 self._predicate = predicate 254 self._initial = initial 255 self._multiplier = multiplier 256 self._maximum = maximum 257 self._deadline = deadline 258 self._on_error = on_error 259 260 def __call__(self, func, on_error=None): 261 """Wrap a callable with retry behavior. 262 263 Args: 264 func (Callable): The callable to add retry behavior to. 265 on_error (Callable[Exception]): A function to call while processing 266 a retryable exception. Any error raised by this function will 267 *not* be caught. 268 269 Returns: 270 Callable: A callable that will invoke ``func`` with retry 271 behavior. 272 """ 273 if self._on_error is not None: 274 on_error = self._on_error 275 276 @functools.wraps(func) 277 def retry_wrapped_func(*args, **kwargs): 278 """A wrapper that calls target function with retry.""" 279 target = functools.partial(func, *args, **kwargs) 280 sleep_generator = exponential_sleep_generator( 281 self._initial, self._maximum, multiplier=self._multiplier 282 ) 283 return retry_target( 284 target, 285 self._predicate, 286 sleep_generator, 287 self._deadline, 288 on_error=on_error, 289 ) 290 291 return retry_wrapped_func 292 293 @property 294 def deadline(self): 295 return self._deadline 296 297 def with_deadline(self, deadline): 298 """Return a copy of this retry with the given deadline. 299 300 Args: 301 deadline (float): How long to keep retrying. 302 303 Returns: 304 Retry: A new retry instance with the given deadline. 305 """ 306 return Retry( 307 predicate=self._predicate, 308 initial=self._initial, 309 maximum=self._maximum, 310 multiplier=self._multiplier, 311 deadline=deadline, 312 on_error=self._on_error, 313 ) 314 315 def with_predicate(self, predicate): 316 """Return a copy of this retry with the given predicate. 317 318 Args: 319 predicate (Callable[Exception]): A callable that should return 320 ``True`` if the given exception is retryable. 321 322 Returns: 323 Retry: A new retry instance with the given predicate. 324 """ 325 return Retry( 326 predicate=predicate, 327 initial=self._initial, 328 maximum=self._maximum, 329 multiplier=self._multiplier, 330 deadline=self._deadline, 331 on_error=self._on_error, 332 ) 333 334 def with_delay(self, initial=None, maximum=None, multiplier=None): 335 """Return a copy of this retry with the given delay options. 336 337 Args: 338 initial (float): The minimum amount of time to delay. This must 339 be greater than 0. 340 maximum (float): The maximum amount of time to delay. 341 multiplier (float): The multiplier applied to the delay. 342 343 Returns: 344 Retry: A new retry instance with the given predicate. 345 """ 346 return Retry( 347 predicate=self._predicate, 348 initial=initial if initial is not None else self._initial, 349 maximum=maximum if maximum is not None else self._maximum, 350 multiplier=multiplier if multiplier is not None else self._multiplier, 351 deadline=self._deadline, 352 on_error=self._on_error, 353 ) 354 355 def __str__(self): 356 return ( 357 "<Retry predicate={}, initial={:.1f}, maximum={:.1f}, " 358 "multiplier={:.1f}, deadline={:.1f}, on_error={}>".format( 359 self._predicate, 360 self._initial, 361 self._maximum, 362 self._multiplier, 363 self._deadline, 364 self._on_error, 365 ) 366 ) 367