1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Helpers for retrying functions with exponential back-off.
16
17The :class:`Retry` decorator can be used to retry functions that raise
18exceptions using exponential backoff. Because a exponential sleep algorithm is
19used, the retry is limited by a `deadline`. The deadline is the maxmimum amount
20of time a method can block. This is used instead of total number of retries
21because it is difficult to ascertain the amount of time a function can block
22when using total number of retries and exponential backoff.
23
24By default, this decorator will retry transient
25API errors (see :func:`if_transient_error`). For example:
26
27.. code-block:: python
28
29    @retry.Retry()
30    def call_flaky_rpc():
31        return client.flaky_rpc()
32
33    # Will retry flaky_rpc() if it raises transient API errors.
34    result = call_flaky_rpc()
35
36You can pass a custom predicate to retry on different exceptions, such as
37waiting for an eventually consistent item to be available:
38
39.. code-block:: python
40
41    @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
42    def check_if_exists():
43        return client.does_thing_exist()
44
45    is_available = check_if_exists()
46
47Some client library methods apply retry automatically. These methods can accept
48a ``retry`` parameter that allows you to configure the behavior:
49
50.. code-block:: python
51
52    my_retry = retry.Retry(deadline=60)
53    result = client.some_method(retry=my_retry)
54
55"""
56
57from __future__ import unicode_literals
58
59import datetime
60import functools
61import logging
62import random
63import time
64
65import requests.exceptions
66
67from google.api_core import datetime_helpers
68from google.api_core import exceptions
69from google.auth import exceptions as auth_exceptions
70
71_LOGGER = logging.getLogger(__name__)
72_DEFAULT_INITIAL_DELAY = 1.0  # seconds
73_DEFAULT_MAXIMUM_DELAY = 60.0  # seconds
74_DEFAULT_DELAY_MULTIPLIER = 2.0
75_DEFAULT_DEADLINE = 60.0 * 2.0  # seconds
76
77
78def if_exception_type(*exception_types):
79    """Creates a predicate to check if the exception is of a given type.
80
81    Args:
82        exception_types (Sequence[:func:`type`]): The exception types to check
83            for.
84
85    Returns:
86        Callable[Exception]: A predicate that returns True if the provided
87            exception is of the given type(s).
88    """
89
90    def if_exception_type_predicate(exception):
91        """Bound predicate for checking an exception type."""
92        return isinstance(exception, exception_types)
93
94    return if_exception_type_predicate
95
96
97# pylint: disable=invalid-name
98# Pylint sees this as a constant, but it is also an alias that should be
99# considered a function.
100if_transient_error = if_exception_type(
101    exceptions.InternalServerError,
102    exceptions.TooManyRequests,
103    exceptions.ServiceUnavailable,
104    requests.exceptions.ConnectionError,
105    requests.exceptions.ChunkedEncodingError,
106    auth_exceptions.TransportError,
107)
108"""A predicate that checks if an exception is a transient API error.
109
110The following server errors are considered transient:
111
112- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
113    ``INTERNAL(13)`` and its subclasses.
114- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
115- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
116- :class:`requests.exceptions.ConnectionError`
117- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
118    chunked encoding but sent an invalid chunk.
119- :class:`google.auth.exceptions.TransportError` - Used to indicate an
120    error occurred during an HTTP request.
121"""
122# pylint: enable=invalid-name
123
124
125def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER):
126    """Generates sleep intervals based on the exponential back-off algorithm.
127
128    This implements the `Truncated Exponential Back-off`_ algorithm.
129
130    .. _Truncated Exponential Back-off:
131        https://cloud.google.com/storage/docs/exponential-backoff
132
133    Args:
134        initial (float): The minimum amount of time to delay. This must
135            be greater than 0.
136        maximum (float): The maximum amount of time to delay.
137        multiplier (float): The multiplier applied to the delay.
138
139    Yields:
140        float: successive sleep intervals.
141    """
142    delay = initial
143    while True:
144        # Introduce jitter by yielding a delay that is uniformly distributed
145        # to average out to the delay time.
146        yield min(random.uniform(0.0, delay * 2.0), maximum)
147        delay = delay * multiplier
148
149
150def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
151    """Call a function and retry if it fails.
152
153    This is the lowest-level retry helper. Generally, you'll use the
154    higher-level retry helper :class:`Retry`.
155
156    Args:
157        target(Callable): The function to call and retry. This must be a
158            nullary function - apply arguments with `functools.partial`.
159        predicate (Callable[Exception]): A callable used to determine if an
160            exception raised by the target should be considered retryable.
161            It should return True to retry or False otherwise.
162        sleep_generator (Iterable[float]): An infinite iterator that determines
163            how long to sleep between retries.
164        deadline (float): How long to keep retrying the target. The last sleep
165            period is shortened as necessary, so that the last retry runs at
166            ``deadline`` (and not considerably beyond it).
167        on_error (Callable[Exception]): A function to call while processing a
168            retryable exception.  Any error raised by this function will *not*
169            be caught.
170
171    Returns:
172        Any: the return value of the target function.
173
174    Raises:
175        google.api_core.RetryError: If the deadline is exceeded while retrying.
176        ValueError: If the sleep generator stops yielding values.
177        Exception: If the target raises a method that isn't retryable.
178    """
179    if deadline is not None:
180        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
181            seconds=deadline
182        )
183    else:
184        deadline_datetime = None
185
186    last_exc = None
187
188    for sleep in sleep_generator:
189        try:
190            return target()
191
192        # pylint: disable=broad-except
193        # This function explicitly must deal with broad exceptions.
194        except Exception as exc:
195            if not predicate(exc):
196                raise
197            last_exc = exc
198            if on_error is not None:
199                on_error(exc)
200
201        now = datetime_helpers.utcnow()
202
203        if deadline_datetime is not None:
204            if deadline_datetime <= now:
205                raise exceptions.RetryError(
206                    "Deadline of {:.1f}s exceeded while calling {}".format(
207                        deadline, target
208                    ),
209                    last_exc,
210                ) from last_exc
211            else:
212                time_to_deadline = (deadline_datetime - now).total_seconds()
213                sleep = min(time_to_deadline, sleep)
214
215        _LOGGER.debug(
216            "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
217        )
218        time.sleep(sleep)
219
220    raise ValueError("Sleep generator stopped yielding sleep values.")
221
222
223class Retry(object):
224    """Exponential retry decorator.
225
226    This class is a decorator used to add exponential back-off retry behavior
227    to an RPC call.
228
229    Although the default behavior is to retry transient API errors, a
230    different predicate can be provided to retry other exceptions.
231
232    Args:
233        predicate (Callable[Exception]): A callable that should return ``True``
234            if the given exception is retryable.
235        initial (float): The minimum amount of time to delay in seconds. This
236            must be greater than 0.
237        maximum (float): The maximum amount of time to delay in seconds.
238        multiplier (float): The multiplier applied to the delay.
239        deadline (float): How long to keep retrying in seconds. The last sleep
240            period is shortened as necessary, so that the last retry runs at
241            ``deadline`` (and not considerably beyond it).
242    """
243
244    def __init__(
245        self,
246        predicate=if_transient_error,
247        initial=_DEFAULT_INITIAL_DELAY,
248        maximum=_DEFAULT_MAXIMUM_DELAY,
249        multiplier=_DEFAULT_DELAY_MULTIPLIER,
250        deadline=_DEFAULT_DEADLINE,
251        on_error=None,
252    ):
253        self._predicate = predicate
254        self._initial = initial
255        self._multiplier = multiplier
256        self._maximum = maximum
257        self._deadline = deadline
258        self._on_error = on_error
259
260    def __call__(self, func, on_error=None):
261        """Wrap a callable with retry behavior.
262
263        Args:
264            func (Callable): The callable to add retry behavior to.
265            on_error (Callable[Exception]): A function to call while processing
266                a retryable exception. Any error raised by this function will
267                *not* be caught.
268
269        Returns:
270            Callable: A callable that will invoke ``func`` with retry
271                behavior.
272        """
273        if self._on_error is not None:
274            on_error = self._on_error
275
276        @functools.wraps(func)
277        def retry_wrapped_func(*args, **kwargs):
278            """A wrapper that calls target function with retry."""
279            target = functools.partial(func, *args, **kwargs)
280            sleep_generator = exponential_sleep_generator(
281                self._initial, self._maximum, multiplier=self._multiplier
282            )
283            return retry_target(
284                target,
285                self._predicate,
286                sleep_generator,
287                self._deadline,
288                on_error=on_error,
289            )
290
291        return retry_wrapped_func
292
293    @property
294    def deadline(self):
295        return self._deadline
296
297    def with_deadline(self, deadline):
298        """Return a copy of this retry with the given deadline.
299
300        Args:
301            deadline (float): How long to keep retrying.
302
303        Returns:
304            Retry: A new retry instance with the given deadline.
305        """
306        return Retry(
307            predicate=self._predicate,
308            initial=self._initial,
309            maximum=self._maximum,
310            multiplier=self._multiplier,
311            deadline=deadline,
312            on_error=self._on_error,
313        )
314
315    def with_predicate(self, predicate):
316        """Return a copy of this retry with the given predicate.
317
318        Args:
319            predicate (Callable[Exception]): A callable that should return
320                ``True`` if the given exception is retryable.
321
322        Returns:
323            Retry: A new retry instance with the given predicate.
324        """
325        return Retry(
326            predicate=predicate,
327            initial=self._initial,
328            maximum=self._maximum,
329            multiplier=self._multiplier,
330            deadline=self._deadline,
331            on_error=self._on_error,
332        )
333
334    def with_delay(self, initial=None, maximum=None, multiplier=None):
335        """Return a copy of this retry with the given delay options.
336
337        Args:
338            initial (float): The minimum amount of time to delay. This must
339                be greater than 0.
340            maximum (float): The maximum amount of time to delay.
341            multiplier (float): The multiplier applied to the delay.
342
343        Returns:
344            Retry: A new retry instance with the given predicate.
345        """
346        return Retry(
347            predicate=self._predicate,
348            initial=initial if initial is not None else self._initial,
349            maximum=maximum if maximum is not None else self._maximum,
350            multiplier=multiplier if multiplier is not None else self._multiplier,
351            deadline=self._deadline,
352            on_error=self._on_error,
353        )
354
355    def __str__(self):
356        return (
357            "<Retry predicate={}, initial={:.1f}, maximum={:.1f}, "
358            "multiplier={:.1f}, deadline={:.1f}, on_error={}>".format(
359                self._predicate,
360                self._initial,
361                self._maximum,
362                self._multiplier,
363                self._deadline,
364                self._on_error,
365            )
366        )
367