1# Licensed to the Apache Software Foundation (ASF) under one or more
2# contributor license agreements.  See the NOTICE file distributed with
3# this work for additional information regarding copyright ownership.
4# The ASF licenses this file to You under the Apache License, Version 2.0
5# (the "License"); you may not use this file except in compliance with
6# the License.  You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import socket
17import ssl
18import time
19from datetime import datetime, timedelta
20from functools import wraps
21import logging
22
23from libcloud.utils.py3 import httplib
24from libcloud.common.exceptions import RateLimitReachedError
25
26__all__ = [
27    'Retry',
28    'RetryForeverOnRateLimitError',
29]
30
31_logger = logging.getLogger(__name__)
32# Error message which indicates a transient SSL error upon which request
33# can be retried
34TRANSIENT_SSL_ERROR = 'The read operation timed out'
35
36
37class TransientSSLError(ssl.SSLError):
38    """Represent transient SSL errors, e.g. timeouts"""
39    pass
40
41
42# Constants used by the ``retry`` class
43# All the time values (timeout, delay, backoff) are in seconds
44DEFAULT_TIMEOUT = 30  # default retry timeout
45DEFAULT_DELAY = 1  # default sleep delay used in each iterator
46DEFAULT_BACKOFF = 1  # retry backup multiplier
47RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
48                    httplib.NotConnected, httplib.ImproperConnectionState,
49                    TransientSSLError)
50
51
52class MinimalRetry:
53
54    def __init__(self, retry_delay=DEFAULT_DELAY,
55                 timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF):
56        """
57        Wrapper around retrying that helps to handle common transient
58        exceptions.
59
60        This minimalistic version only retries SSL errors and rate limiting.
61
62        :param retry_delay: retry delay between the attempts.
63        :param timeout: maximum time to wait.
64        :param backoff: multiplier added to delay between attempts.
65
66        :Example:
67
68        retry_request = MinimalRetry(timeout=1, retry_delay=1, backoff=1)
69        retry_request(self.connection.request)()
70        """
71
72        if retry_delay is None:
73            retry_delay = DEFAULT_DELAY
74        if timeout is None:
75            timeout = DEFAULT_TIMEOUT
76        if backoff is None:
77            backoff = DEFAULT_BACKOFF
78
79        timeout = max(timeout, 0)
80
81        self.retry_delay = retry_delay
82        self.timeout = timeout
83        self.backoff = backoff
84
85    def __call__(self, func):
86        def transform_ssl_error(function, *args, **kwargs):
87            try:
88                return function(*args, **kwargs)
89            except ssl.SSLError as exc:
90                if TRANSIENT_SSL_ERROR in str(exc):
91                    raise TransientSSLError(*exc.args)
92
93                raise exc
94
95        @wraps(func)
96        def retry_loop(*args, **kwargs):
97            current_delay = self.retry_delay
98            end = datetime.now() + timedelta(seconds=self.timeout)
99            last_exc = None
100
101            while datetime.now() < end:
102                try:
103                    return transform_ssl_error(func, *args, **kwargs)
104                except Exception as exc:
105                    last_exc = exc
106
107                    if isinstance(exc, RateLimitReachedError):
108                        _logger.debug("You are being rate limited, backing "
109                                      "off...")
110
111                        # NOTE: Retry after defaults to 0 in the
112                        # RateLimitReachedError class so we a use more
113                        # reasonable default in case that attribute is not
114                        # present. This way we prevent busy waiting, etc.
115                        retry_after = exc.retry_after if exc.retry_after else 2
116                        time.sleep(retry_after)
117
118                        # Reset delay if we're told to wait due to rate
119                        # limiting
120                        current_delay = self.retry_delay
121                    elif self.should_retry(exc):
122                        time.sleep(current_delay)
123                        current_delay *= self.backoff
124                    else:
125                        raise
126
127            raise last_exc
128
129        return retry_loop
130
131    def should_retry(self, exception):
132        return False
133
134
135class Retry(MinimalRetry):
136
137    def __init__(self, retry_exceptions=RETRY_EXCEPTIONS,
138                 retry_delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT,
139                 backoff=DEFAULT_BACKOFF):
140        """
141        Wrapper around retrying that helps to handle common transient
142        exceptions.
143
144        This version retries the errors that
145        `libcloud.utils.retry:MinimalRetry` retries and all errors of the
146        exception types that are given.
147
148        :param retry_exceptions: types of exceptions to retry on.
149        :param retry_delay: retry delay between the attempts.
150        :param timeout: maximum time to wait.
151        :param backoff: multiplier added to delay between attempts.
152
153        :Example:
154
155        retry_request = Retry(retry_exceptions=(httplib.NotConnected,),
156                              timeout=1, retry_delay=1, backoff=1)
157        retry_request(self.connection.request)()
158        """
159
160        super().__init__(retry_delay=retry_delay, timeout=timeout,
161                         backoff=backoff)
162        if retry_exceptions is None:
163            retry_exceptions = RETRY_EXCEPTIONS
164        self.retry_exceptions = retry_exceptions
165
166    def should_retry(self, exception):
167        return isinstance(exception, tuple(self.retry_exceptions))
168
169
170class RetryForeverOnRateLimitError(Retry):
171    """
172    This class is only here for backward compatibility reasons with
173    pre-Libcloud v3.3.2.
174
175    If works by ignoring timeout argument and retrying forever until API
176    is returning 429 RateLimitReached errors.
177
178    In most cases using this class is not a good idea since it can cause code
179    to hang and retry for ever in case API continues to return retry limit
180    reached.
181    """
182
183    def __call__(self, func):
184        def transform_ssl_error(function, *args, **kwargs):
185            try:
186                return function(*args, **kwargs)
187            except ssl.SSLError as exc:
188                if TRANSIENT_SSL_ERROR in str(exc):
189                    raise TransientSSLError(*exc.args)
190
191                raise exc
192
193        @wraps(func)
194        def retry_loop(*args, **kwargs):
195            current_delay = self.retry_delay
196            end = datetime.now() + timedelta(seconds=self.timeout)
197
198            while True:
199                try:
200                    return transform_ssl_error(func, *args, **kwargs)
201                except Exception as exc:
202                    if isinstance(exc, RateLimitReachedError):
203                        time.sleep(exc.retry_after)
204
205                        # Reset retries if we're told to wait due to rate
206                        # limiting
207                        current_delay = self.retry_delay
208                        end = datetime.now() + timedelta(
209                            seconds=exc.retry_after + self.timeout)
210                    elif datetime.now() >= end:
211                        raise
212                    elif self.should_retry(exc):
213                        time.sleep(current_delay)
214                        current_delay *= self.backoff
215                    else:
216                        raise
217
218        return retry_loop
219