1// +build !providerless
2
3/*
4Copyright 2019 The Kubernetes Authors.
5
6Licensed under the Apache License, Version 2.0 (the "License");
7you may not use this file except in compliance with the License.
8You may obtain a copy of the License at
9
10    http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in writing, software
13distributed under the License is distributed on an "AS IS" BASIS,
14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15See the License for the specific language governing permissions and
16limitations under the License.
17*/
18
19package retry
20
21import (
22	"bytes"
23	"fmt"
24	"io/ioutil"
25	"net/http"
26	"strconv"
27	"strings"
28	"time"
29
30	"k8s.io/klog/v2"
31)
32
33const (
34	// RetryAfterHeaderKey is the retry-after header key in ARM responses.
35	RetryAfterHeaderKey = "Retry-After"
36)
37
38var (
39	// The function to get current time.
40	now = time.Now
41
42	// StatusCodesForRetry are a defined group of status code for which the client will retry.
43	StatusCodesForRetry = []int{
44		http.StatusRequestTimeout,      // 408
45		http.StatusInternalServerError, // 500
46		http.StatusBadGateway,          // 502
47		http.StatusServiceUnavailable,  // 503
48		http.StatusGatewayTimeout,      // 504
49	}
50)
51
52// Error indicates an error returned by Azure APIs.
53type Error struct {
54	// Retriable indicates whether the request is retriable.
55	Retriable bool
56	// HTTPStatusCode indicates the response HTTP status code.
57	HTTPStatusCode int
58	// RetryAfter indicates the time when the request should retry after throttling.
59	// A throttled request is retriable.
60	RetryAfter time.Time
61	// RetryAfter indicates the raw error from API.
62	RawError error
63}
64
65// Error returns the error.
66// Note that Error doesn't implement error interface because (nil *Error) != (nil error).
67func (err *Error) Error() error {
68	if err == nil {
69		return nil
70	}
71
72	// Convert time to seconds for better logging.
73	retryAfterSeconds := 0
74	curTime := now()
75	if err.RetryAfter.After(curTime) {
76		retryAfterSeconds = int(err.RetryAfter.Sub(curTime) / time.Second)
77	}
78
79	return fmt.Errorf("Retriable: %v, RetryAfter: %ds, HTTPStatusCode: %d, RawError: %v",
80		err.Retriable, retryAfterSeconds, err.HTTPStatusCode, err.RawError)
81}
82
83// IsThrottled returns true the if the request is being throttled.
84func (err *Error) IsThrottled() bool {
85	if err == nil {
86		return false
87	}
88
89	return err.HTTPStatusCode == http.StatusTooManyRequests || err.RetryAfter.After(now())
90}
91
92// IsNotFound returns true the if the requested object wasn't found
93func (err *Error) IsNotFound() bool {
94	if err == nil {
95		return false
96	}
97
98	return err.HTTPStatusCode == http.StatusNotFound
99}
100
101// NewError creates a new Error.
102func NewError(retriable bool, err error) *Error {
103	return &Error{
104		Retriable: retriable,
105		RawError:  err,
106	}
107}
108
109// GetRetriableError gets new retriable Error.
110func GetRetriableError(err error) *Error {
111	return &Error{
112		Retriable: true,
113		RawError:  err,
114	}
115}
116
117// GetRateLimitError creates a new error for rate limiting.
118func GetRateLimitError(isWrite bool, opName string) *Error {
119	opType := "read"
120	if isWrite {
121		opType = "write"
122	}
123	return GetRetriableError(fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", opType, opName))
124}
125
126// GetThrottlingError creates a new error for throttling.
127func GetThrottlingError(operation, reason string, retryAfter time.Time) *Error {
128	rawError := fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", operation, reason)
129	return &Error{
130		Retriable:  true,
131		RawError:   rawError,
132		RetryAfter: retryAfter,
133	}
134}
135
136// GetError gets a new Error based on resp and error.
137func GetError(resp *http.Response, err error) *Error {
138	if err == nil && resp == nil {
139		return nil
140	}
141
142	if err == nil && resp != nil && isSuccessHTTPResponse(resp) {
143		// HTTP 2xx suggests a successful response
144		return nil
145	}
146
147	retryAfter := time.Time{}
148	if retryAfterDuration := getRetryAfter(resp); retryAfterDuration != 0 {
149		retryAfter = now().Add(retryAfterDuration)
150	}
151	return &Error{
152		RawError:       getRawError(resp, err),
153		RetryAfter:     retryAfter,
154		Retriable:      shouldRetryHTTPRequest(resp, err),
155		HTTPStatusCode: getHTTPStatusCode(resp),
156	}
157}
158
159// isSuccessHTTPResponse determines if the response from an HTTP request suggests success
160func isSuccessHTTPResponse(resp *http.Response) bool {
161	if resp == nil {
162		return false
163	}
164
165	// HTTP 2xx suggests a successful response
166	if 199 < resp.StatusCode && resp.StatusCode < 300 {
167		return true
168	}
169
170	return false
171}
172
173func getRawError(resp *http.Response, err error) error {
174	if err != nil {
175		return err
176	}
177
178	if resp == nil || resp.Body == nil {
179		return fmt.Errorf("empty HTTP response")
180	}
181
182	// return the http status if it is unable to get response body.
183	defer resp.Body.Close()
184	respBody, _ := ioutil.ReadAll(resp.Body)
185	resp.Body = ioutil.NopCloser(bytes.NewReader(respBody))
186	if len(respBody) == 0 {
187		return fmt.Errorf("HTTP status code (%d)", resp.StatusCode)
188	}
189
190	// return the raw response body.
191	return fmt.Errorf("%s", string(respBody))
192}
193
194func getHTTPStatusCode(resp *http.Response) int {
195	if resp == nil {
196		return -1
197	}
198
199	return resp.StatusCode
200}
201
202// shouldRetryHTTPRequest determines if the request is retriable.
203func shouldRetryHTTPRequest(resp *http.Response, err error) bool {
204	if resp != nil {
205		for _, code := range StatusCodesForRetry {
206			if resp.StatusCode == code {
207				return true
208			}
209		}
210
211		// should retry on <200, error>.
212		if isSuccessHTTPResponse(resp) && err != nil {
213			return true
214		}
215
216		return false
217	}
218
219	// should retry when error is not nil and no http.Response.
220	if err != nil {
221		return true
222	}
223
224	return false
225}
226
227// getRetryAfter gets the retryAfter from http response.
228// The value of Retry-After can be either the number of seconds or a date in RFC1123 format.
229func getRetryAfter(resp *http.Response) time.Duration {
230	if resp == nil {
231		return 0
232	}
233
234	ra := resp.Header.Get(RetryAfterHeaderKey)
235	if ra == "" {
236		return 0
237	}
238
239	var dur time.Duration
240	if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 {
241		dur = time.Duration(retryAfter) * time.Second
242	} else if t, err := time.Parse(time.RFC1123, ra); err == nil {
243		dur = t.Sub(now())
244	}
245	return dur
246}
247
248// GetErrorWithRetriableHTTPStatusCodes gets an error with RetriableHTTPStatusCodes.
249// It is used to retry on some HTTPStatusCodes.
250func GetErrorWithRetriableHTTPStatusCodes(resp *http.Response, err error, retriableHTTPStatusCodes []int) *Error {
251	rerr := GetError(resp, err)
252	if rerr == nil {
253		return nil
254	}
255
256	for _, code := range retriableHTTPStatusCodes {
257		if rerr.HTTPStatusCode == code {
258			rerr.Retriable = true
259			break
260		}
261	}
262
263	return rerr
264}
265
266// GetStatusNotFoundAndForbiddenIgnoredError gets an error with StatusNotFound and StatusForbidden ignored.
267// It is only used in DELETE operations.
268func GetStatusNotFoundAndForbiddenIgnoredError(resp *http.Response, err error) *Error {
269	rerr := GetError(resp, err)
270	if rerr == nil {
271		return nil
272	}
273
274	// Returns nil when it is StatusNotFound error.
275	if rerr.HTTPStatusCode == http.StatusNotFound {
276		klog.V(3).Infof("Ignoring StatusNotFound error: %v", rerr)
277		return nil
278	}
279
280	// Returns nil if the status code is StatusForbidden.
281	// This happens when AuthorizationFailed is reported from Azure API.
282	if rerr.HTTPStatusCode == http.StatusForbidden {
283		klog.V(3).Infof("Ignoring StatusForbidden error: %v", rerr)
284		return nil
285	}
286
287	return rerr
288}
289
290// IsErrorRetriable returns true if the error is retriable.
291func IsErrorRetriable(err error) bool {
292	if err == nil {
293		return false
294	}
295
296	return strings.Contains(err.Error(), "Retriable: true")
297}
298
299// HasStatusForbiddenOrIgnoredError return true if the given error code is part of the error message
300// This should only be used when trying to delete resources
301func HasStatusForbiddenOrIgnoredError(err error) bool {
302	if err == nil {
303		return false
304	}
305
306	if strings.Contains(err.Error(), fmt.Sprintf("HTTPStatusCode: %d", http.StatusNotFound)) {
307		return true
308	}
309
310	if strings.Contains(err.Error(), fmt.Sprintf("HTTPStatusCode: %d", http.StatusForbidden)) {
311		return true
312	}
313	return false
314}
315