1// +build !providerless 2 3/* 4Copyright 2019 The Kubernetes Authors. 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19package retry 20 21import ( 22 "bytes" 23 "fmt" 24 "io/ioutil" 25 "net/http" 26 "strconv" 27 "strings" 28 "time" 29 30 "k8s.io/klog/v2" 31) 32 33const ( 34 // RetryAfterHeaderKey is the retry-after header key in ARM responses. 35 RetryAfterHeaderKey = "Retry-After" 36) 37 38var ( 39 // The function to get current time. 40 now = time.Now 41 42 // StatusCodesForRetry are a defined group of status code for which the client will retry. 43 StatusCodesForRetry = []int{ 44 http.StatusRequestTimeout, // 408 45 http.StatusInternalServerError, // 500 46 http.StatusBadGateway, // 502 47 http.StatusServiceUnavailable, // 503 48 http.StatusGatewayTimeout, // 504 49 } 50) 51 52// Error indicates an error returned by Azure APIs. 53type Error struct { 54 // Retriable indicates whether the request is retriable. 55 Retriable bool 56 // HTTPStatusCode indicates the response HTTP status code. 57 HTTPStatusCode int 58 // RetryAfter indicates the time when the request should retry after throttling. 59 // A throttled request is retriable. 60 RetryAfter time.Time 61 // RetryAfter indicates the raw error from API. 62 RawError error 63} 64 65// Error returns the error. 66// Note that Error doesn't implement error interface because (nil *Error) != (nil error). 67func (err *Error) Error() error { 68 if err == nil { 69 return nil 70 } 71 72 // Convert time to seconds for better logging. 73 retryAfterSeconds := 0 74 curTime := now() 75 if err.RetryAfter.After(curTime) { 76 retryAfterSeconds = int(err.RetryAfter.Sub(curTime) / time.Second) 77 } 78 79 return fmt.Errorf("Retriable: %v, RetryAfter: %ds, HTTPStatusCode: %d, RawError: %v", 80 err.Retriable, retryAfterSeconds, err.HTTPStatusCode, err.RawError) 81} 82 83// IsThrottled returns true the if the request is being throttled. 84func (err *Error) IsThrottled() bool { 85 if err == nil { 86 return false 87 } 88 89 return err.HTTPStatusCode == http.StatusTooManyRequests || err.RetryAfter.After(now()) 90} 91 92// IsNotFound returns true the if the requested object wasn't found 93func (err *Error) IsNotFound() bool { 94 if err == nil { 95 return false 96 } 97 98 return err.HTTPStatusCode == http.StatusNotFound 99} 100 101// NewError creates a new Error. 102func NewError(retriable bool, err error) *Error { 103 return &Error{ 104 Retriable: retriable, 105 RawError: err, 106 } 107} 108 109// GetRetriableError gets new retriable Error. 110func GetRetriableError(err error) *Error { 111 return &Error{ 112 Retriable: true, 113 RawError: err, 114 } 115} 116 117// GetRateLimitError creates a new error for rate limiting. 118func GetRateLimitError(isWrite bool, opName string) *Error { 119 opType := "read" 120 if isWrite { 121 opType = "write" 122 } 123 return GetRetriableError(fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", opType, opName)) 124} 125 126// GetThrottlingError creates a new error for throttling. 127func GetThrottlingError(operation, reason string, retryAfter time.Time) *Error { 128 rawError := fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", operation, reason) 129 return &Error{ 130 Retriable: true, 131 RawError: rawError, 132 RetryAfter: retryAfter, 133 } 134} 135 136// GetError gets a new Error based on resp and error. 137func GetError(resp *http.Response, err error) *Error { 138 if err == nil && resp == nil { 139 return nil 140 } 141 142 if err == nil && resp != nil && isSuccessHTTPResponse(resp) { 143 // HTTP 2xx suggests a successful response 144 return nil 145 } 146 147 retryAfter := time.Time{} 148 if retryAfterDuration := getRetryAfter(resp); retryAfterDuration != 0 { 149 retryAfter = now().Add(retryAfterDuration) 150 } 151 return &Error{ 152 RawError: getRawError(resp, err), 153 RetryAfter: retryAfter, 154 Retriable: shouldRetryHTTPRequest(resp, err), 155 HTTPStatusCode: getHTTPStatusCode(resp), 156 } 157} 158 159// isSuccessHTTPResponse determines if the response from an HTTP request suggests success 160func isSuccessHTTPResponse(resp *http.Response) bool { 161 if resp == nil { 162 return false 163 } 164 165 // HTTP 2xx suggests a successful response 166 if 199 < resp.StatusCode && resp.StatusCode < 300 { 167 return true 168 } 169 170 return false 171} 172 173func getRawError(resp *http.Response, err error) error { 174 if err != nil { 175 return err 176 } 177 178 if resp == nil || resp.Body == nil { 179 return fmt.Errorf("empty HTTP response") 180 } 181 182 // return the http status if it is unable to get response body. 183 defer resp.Body.Close() 184 respBody, _ := ioutil.ReadAll(resp.Body) 185 resp.Body = ioutil.NopCloser(bytes.NewReader(respBody)) 186 if len(respBody) == 0 { 187 return fmt.Errorf("HTTP status code (%d)", resp.StatusCode) 188 } 189 190 // return the raw response body. 191 return fmt.Errorf("%s", string(respBody)) 192} 193 194func getHTTPStatusCode(resp *http.Response) int { 195 if resp == nil { 196 return -1 197 } 198 199 return resp.StatusCode 200} 201 202// shouldRetryHTTPRequest determines if the request is retriable. 203func shouldRetryHTTPRequest(resp *http.Response, err error) bool { 204 if resp != nil { 205 for _, code := range StatusCodesForRetry { 206 if resp.StatusCode == code { 207 return true 208 } 209 } 210 211 // should retry on <200, error>. 212 if isSuccessHTTPResponse(resp) && err != nil { 213 return true 214 } 215 216 return false 217 } 218 219 // should retry when error is not nil and no http.Response. 220 if err != nil { 221 return true 222 } 223 224 return false 225} 226 227// getRetryAfter gets the retryAfter from http response. 228// The value of Retry-After can be either the number of seconds or a date in RFC1123 format. 229func getRetryAfter(resp *http.Response) time.Duration { 230 if resp == nil { 231 return 0 232 } 233 234 ra := resp.Header.Get(RetryAfterHeaderKey) 235 if ra == "" { 236 return 0 237 } 238 239 var dur time.Duration 240 if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 { 241 dur = time.Duration(retryAfter) * time.Second 242 } else if t, err := time.Parse(time.RFC1123, ra); err == nil { 243 dur = t.Sub(now()) 244 } 245 return dur 246} 247 248// GetErrorWithRetriableHTTPStatusCodes gets an error with RetriableHTTPStatusCodes. 249// It is used to retry on some HTTPStatusCodes. 250func GetErrorWithRetriableHTTPStatusCodes(resp *http.Response, err error, retriableHTTPStatusCodes []int) *Error { 251 rerr := GetError(resp, err) 252 if rerr == nil { 253 return nil 254 } 255 256 for _, code := range retriableHTTPStatusCodes { 257 if rerr.HTTPStatusCode == code { 258 rerr.Retriable = true 259 break 260 } 261 } 262 263 return rerr 264} 265 266// GetStatusNotFoundAndForbiddenIgnoredError gets an error with StatusNotFound and StatusForbidden ignored. 267// It is only used in DELETE operations. 268func GetStatusNotFoundAndForbiddenIgnoredError(resp *http.Response, err error) *Error { 269 rerr := GetError(resp, err) 270 if rerr == nil { 271 return nil 272 } 273 274 // Returns nil when it is StatusNotFound error. 275 if rerr.HTTPStatusCode == http.StatusNotFound { 276 klog.V(3).Infof("Ignoring StatusNotFound error: %v", rerr) 277 return nil 278 } 279 280 // Returns nil if the status code is StatusForbidden. 281 // This happens when AuthorizationFailed is reported from Azure API. 282 if rerr.HTTPStatusCode == http.StatusForbidden { 283 klog.V(3).Infof("Ignoring StatusForbidden error: %v", rerr) 284 return nil 285 } 286 287 return rerr 288} 289 290// IsErrorRetriable returns true if the error is retriable. 291func IsErrorRetriable(err error) bool { 292 if err == nil { 293 return false 294 } 295 296 return strings.Contains(err.Error(), "Retriable: true") 297} 298 299// HasStatusForbiddenOrIgnoredError return true if the given error code is part of the error message 300// This should only be used when trying to delete resources 301func HasStatusForbiddenOrIgnoredError(err error) bool { 302 if err == nil { 303 return false 304 } 305 306 if strings.Contains(err.Error(), fmt.Sprintf("HTTPStatusCode: %d", http.StatusNotFound)) { 307 return true 308 } 309 310 if strings.Contains(err.Error(), fmt.Sprintf("HTTPStatusCode: %d", http.StatusForbidden)) { 311 return true 312 } 313 return false 314} 315