1package request 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "reflect" 10 "strings" 11 "time" 12 13 "github.com/aws/aws-sdk-go/aws" 14 "github.com/aws/aws-sdk-go/aws/awserr" 15 "github.com/aws/aws-sdk-go/aws/client/metadata" 16 "github.com/aws/aws-sdk-go/internal/sdkio" 17) 18 19const ( 20 // ErrCodeSerialization is the serialization error code that is received 21 // during protocol unmarshaling. 22 ErrCodeSerialization = "SerializationError" 23 24 // ErrCodeRead is an error that is returned during HTTP reads. 25 ErrCodeRead = "ReadError" 26 27 // ErrCodeResponseTimeout is the connection timeout error that is received 28 // during body reads. 29 ErrCodeResponseTimeout = "ResponseTimeout" 30 31 // ErrCodeInvalidPresignExpire is returned when the expire time provided to 32 // presign is invalid 33 ErrCodeInvalidPresignExpire = "InvalidPresignExpireError" 34 35 // CanceledErrorCode is the error code that will be returned by an 36 // API request that was canceled. Requests given a aws.Context may 37 // return this error when canceled. 38 CanceledErrorCode = "RequestCanceled" 39) 40 41// A Request is the service request to be made. 42type Request struct { 43 Config aws.Config 44 ClientInfo metadata.ClientInfo 45 Handlers Handlers 46 47 Retryer 48 AttemptTime time.Time 49 Time time.Time 50 Operation *Operation 51 HTTPRequest *http.Request 52 HTTPResponse *http.Response 53 Body io.ReadSeeker 54 BodyStart int64 // offset from beginning of Body that the request body starts 55 Params interface{} 56 Error error 57 Data interface{} 58 RequestID string 59 RetryCount int 60 Retryable *bool 61 RetryDelay time.Duration 62 NotHoist bool 63 SignedHeaderVals http.Header 64 LastSignedAt time.Time 65 DisableFollowRedirects bool 66 67 // A value greater than 0 instructs the request to be signed as Presigned URL 68 // You should not set this field directly. Instead use Request's 69 // Presign or PresignRequest methods. 70 ExpireTime time.Duration 71 72 context aws.Context 73 74 built bool 75 76 // Need to persist an intermediate body between the input Body and HTTP 77 // request body because the HTTP Client's transport can maintain a reference 78 // to the HTTP request's body after the client has returned. This value is 79 // safe to use concurrently and wrap the input Body for each HTTP request. 80 safeBody *offsetReader 81} 82 83// An Operation is the service API operation to be made. 84type Operation struct { 85 Name string 86 HTTPMethod string 87 HTTPPath string 88 *Paginator 89 90 BeforePresignFn func(r *Request) error 91} 92 93// New returns a new Request pointer for the service API 94// operation and parameters. 95// 96// Params is any value of input parameters to be the request payload. 97// Data is pointer value to an object which the request's response 98// payload will be deserialized to. 99func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, 100 retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request { 101 102 method := operation.HTTPMethod 103 if method == "" { 104 method = "POST" 105 } 106 107 httpReq, _ := http.NewRequest(method, "", nil) 108 109 var err error 110 httpReq.URL, err = url.Parse(clientInfo.Endpoint + operation.HTTPPath) 111 if err != nil { 112 httpReq.URL = &url.URL{} 113 err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err) 114 } 115 116 SanitizeHostForHeader(httpReq) 117 118 r := &Request{ 119 Config: cfg, 120 ClientInfo: clientInfo, 121 Handlers: handlers.Copy(), 122 123 Retryer: retryer, 124 Time: time.Now(), 125 ExpireTime: 0, 126 Operation: operation, 127 HTTPRequest: httpReq, 128 Body: nil, 129 Params: params, 130 Error: err, 131 Data: data, 132 } 133 r.SetBufferBody([]byte{}) 134 135 return r 136} 137 138// A Option is a functional option that can augment or modify a request when 139// using a WithContext API operation method. 140type Option func(*Request) 141 142// WithGetResponseHeader builds a request Option which will retrieve a single 143// header value from the HTTP Response. If there are multiple values for the 144// header key use WithGetResponseHeaders instead to access the http.Header 145// map directly. The passed in val pointer must be non-nil. 146// 147// This Option can be used multiple times with a single API operation. 148// 149// var id2, versionID string 150// svc.PutObjectWithContext(ctx, params, 151// request.WithGetResponseHeader("x-amz-id-2", &id2), 152// request.WithGetResponseHeader("x-amz-version-id", &versionID), 153// ) 154func WithGetResponseHeader(key string, val *string) Option { 155 return func(r *Request) { 156 r.Handlers.Complete.PushBack(func(req *Request) { 157 *val = req.HTTPResponse.Header.Get(key) 158 }) 159 } 160} 161 162// WithGetResponseHeaders builds a request Option which will retrieve the 163// headers from the HTTP response and assign them to the passed in headers 164// variable. The passed in headers pointer must be non-nil. 165// 166// var headers http.Header 167// svc.PutObjectWithContext(ctx, params, request.WithGetResponseHeaders(&headers)) 168func WithGetResponseHeaders(headers *http.Header) Option { 169 return func(r *Request) { 170 r.Handlers.Complete.PushBack(func(req *Request) { 171 *headers = req.HTTPResponse.Header 172 }) 173 } 174} 175 176// WithLogLevel is a request option that will set the request to use a specific 177// log level when the request is made. 178// 179// svc.PutObjectWithContext(ctx, params, request.WithLogLevel(aws.LogDebugWithHTTPBody) 180func WithLogLevel(l aws.LogLevelType) Option { 181 return func(r *Request) { 182 r.Config.LogLevel = aws.LogLevel(l) 183 } 184} 185 186// ApplyOptions will apply each option to the request calling them in the order 187// the were provided. 188func (r *Request) ApplyOptions(opts ...Option) { 189 for _, opt := range opts { 190 opt(r) 191 } 192} 193 194// Context will always returns a non-nil context. If Request does not have a 195// context aws.BackgroundContext will be returned. 196func (r *Request) Context() aws.Context { 197 if r.context != nil { 198 return r.context 199 } 200 return aws.BackgroundContext() 201} 202 203// SetContext adds a Context to the current request that can be used to cancel 204// a in-flight request. The Context value must not be nil, or this method will 205// panic. 206// 207// Unlike http.Request.WithContext, SetContext does not return a copy of the 208// Request. It is not safe to use use a single Request value for multiple 209// requests. A new Request should be created for each API operation request. 210// 211// Go 1.6 and below: 212// The http.Request's Cancel field will be set to the Done() value of 213// the context. This will overwrite the Cancel field's value. 214// 215// Go 1.7 and above: 216// The http.Request.WithContext will be used to set the context on the underlying 217// http.Request. This will create a shallow copy of the http.Request. The SDK 218// may create sub contexts in the future for nested requests such as retries. 219func (r *Request) SetContext(ctx aws.Context) { 220 if ctx == nil { 221 panic("context cannot be nil") 222 } 223 setRequestContext(r, ctx) 224} 225 226// WillRetry returns if the request's can be retried. 227func (r *Request) WillRetry() bool { 228 if !aws.IsReaderSeekable(r.Body) && r.HTTPRequest.Body != NoBody { 229 return false 230 } 231 return r.Error != nil && aws.BoolValue(r.Retryable) && r.RetryCount < r.MaxRetries() 232} 233 234// ParamsFilled returns if the request's parameters have been populated 235// and the parameters are valid. False is returned if no parameters are 236// provided or invalid. 237func (r *Request) ParamsFilled() bool { 238 return r.Params != nil && reflect.ValueOf(r.Params).Elem().IsValid() 239} 240 241// DataFilled returns true if the request's data for response deserialization 242// target has been set and is a valid. False is returned if data is not 243// set, or is invalid. 244func (r *Request) DataFilled() bool { 245 return r.Data != nil && reflect.ValueOf(r.Data).Elem().IsValid() 246} 247 248// SetBufferBody will set the request's body bytes that will be sent to 249// the service API. 250func (r *Request) SetBufferBody(buf []byte) { 251 r.SetReaderBody(bytes.NewReader(buf)) 252} 253 254// SetStringBody sets the body of the request to be backed by a string. 255func (r *Request) SetStringBody(s string) { 256 r.SetReaderBody(strings.NewReader(s)) 257} 258 259// SetReaderBody will set the request's body reader. 260func (r *Request) SetReaderBody(reader io.ReadSeeker) { 261 r.Body = reader 262 r.BodyStart, _ = reader.Seek(0, sdkio.SeekCurrent) // Get the Bodies current offset. 263 r.ResetBody() 264} 265 266// Presign returns the request's signed URL. Error will be returned 267// if the signing fails. The expire parameter is only used for presigned Amazon 268// S3 API requests. All other AWS services will use a fixed expiration 269// time of 15 minutes. 270// 271// It is invalid to create a presigned URL with a expire duration 0 or less. An 272// error is returned if expire duration is 0 or less. 273func (r *Request) Presign(expire time.Duration) (string, error) { 274 r = r.copy() 275 276 // Presign requires all headers be hoisted. There is no way to retrieve 277 // the signed headers not hoisted without this. Making the presigned URL 278 // useless. 279 r.NotHoist = false 280 281 u, _, err := getPresignedURL(r, expire) 282 return u, err 283} 284 285// PresignRequest behaves just like presign, with the addition of returning a 286// set of headers that were signed. The expire parameter is only used for 287// presigned Amazon S3 API requests. All other AWS services will use a fixed 288// expiration time of 15 minutes. 289// 290// It is invalid to create a presigned URL with a expire duration 0 or less. An 291// error is returned if expire duration is 0 or less. 292// 293// Returns the URL string for the API operation with signature in the query string, 294// and the HTTP headers that were included in the signature. These headers must 295// be included in any HTTP request made with the presigned URL. 296// 297// To prevent hoisting any headers to the query string set NotHoist to true on 298// this Request value prior to calling PresignRequest. 299func (r *Request) PresignRequest(expire time.Duration) (string, http.Header, error) { 300 r = r.copy() 301 return getPresignedURL(r, expire) 302} 303 304// IsPresigned returns true if the request represents a presigned API url. 305func (r *Request) IsPresigned() bool { 306 return r.ExpireTime != 0 307} 308 309func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, error) { 310 if expire <= 0 { 311 return "", nil, awserr.New( 312 ErrCodeInvalidPresignExpire, 313 "presigned URL requires an expire duration greater than 0", 314 nil, 315 ) 316 } 317 318 r.ExpireTime = expire 319 320 if r.Operation.BeforePresignFn != nil { 321 if err := r.Operation.BeforePresignFn(r); err != nil { 322 return "", nil, err 323 } 324 } 325 326 if err := r.Sign(); err != nil { 327 return "", nil, err 328 } 329 330 return r.HTTPRequest.URL.String(), r.SignedHeaderVals, nil 331} 332 333func debugLogReqError(r *Request, stage string, retrying bool, err error) { 334 if !r.Config.LogLevel.Matches(aws.LogDebugWithRequestErrors) { 335 return 336 } 337 338 retryStr := "not retrying" 339 if retrying { 340 retryStr = "will retry" 341 } 342 343 r.Config.Logger.Log(fmt.Sprintf("DEBUG: %s %s/%s failed, %s, error %v", 344 stage, r.ClientInfo.ServiceName, r.Operation.Name, retryStr, err)) 345} 346 347// Build will build the request's object so it can be signed and sent 348// to the service. Build will also validate all the request's parameters. 349// Any additional build Handlers set on this request will be run 350// in the order they were set. 351// 352// The request will only be built once. Multiple calls to build will have 353// no effect. 354// 355// If any Validate or Build errors occur the build will stop and the error 356// which occurred will be returned. 357func (r *Request) Build() error { 358 if !r.built { 359 r.Handlers.Validate.Run(r) 360 if r.Error != nil { 361 debugLogReqError(r, "Validate Request", false, r.Error) 362 return r.Error 363 } 364 r.Handlers.Build.Run(r) 365 if r.Error != nil { 366 debugLogReqError(r, "Build Request", false, r.Error) 367 return r.Error 368 } 369 r.built = true 370 } 371 372 return r.Error 373} 374 375// Sign will sign the request, returning error if errors are encountered. 376// 377// Sign will build the request prior to signing. All Sign Handlers will 378// be executed in the order they were set. 379func (r *Request) Sign() error { 380 r.Build() 381 if r.Error != nil { 382 debugLogReqError(r, "Build Request", false, r.Error) 383 return r.Error 384 } 385 386 r.Handlers.Sign.Run(r) 387 return r.Error 388} 389 390func (r *Request) getNextRequestBody() (io.ReadCloser, error) { 391 if r.safeBody != nil { 392 r.safeBody.Close() 393 } 394 395 r.safeBody = newOffsetReader(r.Body, r.BodyStart) 396 397 // Go 1.8 tightened and clarified the rules code needs to use when building 398 // requests with the http package. Go 1.8 removed the automatic detection 399 // of if the Request.Body was empty, or actually had bytes in it. The SDK 400 // always sets the Request.Body even if it is empty and should not actually 401 // be sent. This is incorrect. 402 // 403 // Go 1.8 did add a http.NoBody value that the SDK can use to tell the http 404 // client that the request really should be sent without a body. The 405 // Request.Body cannot be set to nil, which is preferable, because the 406 // field is exported and could introduce nil pointer dereferences for users 407 // of the SDK if they used that field. 408 // 409 // Related golang/go#18257 410 l, err := aws.SeekerLen(r.Body) 411 if err != nil { 412 return nil, awserr.New(ErrCodeSerialization, "failed to compute request body size", err) 413 } 414 415 var body io.ReadCloser 416 if l == 0 { 417 body = NoBody 418 } else if l > 0 { 419 body = r.safeBody 420 } else { 421 // Hack to prevent sending bodies for methods where the body 422 // should be ignored by the server. Sending bodies on these 423 // methods without an associated ContentLength will cause the 424 // request to socket timeout because the server does not handle 425 // Transfer-Encoding: chunked bodies for these methods. 426 // 427 // This would only happen if a aws.ReaderSeekerCloser was used with 428 // a io.Reader that was not also an io.Seeker, or did not implement 429 // Len() method. 430 switch r.Operation.HTTPMethod { 431 case "GET", "HEAD", "DELETE": 432 body = NoBody 433 default: 434 body = r.safeBody 435 } 436 } 437 438 return body, nil 439} 440 441// GetBody will return an io.ReadSeeker of the Request's underlying 442// input body with a concurrency safe wrapper. 443func (r *Request) GetBody() io.ReadSeeker { 444 return r.safeBody 445} 446 447// Send will send the request, returning error if errors are encountered. 448// 449// Send will sign the request prior to sending. All Send Handlers will 450// be executed in the order they were set. 451// 452// Canceling a request is non-deterministic. If a request has been canceled, 453// then the transport will choose, randomly, one of the state channels during 454// reads or getting the connection. 455// 456// readLoop() and getConn(req *Request, cm connectMethod) 457// https://github.com/golang/go/blob/master/src/net/http/transport.go 458// 459// Send will not close the request.Request's body. 460func (r *Request) Send() error { 461 defer func() { 462 // Regardless of success or failure of the request trigger the Complete 463 // request handlers. 464 r.Handlers.Complete.Run(r) 465 }() 466 467 if err := r.Error; err != nil { 468 return err 469 } 470 471 for { 472 r.Error = nil 473 r.AttemptTime = time.Now() 474 475 if err := r.Sign(); err != nil { 476 debugLogReqError(r, "Sign Request", false, err) 477 return err 478 } 479 480 if err := r.sendRequest(); err == nil { 481 return nil 482 } else if !shouldRetryCancel(r.Error) { 483 return err 484 } else { 485 r.Handlers.Retry.Run(r) 486 r.Handlers.AfterRetry.Run(r) 487 488 if r.Error != nil || !aws.BoolValue(r.Retryable) { 489 return r.Error 490 } 491 492 r.prepareRetry() 493 continue 494 } 495 } 496} 497 498func (r *Request) prepareRetry() { 499 if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { 500 r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", 501 r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) 502 } 503 504 // The previous http.Request will have a reference to the r.Body 505 // and the HTTP Client's Transport may still be reading from 506 // the request's body even though the Client's Do returned. 507 r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) 508 r.ResetBody() 509 510 // Closing response body to ensure that no response body is leaked 511 // between retry attempts. 512 if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { 513 r.HTTPResponse.Body.Close() 514 } 515} 516 517func (r *Request) sendRequest() (sendErr error) { 518 defer r.Handlers.CompleteAttempt.Run(r) 519 520 r.Retryable = nil 521 r.Handlers.Send.Run(r) 522 if r.Error != nil { 523 debugLogReqError(r, "Send Request", r.WillRetry(), r.Error) 524 return r.Error 525 } 526 527 r.Handlers.UnmarshalMeta.Run(r) 528 r.Handlers.ValidateResponse.Run(r) 529 if r.Error != nil { 530 r.Handlers.UnmarshalError.Run(r) 531 debugLogReqError(r, "Validate Response", r.WillRetry(), r.Error) 532 return r.Error 533 } 534 535 r.Handlers.Unmarshal.Run(r) 536 if r.Error != nil { 537 debugLogReqError(r, "Unmarshal Response", r.WillRetry(), r.Error) 538 return r.Error 539 } 540 541 return nil 542} 543 544// copy will copy a request which will allow for local manipulation of the 545// request. 546func (r *Request) copy() *Request { 547 req := &Request{} 548 *req = *r 549 req.Handlers = r.Handlers.Copy() 550 op := *r.Operation 551 req.Operation = &op 552 return req 553} 554 555// AddToUserAgent adds the string to the end of the request's current user agent. 556func AddToUserAgent(r *Request, s string) { 557 curUA := r.HTTPRequest.Header.Get("User-Agent") 558 if len(curUA) > 0 { 559 s = curUA + " " + s 560 } 561 r.HTTPRequest.Header.Set("User-Agent", s) 562} 563 564type temporary interface { 565 Temporary() bool 566} 567 568func shouldRetryCancel(err error) bool { 569 switch err := err.(type) { 570 case awserr.Error: 571 if err.Code() == CanceledErrorCode { 572 return false 573 } 574 return shouldRetryCancel(err.OrigErr()) 575 case *url.Error: 576 if strings.Contains(err.Error(), "connection refused") { 577 // Refused connections should be retried as the service may not yet 578 // be running on the port. Go TCP dial considers refused 579 // connections as not temporary. 580 return true 581 } 582 // *url.Error only implements Temporary after golang 1.6 but since 583 // url.Error only wraps the error: 584 return shouldRetryCancel(err.Err) 585 case temporary: 586 // If the error is temporary, we want to allow continuation of the 587 // retry process 588 return err.Temporary() 589 case nil: 590 // `awserr.Error.OrigErr()` can be nil, meaning there was an error but 591 // because we don't know the cause, it is marked as retryable. See 592 // TestRequest4xxUnretryable for an example. 593 return true 594 default: 595 switch err.Error() { 596 case "net/http: request canceled", 597 "net/http: request canceled while waiting for connection": 598 // known 1.5 error case when an http request is cancelled 599 return false 600 } 601 // here we don't know the error; so we allow a retry. 602 return true 603 } 604} 605 606// SanitizeHostForHeader removes default port from host and updates request.Host 607func SanitizeHostForHeader(r *http.Request) { 608 host := getHost(r) 609 port := portOnly(host) 610 if port != "" && isDefaultPort(r.URL.Scheme, port) { 611 r.Host = stripPort(host) 612 } 613} 614 615// Returns host from request 616func getHost(r *http.Request) string { 617 if r.Host != "" { 618 return r.Host 619 } 620 621 return r.URL.Host 622} 623 624// Hostname returns u.Host, without any port number. 625// 626// If Host is an IPv6 literal with a port number, Hostname returns the 627// IPv6 literal without the square brackets. IPv6 literals may include 628// a zone identifier. 629// 630// Copied from the Go 1.8 standard library (net/url) 631func stripPort(hostport string) string { 632 colon := strings.IndexByte(hostport, ':') 633 if colon == -1 { 634 return hostport 635 } 636 if i := strings.IndexByte(hostport, ']'); i != -1 { 637 return strings.TrimPrefix(hostport[:i], "[") 638 } 639 return hostport[:colon] 640} 641 642// Port returns the port part of u.Host, without the leading colon. 643// If u.Host doesn't contain a port, Port returns an empty string. 644// 645// Copied from the Go 1.8 standard library (net/url) 646func portOnly(hostport string) string { 647 colon := strings.IndexByte(hostport, ':') 648 if colon == -1 { 649 return "" 650 } 651 if i := strings.Index(hostport, "]:"); i != -1 { 652 return hostport[i+len("]:"):] 653 } 654 if strings.Contains(hostport, "]") { 655 return "" 656 } 657 return hostport[colon+len(":"):] 658} 659 660// Returns true if the specified URI is using the standard port 661// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs) 662func isDefaultPort(scheme, port string) bool { 663 if port == "" { 664 return true 665 } 666 667 lowerCaseScheme := strings.ToLower(scheme) 668 if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") { 669 return true 670 } 671 672 return false 673} 674