1package request 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "reflect" 10 "strings" 11 "time" 12 13 "github.com/aws/aws-sdk-go/aws" 14 "github.com/aws/aws-sdk-go/aws/awserr" 15 "github.com/aws/aws-sdk-go/aws/client/metadata" 16 "github.com/aws/aws-sdk-go/internal/sdkio" 17) 18 19const ( 20 // ErrCodeSerialization is the serialization error code that is received 21 // during protocol unmarshaling. 22 ErrCodeSerialization = "SerializationError" 23 24 // ErrCodeRead is an error that is returned during HTTP reads. 25 ErrCodeRead = "ReadError" 26 27 // ErrCodeResponseTimeout is the connection timeout error that is received 28 // during body reads. 29 ErrCodeResponseTimeout = "ResponseTimeout" 30 31 // ErrCodeInvalidPresignExpire is returned when the expire time provided to 32 // presign is invalid 33 ErrCodeInvalidPresignExpire = "InvalidPresignExpireError" 34 35 // CanceledErrorCode is the error code that will be returned by an 36 // API request that was canceled. Requests given a aws.Context may 37 // return this error when canceled. 38 CanceledErrorCode = "RequestCanceled" 39 40 // ErrCodeRequestError is an error preventing the SDK from continuing to 41 // process the request. 42 ErrCodeRequestError = "RequestError" 43) 44 45// A Request is the service request to be made. 46type Request struct { 47 Config aws.Config 48 ClientInfo metadata.ClientInfo 49 Handlers Handlers 50 51 Retryer 52 AttemptTime time.Time 53 Time time.Time 54 Operation *Operation 55 HTTPRequest *http.Request 56 HTTPResponse *http.Response 57 Body io.ReadSeeker 58 streamingBody io.ReadCloser 59 BodyStart int64 // offset from beginning of Body that the request body starts 60 Params interface{} 61 Error error 62 Data interface{} 63 RequestID string 64 RetryCount int 65 Retryable *bool 66 RetryDelay time.Duration 67 NotHoist bool 68 SignedHeaderVals http.Header 69 LastSignedAt time.Time 70 DisableFollowRedirects bool 71 72 // Additional API error codes that should be retried. IsErrorRetryable 73 // will consider these codes in addition to its built in cases. 74 RetryErrorCodes []string 75 76 // Additional API error codes that should be retried with throttle backoff 77 // delay. IsErrorThrottle will consider these codes in addition to its 78 // built in cases. 79 ThrottleErrorCodes []string 80 81 // A value greater than 0 instructs the request to be signed as Presigned URL 82 // You should not set this field directly. Instead use Request's 83 // Presign or PresignRequest methods. 84 ExpireTime time.Duration 85 86 context aws.Context 87 88 built bool 89 90 // Need to persist an intermediate body between the input Body and HTTP 91 // request body because the HTTP Client's transport can maintain a reference 92 // to the HTTP request's body after the client has returned. This value is 93 // safe to use concurrently and wrap the input Body for each HTTP request. 94 safeBody *offsetReader 95} 96 97// An Operation is the service API operation to be made. 98type Operation struct { 99 Name string 100 HTTPMethod string 101 HTTPPath string 102 *Paginator 103 104 BeforePresignFn func(r *Request) error 105} 106 107// New returns a new Request pointer for the service API operation and 108// parameters. 109// 110// A Retryer should be provided to direct how the request is retried. If 111// Retryer is nil, a default no retry value will be used. You can use 112// NoOpRetryer in the Client package to disable retry behavior directly. 113// 114// Params is any value of input parameters to be the request payload. 115// Data is pointer value to an object which the request's response 116// payload will be deserialized to. 117func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, 118 retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request { 119 120 if retryer == nil { 121 retryer = noOpRetryer{} 122 } 123 124 method := operation.HTTPMethod 125 if method == "" { 126 method = "POST" 127 } 128 129 httpReq, _ := http.NewRequest(method, "", nil) 130 131 var err error 132 httpReq.URL, err = url.Parse(clientInfo.Endpoint + operation.HTTPPath) 133 if err != nil { 134 httpReq.URL = &url.URL{} 135 err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err) 136 } 137 138 r := &Request{ 139 Config: cfg, 140 ClientInfo: clientInfo, 141 Handlers: handlers.Copy(), 142 143 Retryer: retryer, 144 Time: time.Now(), 145 ExpireTime: 0, 146 Operation: operation, 147 HTTPRequest: httpReq, 148 Body: nil, 149 Params: params, 150 Error: err, 151 Data: data, 152 } 153 r.SetBufferBody([]byte{}) 154 155 return r 156} 157 158// A Option is a functional option that can augment or modify a request when 159// using a WithContext API operation method. 160type Option func(*Request) 161 162// WithGetResponseHeader builds a request Option which will retrieve a single 163// header value from the HTTP Response. If there are multiple values for the 164// header key use WithGetResponseHeaders instead to access the http.Header 165// map directly. The passed in val pointer must be non-nil. 166// 167// This Option can be used multiple times with a single API operation. 168// 169// var id2, versionID string 170// svc.PutObjectWithContext(ctx, params, 171// request.WithGetResponseHeader("x-amz-id-2", &id2), 172// request.WithGetResponseHeader("x-amz-version-id", &versionID), 173// ) 174func WithGetResponseHeader(key string, val *string) Option { 175 return func(r *Request) { 176 r.Handlers.Complete.PushBack(func(req *Request) { 177 *val = req.HTTPResponse.Header.Get(key) 178 }) 179 } 180} 181 182// WithGetResponseHeaders builds a request Option which will retrieve the 183// headers from the HTTP response and assign them to the passed in headers 184// variable. The passed in headers pointer must be non-nil. 185// 186// var headers http.Header 187// svc.PutObjectWithContext(ctx, params, request.WithGetResponseHeaders(&headers)) 188func WithGetResponseHeaders(headers *http.Header) Option { 189 return func(r *Request) { 190 r.Handlers.Complete.PushBack(func(req *Request) { 191 *headers = req.HTTPResponse.Header 192 }) 193 } 194} 195 196// WithLogLevel is a request option that will set the request to use a specific 197// log level when the request is made. 198// 199// svc.PutObjectWithContext(ctx, params, request.WithLogLevel(aws.LogDebugWithHTTPBody) 200func WithLogLevel(l aws.LogLevelType) Option { 201 return func(r *Request) { 202 r.Config.LogLevel = aws.LogLevel(l) 203 } 204} 205 206// ApplyOptions will apply each option to the request calling them in the order 207// the were provided. 208func (r *Request) ApplyOptions(opts ...Option) { 209 for _, opt := range opts { 210 opt(r) 211 } 212} 213 214// Context will always returns a non-nil context. If Request does not have a 215// context aws.BackgroundContext will be returned. 216func (r *Request) Context() aws.Context { 217 if r.context != nil { 218 return r.context 219 } 220 return aws.BackgroundContext() 221} 222 223// SetContext adds a Context to the current request that can be used to cancel 224// a in-flight request. The Context value must not be nil, or this method will 225// panic. 226// 227// Unlike http.Request.WithContext, SetContext does not return a copy of the 228// Request. It is not safe to use use a single Request value for multiple 229// requests. A new Request should be created for each API operation request. 230// 231// Go 1.6 and below: 232// The http.Request's Cancel field will be set to the Done() value of 233// the context. This will overwrite the Cancel field's value. 234// 235// Go 1.7 and above: 236// The http.Request.WithContext will be used to set the context on the underlying 237// http.Request. This will create a shallow copy of the http.Request. The SDK 238// may create sub contexts in the future for nested requests such as retries. 239func (r *Request) SetContext(ctx aws.Context) { 240 if ctx == nil { 241 panic("context cannot be nil") 242 } 243 setRequestContext(r, ctx) 244} 245 246// WillRetry returns if the request's can be retried. 247func (r *Request) WillRetry() bool { 248 if !aws.IsReaderSeekable(r.Body) && r.HTTPRequest.Body != NoBody { 249 return false 250 } 251 return r.Error != nil && aws.BoolValue(r.Retryable) && r.RetryCount < r.MaxRetries() 252} 253 254func fmtAttemptCount(retryCount, maxRetries int) string { 255 return fmt.Sprintf("attempt %v/%v", retryCount, maxRetries) 256} 257 258// ParamsFilled returns if the request's parameters have been populated 259// and the parameters are valid. False is returned if no parameters are 260// provided or invalid. 261func (r *Request) ParamsFilled() bool { 262 return r.Params != nil && reflect.ValueOf(r.Params).Elem().IsValid() 263} 264 265// DataFilled returns true if the request's data for response deserialization 266// target has been set and is a valid. False is returned if data is not 267// set, or is invalid. 268func (r *Request) DataFilled() bool { 269 return r.Data != nil && reflect.ValueOf(r.Data).Elem().IsValid() 270} 271 272// SetBufferBody will set the request's body bytes that will be sent to 273// the service API. 274func (r *Request) SetBufferBody(buf []byte) { 275 r.SetReaderBody(bytes.NewReader(buf)) 276} 277 278// SetStringBody sets the body of the request to be backed by a string. 279func (r *Request) SetStringBody(s string) { 280 r.SetReaderBody(strings.NewReader(s)) 281} 282 283// SetReaderBody will set the request's body reader. 284func (r *Request) SetReaderBody(reader io.ReadSeeker) { 285 r.Body = reader 286 287 if aws.IsReaderSeekable(reader) { 288 var err error 289 // Get the Bodies current offset so retries will start from the same 290 // initial position. 291 r.BodyStart, err = reader.Seek(0, sdkio.SeekCurrent) 292 if err != nil { 293 r.Error = awserr.New(ErrCodeSerialization, 294 "failed to determine start of request body", err) 295 return 296 } 297 } 298 r.ResetBody() 299} 300 301// SetStreamingBody set the reader to be used for the request that will stream 302// bytes to the server. Request's Body must not be set to any reader. 303func (r *Request) SetStreamingBody(reader io.ReadCloser) { 304 r.streamingBody = reader 305 r.SetReaderBody(aws.ReadSeekCloser(reader)) 306} 307 308// Presign returns the request's signed URL. Error will be returned 309// if the signing fails. The expire parameter is only used for presigned Amazon 310// S3 API requests. All other AWS services will use a fixed expiration 311// time of 15 minutes. 312// 313// It is invalid to create a presigned URL with a expire duration 0 or less. An 314// error is returned if expire duration is 0 or less. 315func (r *Request) Presign(expire time.Duration) (string, error) { 316 r = r.copy() 317 318 // Presign requires all headers be hoisted. There is no way to retrieve 319 // the signed headers not hoisted without this. Making the presigned URL 320 // useless. 321 r.NotHoist = false 322 323 u, _, err := getPresignedURL(r, expire) 324 return u, err 325} 326 327// PresignRequest behaves just like presign, with the addition of returning a 328// set of headers that were signed. The expire parameter is only used for 329// presigned Amazon S3 API requests. All other AWS services will use a fixed 330// expiration time of 15 minutes. 331// 332// It is invalid to create a presigned URL with a expire duration 0 or less. An 333// error is returned if expire duration is 0 or less. 334// 335// Returns the URL string for the API operation with signature in the query string, 336// and the HTTP headers that were included in the signature. These headers must 337// be included in any HTTP request made with the presigned URL. 338// 339// To prevent hoisting any headers to the query string set NotHoist to true on 340// this Request value prior to calling PresignRequest. 341func (r *Request) PresignRequest(expire time.Duration) (string, http.Header, error) { 342 r = r.copy() 343 return getPresignedURL(r, expire) 344} 345 346// IsPresigned returns true if the request represents a presigned API url. 347func (r *Request) IsPresigned() bool { 348 return r.ExpireTime != 0 349} 350 351func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, error) { 352 if expire <= 0 { 353 return "", nil, awserr.New( 354 ErrCodeInvalidPresignExpire, 355 "presigned URL requires an expire duration greater than 0", 356 nil, 357 ) 358 } 359 360 r.ExpireTime = expire 361 362 if r.Operation.BeforePresignFn != nil { 363 if err := r.Operation.BeforePresignFn(r); err != nil { 364 return "", nil, err 365 } 366 } 367 368 if err := r.Sign(); err != nil { 369 return "", nil, err 370 } 371 372 return r.HTTPRequest.URL.String(), r.SignedHeaderVals, nil 373} 374 375const ( 376 notRetrying = "not retrying" 377) 378 379func debugLogReqError(r *Request, stage, retryStr string, err error) { 380 if !r.Config.LogLevel.Matches(aws.LogDebugWithRequestErrors) { 381 return 382 } 383 384 r.Config.Logger.Log(fmt.Sprintf("DEBUG: %s %s/%s failed, %s, error %v", 385 stage, r.ClientInfo.ServiceName, r.Operation.Name, retryStr, err)) 386} 387 388// Build will build the request's object so it can be signed and sent 389// to the service. Build will also validate all the request's parameters. 390// Any additional build Handlers set on this request will be run 391// in the order they were set. 392// 393// The request will only be built once. Multiple calls to build will have 394// no effect. 395// 396// If any Validate or Build errors occur the build will stop and the error 397// which occurred will be returned. 398func (r *Request) Build() error { 399 if !r.built { 400 r.Handlers.Validate.Run(r) 401 if r.Error != nil { 402 debugLogReqError(r, "Validate Request", notRetrying, r.Error) 403 return r.Error 404 } 405 r.Handlers.Build.Run(r) 406 if r.Error != nil { 407 debugLogReqError(r, "Build Request", notRetrying, r.Error) 408 return r.Error 409 } 410 r.built = true 411 } 412 413 return r.Error 414} 415 416// Sign will sign the request, returning error if errors are encountered. 417// 418// Sign will build the request prior to signing. All Sign Handlers will 419// be executed in the order they were set. 420func (r *Request) Sign() error { 421 r.Build() 422 if r.Error != nil { 423 debugLogReqError(r, "Build Request", notRetrying, r.Error) 424 return r.Error 425 } 426 427 SanitizeHostForHeader(r.HTTPRequest) 428 429 r.Handlers.Sign.Run(r) 430 return r.Error 431} 432 433func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) { 434 if r.streamingBody != nil { 435 return r.streamingBody, nil 436 } 437 438 if r.safeBody != nil { 439 r.safeBody.Close() 440 } 441 442 r.safeBody, err = newOffsetReader(r.Body, r.BodyStart) 443 if err != nil { 444 return nil, awserr.New(ErrCodeSerialization, 445 "failed to get next request body reader", err) 446 } 447 448 // Go 1.8 tightened and clarified the rules code needs to use when building 449 // requests with the http package. Go 1.8 removed the automatic detection 450 // of if the Request.Body was empty, or actually had bytes in it. The SDK 451 // always sets the Request.Body even if it is empty and should not actually 452 // be sent. This is incorrect. 453 // 454 // Go 1.8 did add a http.NoBody value that the SDK can use to tell the http 455 // client that the request really should be sent without a body. The 456 // Request.Body cannot be set to nil, which is preferable, because the 457 // field is exported and could introduce nil pointer dereferences for users 458 // of the SDK if they used that field. 459 // 460 // Related golang/go#18257 461 l, err := aws.SeekerLen(r.Body) 462 if err != nil { 463 return nil, awserr.New(ErrCodeSerialization, 464 "failed to compute request body size", err) 465 } 466 467 if l == 0 { 468 body = NoBody 469 } else if l > 0 { 470 body = r.safeBody 471 } else { 472 // Hack to prevent sending bodies for methods where the body 473 // should be ignored by the server. Sending bodies on these 474 // methods without an associated ContentLength will cause the 475 // request to socket timeout because the server does not handle 476 // Transfer-Encoding: chunked bodies for these methods. 477 // 478 // This would only happen if a aws.ReaderSeekerCloser was used with 479 // a io.Reader that was not also an io.Seeker, or did not implement 480 // Len() method. 481 switch r.Operation.HTTPMethod { 482 case "GET", "HEAD", "DELETE": 483 body = NoBody 484 default: 485 body = r.safeBody 486 } 487 } 488 489 return body, nil 490} 491 492// GetBody will return an io.ReadSeeker of the Request's underlying 493// input body with a concurrency safe wrapper. 494func (r *Request) GetBody() io.ReadSeeker { 495 return r.safeBody 496} 497 498// Send will send the request, returning error if errors are encountered. 499// 500// Send will sign the request prior to sending. All Send Handlers will 501// be executed in the order they were set. 502// 503// Canceling a request is non-deterministic. If a request has been canceled, 504// then the transport will choose, randomly, one of the state channels during 505// reads or getting the connection. 506// 507// readLoop() and getConn(req *Request, cm connectMethod) 508// https://github.com/golang/go/blob/master/src/net/http/transport.go 509// 510// Send will not close the request.Request's body. 511func (r *Request) Send() error { 512 defer func() { 513 // Regardless of success or failure of the request trigger the Complete 514 // request handlers. 515 r.Handlers.Complete.Run(r) 516 }() 517 518 if err := r.Error; err != nil { 519 return err 520 } 521 522 for { 523 r.Error = nil 524 r.AttemptTime = time.Now() 525 526 if err := r.Sign(); err != nil { 527 debugLogReqError(r, "Sign Request", notRetrying, err) 528 return err 529 } 530 531 if err := r.sendRequest(); err == nil { 532 return nil 533 } 534 r.Handlers.Retry.Run(r) 535 r.Handlers.AfterRetry.Run(r) 536 537 if r.Error != nil || !aws.BoolValue(r.Retryable) { 538 return r.Error 539 } 540 541 if err := r.prepareRetry(); err != nil { 542 r.Error = err 543 return err 544 } 545 } 546} 547 548func (r *Request) prepareRetry() error { 549 if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { 550 r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", 551 r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) 552 } 553 554 // The previous http.Request will have a reference to the r.Body 555 // and the HTTP Client's Transport may still be reading from 556 // the request's body even though the Client's Do returned. 557 r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) 558 r.ResetBody() 559 if err := r.Error; err != nil { 560 return awserr.New(ErrCodeSerialization, 561 "failed to prepare body for retry", err) 562 563 } 564 565 // Closing response body to ensure that no response body is leaked 566 // between retry attempts. 567 if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { 568 r.HTTPResponse.Body.Close() 569 } 570 571 return nil 572} 573 574func (r *Request) sendRequest() (sendErr error) { 575 defer r.Handlers.CompleteAttempt.Run(r) 576 577 r.Retryable = nil 578 r.Handlers.Send.Run(r) 579 if r.Error != nil { 580 debugLogReqError(r, "Send Request", 581 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 582 r.Error) 583 return r.Error 584 } 585 586 r.Handlers.UnmarshalMeta.Run(r) 587 r.Handlers.ValidateResponse.Run(r) 588 if r.Error != nil { 589 r.Handlers.UnmarshalError.Run(r) 590 debugLogReqError(r, "Validate Response", 591 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 592 r.Error) 593 return r.Error 594 } 595 596 r.Handlers.Unmarshal.Run(r) 597 if r.Error != nil { 598 debugLogReqError(r, "Unmarshal Response", 599 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 600 r.Error) 601 return r.Error 602 } 603 604 return nil 605} 606 607// copy will copy a request which will allow for local manipulation of the 608// request. 609func (r *Request) copy() *Request { 610 req := &Request{} 611 *req = *r 612 req.Handlers = r.Handlers.Copy() 613 op := *r.Operation 614 req.Operation = &op 615 return req 616} 617 618// AddToUserAgent adds the string to the end of the request's current user agent. 619func AddToUserAgent(r *Request, s string) { 620 curUA := r.HTTPRequest.Header.Get("User-Agent") 621 if len(curUA) > 0 { 622 s = curUA + " " + s 623 } 624 r.HTTPRequest.Header.Set("User-Agent", s) 625} 626 627// SanitizeHostForHeader removes default port from host and updates request.Host 628func SanitizeHostForHeader(r *http.Request) { 629 host := getHost(r) 630 port := portOnly(host) 631 if port != "" && isDefaultPort(r.URL.Scheme, port) { 632 r.Host = stripPort(host) 633 } 634} 635 636// Returns host from request 637func getHost(r *http.Request) string { 638 if r.Host != "" { 639 return r.Host 640 } 641 642 if r.URL == nil { 643 return "" 644 } 645 646 return r.URL.Host 647} 648 649// Hostname returns u.Host, without any port number. 650// 651// If Host is an IPv6 literal with a port number, Hostname returns the 652// IPv6 literal without the square brackets. IPv6 literals may include 653// a zone identifier. 654// 655// Copied from the Go 1.8 standard library (net/url) 656func stripPort(hostport string) string { 657 colon := strings.IndexByte(hostport, ':') 658 if colon == -1 { 659 return hostport 660 } 661 if i := strings.IndexByte(hostport, ']'); i != -1 { 662 return strings.TrimPrefix(hostport[:i], "[") 663 } 664 return hostport[:colon] 665} 666 667// Port returns the port part of u.Host, without the leading colon. 668// If u.Host doesn't contain a port, Port returns an empty string. 669// 670// Copied from the Go 1.8 standard library (net/url) 671func portOnly(hostport string) string { 672 colon := strings.IndexByte(hostport, ':') 673 if colon == -1 { 674 return "" 675 } 676 if i := strings.Index(hostport, "]:"); i != -1 { 677 return hostport[i+len("]:"):] 678 } 679 if strings.Contains(hostport, "]") { 680 return "" 681 } 682 return hostport[colon+len(":"):] 683} 684 685// Returns true if the specified URI is using the standard port 686// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs) 687func isDefaultPort(scheme, port string) bool { 688 if port == "" { 689 return true 690 } 691 692 lowerCaseScheme := strings.ToLower(scheme) 693 if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") { 694 return true 695 } 696 697 return false 698} 699