1package request 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "reflect" 10 "strings" 11 "time" 12 13 "github.com/aws/aws-sdk-go/aws" 14 "github.com/aws/aws-sdk-go/aws/awserr" 15 "github.com/aws/aws-sdk-go/aws/client/metadata" 16 "github.com/aws/aws-sdk-go/internal/sdkio" 17) 18 19const ( 20 // ErrCodeSerialization is the serialization error code that is received 21 // during protocol unmarshaling. 22 ErrCodeSerialization = "SerializationError" 23 24 // ErrCodeRead is an error that is returned during HTTP reads. 25 ErrCodeRead = "ReadError" 26 27 // ErrCodeResponseTimeout is the connection timeout error that is received 28 // during body reads. 29 ErrCodeResponseTimeout = "ResponseTimeout" 30 31 // ErrCodeInvalidPresignExpire is returned when the expire time provided to 32 // presign is invalid 33 ErrCodeInvalidPresignExpire = "InvalidPresignExpireError" 34 35 // CanceledErrorCode is the error code that will be returned by an 36 // API request that was canceled. Requests given a aws.Context may 37 // return this error when canceled. 38 CanceledErrorCode = "RequestCanceled" 39) 40 41// A Request is the service request to be made. 42type Request struct { 43 Config aws.Config 44 ClientInfo metadata.ClientInfo 45 Handlers Handlers 46 47 Retryer 48 AttemptTime time.Time 49 Time time.Time 50 Operation *Operation 51 HTTPRequest *http.Request 52 HTTPResponse *http.Response 53 Body io.ReadSeeker 54 BodyStart int64 // offset from beginning of Body that the request body starts 55 Params interface{} 56 Error error 57 Data interface{} 58 RequestID string 59 RetryCount int 60 Retryable *bool 61 RetryDelay time.Duration 62 NotHoist bool 63 SignedHeaderVals http.Header 64 LastSignedAt time.Time 65 DisableFollowRedirects bool 66 67 // Additional API error codes that should be retried. IsErrorRetryable 68 // will consider these codes in addition to its built in cases. 69 RetryErrorCodes []string 70 71 // Additional API error codes that should be retried with throttle backoff 72 // delay. IsErrorThrottle will consider these codes in addition to its 73 // built in cases. 74 ThrottleErrorCodes []string 75 76 // A value greater than 0 instructs the request to be signed as Presigned URL 77 // You should not set this field directly. Instead use Request's 78 // Presign or PresignRequest methods. 79 ExpireTime time.Duration 80 81 context aws.Context 82 83 built bool 84 85 // Need to persist an intermediate body between the input Body and HTTP 86 // request body because the HTTP Client's transport can maintain a reference 87 // to the HTTP request's body after the client has returned. This value is 88 // safe to use concurrently and wrap the input Body for each HTTP request. 89 safeBody *offsetReader 90} 91 92// An Operation is the service API operation to be made. 93type Operation struct { 94 Name string 95 HTTPMethod string 96 HTTPPath string 97 *Paginator 98 99 BeforePresignFn func(r *Request) error 100} 101 102// New returns a new Request pointer for the service API operation and 103// parameters. 104// 105// A Retryer should be provided to direct how the request is retried. If 106// Retryer is nil, a default no retry value will be used. You can use 107// NoOpRetryer in the Client package to disable retry behavior directly. 108// 109// Params is any value of input parameters to be the request payload. 110// Data is pointer value to an object which the request's response 111// payload will be deserialized to. 112func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, 113 retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request { 114 115 if retryer == nil { 116 retryer = noOpRetryer{} 117 } 118 119 method := operation.HTTPMethod 120 if method == "" { 121 method = "POST" 122 } 123 124 httpReq, _ := http.NewRequest(method, "", nil) 125 126 var err error 127 httpReq.URL, err = url.Parse(clientInfo.Endpoint + operation.HTTPPath) 128 if err != nil { 129 httpReq.URL = &url.URL{} 130 err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err) 131 } 132 133 SanitizeHostForHeader(httpReq) 134 135 r := &Request{ 136 Config: cfg, 137 ClientInfo: clientInfo, 138 Handlers: handlers.Copy(), 139 140 Retryer: retryer, 141 Time: time.Now(), 142 ExpireTime: 0, 143 Operation: operation, 144 HTTPRequest: httpReq, 145 Body: nil, 146 Params: params, 147 Error: err, 148 Data: data, 149 } 150 r.SetBufferBody([]byte{}) 151 152 return r 153} 154 155// A Option is a functional option that can augment or modify a request when 156// using a WithContext API operation method. 157type Option func(*Request) 158 159// WithGetResponseHeader builds a request Option which will retrieve a single 160// header value from the HTTP Response. If there are multiple values for the 161// header key use WithGetResponseHeaders instead to access the http.Header 162// map directly. The passed in val pointer must be non-nil. 163// 164// This Option can be used multiple times with a single API operation. 165// 166// var id2, versionID string 167// svc.PutObjectWithContext(ctx, params, 168// request.WithGetResponseHeader("x-amz-id-2", &id2), 169// request.WithGetResponseHeader("x-amz-version-id", &versionID), 170// ) 171func WithGetResponseHeader(key string, val *string) Option { 172 return func(r *Request) { 173 r.Handlers.Complete.PushBack(func(req *Request) { 174 *val = req.HTTPResponse.Header.Get(key) 175 }) 176 } 177} 178 179// WithGetResponseHeaders builds a request Option which will retrieve the 180// headers from the HTTP response and assign them to the passed in headers 181// variable. The passed in headers pointer must be non-nil. 182// 183// var headers http.Header 184// svc.PutObjectWithContext(ctx, params, request.WithGetResponseHeaders(&headers)) 185func WithGetResponseHeaders(headers *http.Header) Option { 186 return func(r *Request) { 187 r.Handlers.Complete.PushBack(func(req *Request) { 188 *headers = req.HTTPResponse.Header 189 }) 190 } 191} 192 193// WithLogLevel is a request option that will set the request to use a specific 194// log level when the request is made. 195// 196// svc.PutObjectWithContext(ctx, params, request.WithLogLevel(aws.LogDebugWithHTTPBody) 197func WithLogLevel(l aws.LogLevelType) Option { 198 return func(r *Request) { 199 r.Config.LogLevel = aws.LogLevel(l) 200 } 201} 202 203// ApplyOptions will apply each option to the request calling them in the order 204// the were provided. 205func (r *Request) ApplyOptions(opts ...Option) { 206 for _, opt := range opts { 207 opt(r) 208 } 209} 210 211// Context will always returns a non-nil context. If Request does not have a 212// context aws.BackgroundContext will be returned. 213func (r *Request) Context() aws.Context { 214 if r.context != nil { 215 return r.context 216 } 217 return aws.BackgroundContext() 218} 219 220// SetContext adds a Context to the current request that can be used to cancel 221// a in-flight request. The Context value must not be nil, or this method will 222// panic. 223// 224// Unlike http.Request.WithContext, SetContext does not return a copy of the 225// Request. It is not safe to use use a single Request value for multiple 226// requests. A new Request should be created for each API operation request. 227// 228// Go 1.6 and below: 229// The http.Request's Cancel field will be set to the Done() value of 230// the context. This will overwrite the Cancel field's value. 231// 232// Go 1.7 and above: 233// The http.Request.WithContext will be used to set the context on the underlying 234// http.Request. This will create a shallow copy of the http.Request. The SDK 235// may create sub contexts in the future for nested requests such as retries. 236func (r *Request) SetContext(ctx aws.Context) { 237 if ctx == nil { 238 panic("context cannot be nil") 239 } 240 setRequestContext(r, ctx) 241} 242 243// WillRetry returns if the request's can be retried. 244func (r *Request) WillRetry() bool { 245 if !aws.IsReaderSeekable(r.Body) && r.HTTPRequest.Body != NoBody { 246 return false 247 } 248 return r.Error != nil && aws.BoolValue(r.Retryable) && r.RetryCount < r.MaxRetries() 249} 250 251func fmtAttemptCount(retryCount, maxRetries int) string { 252 return fmt.Sprintf("attempt %v/%v", retryCount, maxRetries) 253} 254 255// ParamsFilled returns if the request's parameters have been populated 256// and the parameters are valid. False is returned if no parameters are 257// provided or invalid. 258func (r *Request) ParamsFilled() bool { 259 return r.Params != nil && reflect.ValueOf(r.Params).Elem().IsValid() 260} 261 262// DataFilled returns true if the request's data for response deserialization 263// target has been set and is a valid. False is returned if data is not 264// set, or is invalid. 265func (r *Request) DataFilled() bool { 266 return r.Data != nil && reflect.ValueOf(r.Data).Elem().IsValid() 267} 268 269// SetBufferBody will set the request's body bytes that will be sent to 270// the service API. 271func (r *Request) SetBufferBody(buf []byte) { 272 r.SetReaderBody(bytes.NewReader(buf)) 273} 274 275// SetStringBody sets the body of the request to be backed by a string. 276func (r *Request) SetStringBody(s string) { 277 r.SetReaderBody(strings.NewReader(s)) 278} 279 280// SetReaderBody will set the request's body reader. 281func (r *Request) SetReaderBody(reader io.ReadSeeker) { 282 r.Body = reader 283 284 if aws.IsReaderSeekable(reader) { 285 var err error 286 // Get the Bodies current offset so retries will start from the same 287 // initial position. 288 r.BodyStart, err = reader.Seek(0, sdkio.SeekCurrent) 289 if err != nil { 290 r.Error = awserr.New(ErrCodeSerialization, 291 "failed to determine start of request body", err) 292 return 293 } 294 } 295 r.ResetBody() 296} 297 298// Presign returns the request's signed URL. Error will be returned 299// if the signing fails. The expire parameter is only used for presigned Amazon 300// S3 API requests. All other AWS services will use a fixed expiration 301// time of 15 minutes. 302// 303// It is invalid to create a presigned URL with a expire duration 0 or less. An 304// error is returned if expire duration is 0 or less. 305func (r *Request) Presign(expire time.Duration) (string, error) { 306 r = r.copy() 307 308 // Presign requires all headers be hoisted. There is no way to retrieve 309 // the signed headers not hoisted without this. Making the presigned URL 310 // useless. 311 r.NotHoist = false 312 313 u, _, err := getPresignedURL(r, expire) 314 return u, err 315} 316 317// PresignRequest behaves just like presign, with the addition of returning a 318// set of headers that were signed. The expire parameter is only used for 319// presigned Amazon S3 API requests. All other AWS services will use a fixed 320// expiration time of 15 minutes. 321// 322// It is invalid to create a presigned URL with a expire duration 0 or less. An 323// error is returned if expire duration is 0 or less. 324// 325// Returns the URL string for the API operation with signature in the query string, 326// and the HTTP headers that were included in the signature. These headers must 327// be included in any HTTP request made with the presigned URL. 328// 329// To prevent hoisting any headers to the query string set NotHoist to true on 330// this Request value prior to calling PresignRequest. 331func (r *Request) PresignRequest(expire time.Duration) (string, http.Header, error) { 332 r = r.copy() 333 return getPresignedURL(r, expire) 334} 335 336// IsPresigned returns true if the request represents a presigned API url. 337func (r *Request) IsPresigned() bool { 338 return r.ExpireTime != 0 339} 340 341func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, error) { 342 if expire <= 0 { 343 return "", nil, awserr.New( 344 ErrCodeInvalidPresignExpire, 345 "presigned URL requires an expire duration greater than 0", 346 nil, 347 ) 348 } 349 350 r.ExpireTime = expire 351 352 if r.Operation.BeforePresignFn != nil { 353 if err := r.Operation.BeforePresignFn(r); err != nil { 354 return "", nil, err 355 } 356 } 357 358 if err := r.Sign(); err != nil { 359 return "", nil, err 360 } 361 362 return r.HTTPRequest.URL.String(), r.SignedHeaderVals, nil 363} 364 365const ( 366 notRetrying = "not retrying" 367) 368 369func debugLogReqError(r *Request, stage, retryStr string, err error) { 370 if !r.Config.LogLevel.Matches(aws.LogDebugWithRequestErrors) { 371 return 372 } 373 374 r.Config.Logger.Log(fmt.Sprintf("DEBUG: %s %s/%s failed, %s, error %v", 375 stage, r.ClientInfo.ServiceName, r.Operation.Name, retryStr, err)) 376} 377 378// Build will build the request's object so it can be signed and sent 379// to the service. Build will also validate all the request's parameters. 380// Any additional build Handlers set on this request will be run 381// in the order they were set. 382// 383// The request will only be built once. Multiple calls to build will have 384// no effect. 385// 386// If any Validate or Build errors occur the build will stop and the error 387// which occurred will be returned. 388func (r *Request) Build() error { 389 if !r.built { 390 r.Handlers.Validate.Run(r) 391 if r.Error != nil { 392 debugLogReqError(r, "Validate Request", notRetrying, r.Error) 393 return r.Error 394 } 395 r.Handlers.Build.Run(r) 396 if r.Error != nil { 397 debugLogReqError(r, "Build Request", notRetrying, r.Error) 398 return r.Error 399 } 400 r.built = true 401 } 402 403 return r.Error 404} 405 406// Sign will sign the request, returning error if errors are encountered. 407// 408// Sign will build the request prior to signing. All Sign Handlers will 409// be executed in the order they were set. 410func (r *Request) Sign() error { 411 r.Build() 412 if r.Error != nil { 413 debugLogReqError(r, "Build Request", notRetrying, r.Error) 414 return r.Error 415 } 416 417 r.Handlers.Sign.Run(r) 418 return r.Error 419} 420 421func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) { 422 if r.safeBody != nil { 423 r.safeBody.Close() 424 } 425 426 r.safeBody, err = newOffsetReader(r.Body, r.BodyStart) 427 if err != nil { 428 return nil, awserr.New(ErrCodeSerialization, 429 "failed to get next request body reader", err) 430 } 431 432 // Go 1.8 tightened and clarified the rules code needs to use when building 433 // requests with the http package. Go 1.8 removed the automatic detection 434 // of if the Request.Body was empty, or actually had bytes in it. The SDK 435 // always sets the Request.Body even if it is empty and should not actually 436 // be sent. This is incorrect. 437 // 438 // Go 1.8 did add a http.NoBody value that the SDK can use to tell the http 439 // client that the request really should be sent without a body. The 440 // Request.Body cannot be set to nil, which is preferable, because the 441 // field is exported and could introduce nil pointer dereferences for users 442 // of the SDK if they used that field. 443 // 444 // Related golang/go#18257 445 l, err := aws.SeekerLen(r.Body) 446 if err != nil { 447 return nil, awserr.New(ErrCodeSerialization, 448 "failed to compute request body size", err) 449 } 450 451 if l == 0 { 452 body = NoBody 453 } else if l > 0 { 454 body = r.safeBody 455 } else { 456 // Hack to prevent sending bodies for methods where the body 457 // should be ignored by the server. Sending bodies on these 458 // methods without an associated ContentLength will cause the 459 // request to socket timeout because the server does not handle 460 // Transfer-Encoding: chunked bodies for these methods. 461 // 462 // This would only happen if a aws.ReaderSeekerCloser was used with 463 // a io.Reader that was not also an io.Seeker, or did not implement 464 // Len() method. 465 switch r.Operation.HTTPMethod { 466 case "GET", "HEAD", "DELETE": 467 body = NoBody 468 default: 469 body = r.safeBody 470 } 471 } 472 473 return body, nil 474} 475 476// GetBody will return an io.ReadSeeker of the Request's underlying 477// input body with a concurrency safe wrapper. 478func (r *Request) GetBody() io.ReadSeeker { 479 return r.safeBody 480} 481 482// Send will send the request, returning error if errors are encountered. 483// 484// Send will sign the request prior to sending. All Send Handlers will 485// be executed in the order they were set. 486// 487// Canceling a request is non-deterministic. If a request has been canceled, 488// then the transport will choose, randomly, one of the state channels during 489// reads or getting the connection. 490// 491// readLoop() and getConn(req *Request, cm connectMethod) 492// https://github.com/golang/go/blob/master/src/net/http/transport.go 493// 494// Send will not close the request.Request's body. 495func (r *Request) Send() error { 496 defer func() { 497 // Regardless of success or failure of the request trigger the Complete 498 // request handlers. 499 r.Handlers.Complete.Run(r) 500 }() 501 502 if err := r.Error; err != nil { 503 return err 504 } 505 506 for { 507 r.Error = nil 508 r.AttemptTime = time.Now() 509 510 if err := r.Sign(); err != nil { 511 debugLogReqError(r, "Sign Request", notRetrying, err) 512 return err 513 } 514 515 if err := r.sendRequest(); err == nil { 516 return nil 517 } 518 r.Handlers.Retry.Run(r) 519 r.Handlers.AfterRetry.Run(r) 520 521 if r.Error != nil || !aws.BoolValue(r.Retryable) { 522 return r.Error 523 } 524 525 if err := r.prepareRetry(); err != nil { 526 r.Error = err 527 return err 528 } 529 } 530} 531 532func (r *Request) prepareRetry() error { 533 if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { 534 r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", 535 r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) 536 } 537 538 // The previous http.Request will have a reference to the r.Body 539 // and the HTTP Client's Transport may still be reading from 540 // the request's body even though the Client's Do returned. 541 r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) 542 r.ResetBody() 543 if err := r.Error; err != nil { 544 return awserr.New(ErrCodeSerialization, 545 "failed to prepare body for retry", err) 546 547 } 548 549 // Closing response body to ensure that no response body is leaked 550 // between retry attempts. 551 if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { 552 r.HTTPResponse.Body.Close() 553 } 554 555 return nil 556} 557 558func (r *Request) sendRequest() (sendErr error) { 559 defer r.Handlers.CompleteAttempt.Run(r) 560 561 r.Retryable = nil 562 r.Handlers.Send.Run(r) 563 if r.Error != nil { 564 debugLogReqError(r, "Send Request", 565 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 566 r.Error) 567 return r.Error 568 } 569 570 r.Handlers.UnmarshalMeta.Run(r) 571 r.Handlers.ValidateResponse.Run(r) 572 if r.Error != nil { 573 r.Handlers.UnmarshalError.Run(r) 574 debugLogReqError(r, "Validate Response", 575 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 576 r.Error) 577 return r.Error 578 } 579 580 r.Handlers.Unmarshal.Run(r) 581 if r.Error != nil { 582 debugLogReqError(r, "Unmarshal Response", 583 fmtAttemptCount(r.RetryCount, r.MaxRetries()), 584 r.Error) 585 return r.Error 586 } 587 588 return nil 589} 590 591// copy will copy a request which will allow for local manipulation of the 592// request. 593func (r *Request) copy() *Request { 594 req := &Request{} 595 *req = *r 596 req.Handlers = r.Handlers.Copy() 597 op := *r.Operation 598 req.Operation = &op 599 return req 600} 601 602// AddToUserAgent adds the string to the end of the request's current user agent. 603func AddToUserAgent(r *Request, s string) { 604 curUA := r.HTTPRequest.Header.Get("User-Agent") 605 if len(curUA) > 0 { 606 s = curUA + " " + s 607 } 608 r.HTTPRequest.Header.Set("User-Agent", s) 609} 610 611// SanitizeHostForHeader removes default port from host and updates request.Host 612func SanitizeHostForHeader(r *http.Request) { 613 host := getHost(r) 614 port := portOnly(host) 615 if port != "" && isDefaultPort(r.URL.Scheme, port) { 616 r.Host = stripPort(host) 617 } 618} 619 620// Returns host from request 621func getHost(r *http.Request) string { 622 if r.Host != "" { 623 return r.Host 624 } 625 626 return r.URL.Host 627} 628 629// Hostname returns u.Host, without any port number. 630// 631// If Host is an IPv6 literal with a port number, Hostname returns the 632// IPv6 literal without the square brackets. IPv6 literals may include 633// a zone identifier. 634// 635// Copied from the Go 1.8 standard library (net/url) 636func stripPort(hostport string) string { 637 colon := strings.IndexByte(hostport, ':') 638 if colon == -1 { 639 return hostport 640 } 641 if i := strings.IndexByte(hostport, ']'); i != -1 { 642 return strings.TrimPrefix(hostport[:i], "[") 643 } 644 return hostport[:colon] 645} 646 647// Port returns the port part of u.Host, without the leading colon. 648// If u.Host doesn't contain a port, Port returns an empty string. 649// 650// Copied from the Go 1.8 standard library (net/url) 651func portOnly(hostport string) string { 652 colon := strings.IndexByte(hostport, ':') 653 if colon == -1 { 654 return "" 655 } 656 if i := strings.Index(hostport, "]:"); i != -1 { 657 return hostport[i+len("]:"):] 658 } 659 if strings.Contains(hostport, "]") { 660 return "" 661 } 662 return hostport[colon+len(":"):] 663} 664 665// Returns true if the specified URI is using the standard port 666// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs) 667func isDefaultPort(scheme, port string) bool { 668 if port == "" { 669 return true 670 } 671 672 lowerCaseScheme := strings.ToLower(scheme) 673 if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") { 674 return true 675 } 676 677 return false 678} 679