1package request 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "net" 8 "net/http" 9 "net/url" 10 "reflect" 11 "strings" 12 "time" 13 14 "github.com/aws/aws-sdk-go/aws" 15 "github.com/aws/aws-sdk-go/aws/awserr" 16 "github.com/aws/aws-sdk-go/aws/client/metadata" 17 "github.com/aws/aws-sdk-go/internal/sdkio" 18) 19 20const ( 21 // ErrCodeSerialization is the serialization error code that is received 22 // during protocol unmarshaling. 23 ErrCodeSerialization = "SerializationError" 24 25 // ErrCodeRead is an error that is returned during HTTP reads. 26 ErrCodeRead = "ReadError" 27 28 // ErrCodeResponseTimeout is the connection timeout error that is received 29 // during body reads. 30 ErrCodeResponseTimeout = "ResponseTimeout" 31 32 // ErrCodeInvalidPresignExpire is returned when the expire time provided to 33 // presign is invalid 34 ErrCodeInvalidPresignExpire = "InvalidPresignExpireError" 35 36 // CanceledErrorCode is the error code that will be returned by an 37 // API request that was canceled. Requests given a aws.Context may 38 // return this error when canceled. 39 CanceledErrorCode = "RequestCanceled" 40) 41 42// A Request is the service request to be made. 43type Request struct { 44 Config aws.Config 45 ClientInfo metadata.ClientInfo 46 Handlers Handlers 47 48 Retryer 49 AttemptTime time.Time 50 Time time.Time 51 Operation *Operation 52 HTTPRequest *http.Request 53 HTTPResponse *http.Response 54 Body io.ReadSeeker 55 BodyStart int64 // offset from beginning of Body that the request body starts 56 Params interface{} 57 Error error 58 Data interface{} 59 RequestID string 60 RetryCount int 61 Retryable *bool 62 RetryDelay time.Duration 63 NotHoist bool 64 SignedHeaderVals http.Header 65 LastSignedAt time.Time 66 DisableFollowRedirects bool 67 68 // A value greater than 0 instructs the request to be signed as Presigned URL 69 // You should not set this field directly. Instead use Request's 70 // Presign or PresignRequest methods. 71 ExpireTime time.Duration 72 73 context aws.Context 74 75 built bool 76 77 // Need to persist an intermediate body between the input Body and HTTP 78 // request body because the HTTP Client's transport can maintain a reference 79 // to the HTTP request's body after the client has returned. This value is 80 // safe to use concurrently and wrap the input Body for each HTTP request. 81 safeBody *offsetReader 82} 83 84// An Operation is the service API operation to be made. 85type Operation struct { 86 Name string 87 HTTPMethod string 88 HTTPPath string 89 *Paginator 90 91 BeforePresignFn func(r *Request) error 92} 93 94// New returns a new Request pointer for the service API 95// operation and parameters. 96// 97// Params is any value of input parameters to be the request payload. 98// Data is pointer value to an object which the request's response 99// payload will be deserialized to. 100func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, 101 retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request { 102 103 method := operation.HTTPMethod 104 if method == "" { 105 method = "POST" 106 } 107 108 httpReq, _ := http.NewRequest(method, "", nil) 109 110 var err error 111 httpReq.URL, err = url.Parse(clientInfo.Endpoint + operation.HTTPPath) 112 if err != nil { 113 httpReq.URL = &url.URL{} 114 err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err) 115 } 116 117 SanitizeHostForHeader(httpReq) 118 119 r := &Request{ 120 Config: cfg, 121 ClientInfo: clientInfo, 122 Handlers: handlers.Copy(), 123 124 Retryer: retryer, 125 AttemptTime: time.Now(), 126 Time: time.Now(), 127 ExpireTime: 0, 128 Operation: operation, 129 HTTPRequest: httpReq, 130 Body: nil, 131 Params: params, 132 Error: err, 133 Data: data, 134 } 135 r.SetBufferBody([]byte{}) 136 137 return r 138} 139 140// A Option is a functional option that can augment or modify a request when 141// using a WithContext API operation method. 142type Option func(*Request) 143 144// WithGetResponseHeader builds a request Option which will retrieve a single 145// header value from the HTTP Response. If there are multiple values for the 146// header key use WithGetResponseHeaders instead to access the http.Header 147// map directly. The passed in val pointer must be non-nil. 148// 149// This Option can be used multiple times with a single API operation. 150// 151// var id2, versionID string 152// svc.PutObjectWithContext(ctx, params, 153// request.WithGetResponseHeader("x-amz-id-2", &id2), 154// request.WithGetResponseHeader("x-amz-version-id", &versionID), 155// ) 156func WithGetResponseHeader(key string, val *string) Option { 157 return func(r *Request) { 158 r.Handlers.Complete.PushBack(func(req *Request) { 159 *val = req.HTTPResponse.Header.Get(key) 160 }) 161 } 162} 163 164// WithGetResponseHeaders builds a request Option which will retrieve the 165// headers from the HTTP response and assign them to the passed in headers 166// variable. The passed in headers pointer must be non-nil. 167// 168// var headers http.Header 169// svc.PutObjectWithContext(ctx, params, request.WithGetResponseHeaders(&headers)) 170func WithGetResponseHeaders(headers *http.Header) Option { 171 return func(r *Request) { 172 r.Handlers.Complete.PushBack(func(req *Request) { 173 *headers = req.HTTPResponse.Header 174 }) 175 } 176} 177 178// WithLogLevel is a request option that will set the request to use a specific 179// log level when the request is made. 180// 181// svc.PutObjectWithContext(ctx, params, request.WithLogLevel(aws.LogDebugWithHTTPBody) 182func WithLogLevel(l aws.LogLevelType) Option { 183 return func(r *Request) { 184 r.Config.LogLevel = aws.LogLevel(l) 185 } 186} 187 188// ApplyOptions will apply each option to the request calling them in the order 189// the were provided. 190func (r *Request) ApplyOptions(opts ...Option) { 191 for _, opt := range opts { 192 opt(r) 193 } 194} 195 196// Context will always returns a non-nil context. If Request does not have a 197// context aws.BackgroundContext will be returned. 198func (r *Request) Context() aws.Context { 199 if r.context != nil { 200 return r.context 201 } 202 return aws.BackgroundContext() 203} 204 205// SetContext adds a Context to the current request that can be used to cancel 206// a in-flight request. The Context value must not be nil, or this method will 207// panic. 208// 209// Unlike http.Request.WithContext, SetContext does not return a copy of the 210// Request. It is not safe to use use a single Request value for multiple 211// requests. A new Request should be created for each API operation request. 212// 213// Go 1.6 and below: 214// The http.Request's Cancel field will be set to the Done() value of 215// the context. This will overwrite the Cancel field's value. 216// 217// Go 1.7 and above: 218// The http.Request.WithContext will be used to set the context on the underlying 219// http.Request. This will create a shallow copy of the http.Request. The SDK 220// may create sub contexts in the future for nested requests such as retries. 221func (r *Request) SetContext(ctx aws.Context) { 222 if ctx == nil { 223 panic("context cannot be nil") 224 } 225 setRequestContext(r, ctx) 226} 227 228// WillRetry returns if the request's can be retried. 229func (r *Request) WillRetry() bool { 230 if !aws.IsReaderSeekable(r.Body) && r.HTTPRequest.Body != NoBody { 231 return false 232 } 233 return r.Error != nil && aws.BoolValue(r.Retryable) && r.RetryCount < r.MaxRetries() 234} 235 236// ParamsFilled returns if the request's parameters have been populated 237// and the parameters are valid. False is returned if no parameters are 238// provided or invalid. 239func (r *Request) ParamsFilled() bool { 240 return r.Params != nil && reflect.ValueOf(r.Params).Elem().IsValid() 241} 242 243// DataFilled returns true if the request's data for response deserialization 244// target has been set and is a valid. False is returned if data is not 245// set, or is invalid. 246func (r *Request) DataFilled() bool { 247 return r.Data != nil && reflect.ValueOf(r.Data).Elem().IsValid() 248} 249 250// SetBufferBody will set the request's body bytes that will be sent to 251// the service API. 252func (r *Request) SetBufferBody(buf []byte) { 253 r.SetReaderBody(bytes.NewReader(buf)) 254} 255 256// SetStringBody sets the body of the request to be backed by a string. 257func (r *Request) SetStringBody(s string) { 258 r.SetReaderBody(strings.NewReader(s)) 259} 260 261// SetReaderBody will set the request's body reader. 262func (r *Request) SetReaderBody(reader io.ReadSeeker) { 263 r.Body = reader 264 r.BodyStart, _ = reader.Seek(0, sdkio.SeekCurrent) // Get the Bodies current offset. 265 r.ResetBody() 266} 267 268// Presign returns the request's signed URL. Error will be returned 269// if the signing fails. 270// 271// It is invalid to create a presigned URL with a expire duration 0 or less. An 272// error is returned if expire duration is 0 or less. 273func (r *Request) Presign(expire time.Duration) (string, error) { 274 r = r.copy() 275 276 // Presign requires all headers be hoisted. There is no way to retrieve 277 // the signed headers not hoisted without this. Making the presigned URL 278 // useless. 279 r.NotHoist = false 280 281 u, _, err := getPresignedURL(r, expire) 282 return u, err 283} 284 285// PresignRequest behaves just like presign, with the addition of returning a 286// set of headers that were signed. 287// 288// It is invalid to create a presigned URL with a expire duration 0 or less. An 289// error is returned if expire duration is 0 or less. 290// 291// Returns the URL string for the API operation with signature in the query string, 292// and the HTTP headers that were included in the signature. These headers must 293// be included in any HTTP request made with the presigned URL. 294// 295// To prevent hoisting any headers to the query string set NotHoist to true on 296// this Request value prior to calling PresignRequest. 297func (r *Request) PresignRequest(expire time.Duration) (string, http.Header, error) { 298 r = r.copy() 299 return getPresignedURL(r, expire) 300} 301 302// IsPresigned returns true if the request represents a presigned API url. 303func (r *Request) IsPresigned() bool { 304 return r.ExpireTime != 0 305} 306 307func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, error) { 308 if expire <= 0 { 309 return "", nil, awserr.New( 310 ErrCodeInvalidPresignExpire, 311 "presigned URL requires an expire duration greater than 0", 312 nil, 313 ) 314 } 315 316 r.ExpireTime = expire 317 318 if r.Operation.BeforePresignFn != nil { 319 if err := r.Operation.BeforePresignFn(r); err != nil { 320 return "", nil, err 321 } 322 } 323 324 if err := r.Sign(); err != nil { 325 return "", nil, err 326 } 327 328 return r.HTTPRequest.URL.String(), r.SignedHeaderVals, nil 329} 330 331func debugLogReqError(r *Request, stage string, retrying bool, err error) { 332 if !r.Config.LogLevel.Matches(aws.LogDebugWithRequestErrors) { 333 return 334 } 335 336 retryStr := "not retrying" 337 if retrying { 338 retryStr = "will retry" 339 } 340 341 r.Config.Logger.Log(fmt.Sprintf("DEBUG: %s %s/%s failed, %s, error %v", 342 stage, r.ClientInfo.ServiceName, r.Operation.Name, retryStr, err)) 343} 344 345// Build will build the request's object so it can be signed and sent 346// to the service. Build will also validate all the request's parameters. 347// Any additional build Handlers set on this request will be run 348// in the order they were set. 349// 350// The request will only be built once. Multiple calls to build will have 351// no effect. 352// 353// If any Validate or Build errors occur the build will stop and the error 354// which occurred will be returned. 355func (r *Request) Build() error { 356 if !r.built { 357 r.Handlers.Validate.Run(r) 358 if r.Error != nil { 359 debugLogReqError(r, "Validate Request", false, r.Error) 360 return r.Error 361 } 362 r.Handlers.Build.Run(r) 363 if r.Error != nil { 364 debugLogReqError(r, "Build Request", false, r.Error) 365 return r.Error 366 } 367 r.built = true 368 } 369 370 return r.Error 371} 372 373// Sign will sign the request, returning error if errors are encountered. 374// 375// Sign will build the request prior to signing. All Sign Handlers will 376// be executed in the order they were set. 377func (r *Request) Sign() error { 378 r.Build() 379 if r.Error != nil { 380 debugLogReqError(r, "Build Request", false, r.Error) 381 return r.Error 382 } 383 384 r.Handlers.Sign.Run(r) 385 return r.Error 386} 387 388func (r *Request) getNextRequestBody() (io.ReadCloser, error) { 389 if r.safeBody != nil { 390 r.safeBody.Close() 391 } 392 393 r.safeBody = newOffsetReader(r.Body, r.BodyStart) 394 395 // Go 1.8 tightened and clarified the rules code needs to use when building 396 // requests with the http package. Go 1.8 removed the automatic detection 397 // of if the Request.Body was empty, or actually had bytes in it. The SDK 398 // always sets the Request.Body even if it is empty and should not actually 399 // be sent. This is incorrect. 400 // 401 // Go 1.8 did add a http.NoBody value that the SDK can use to tell the http 402 // client that the request really should be sent without a body. The 403 // Request.Body cannot be set to nil, which is preferable, because the 404 // field is exported and could introduce nil pointer dereferences for users 405 // of the SDK if they used that field. 406 // 407 // Related golang/go#18257 408 l, err := aws.SeekerLen(r.Body) 409 if err != nil { 410 return nil, awserr.New(ErrCodeSerialization, "failed to compute request body size", err) 411 } 412 413 var body io.ReadCloser 414 if l == 0 { 415 body = NoBody 416 } else if l > 0 { 417 body = r.safeBody 418 } else { 419 // Hack to prevent sending bodies for methods where the body 420 // should be ignored by the server. Sending bodies on these 421 // methods without an associated ContentLength will cause the 422 // request to socket timeout because the server does not handle 423 // Transfer-Encoding: chunked bodies for these methods. 424 // 425 // This would only happen if a aws.ReaderSeekerCloser was used with 426 // a io.Reader that was not also an io.Seeker, or did not implement 427 // Len() method. 428 switch r.Operation.HTTPMethod { 429 case "GET", "HEAD", "DELETE": 430 body = NoBody 431 default: 432 body = r.safeBody 433 } 434 } 435 436 return body, nil 437} 438 439// GetBody will return an io.ReadSeeker of the Request's underlying 440// input body with a concurrency safe wrapper. 441func (r *Request) GetBody() io.ReadSeeker { 442 return r.safeBody 443} 444 445// Send will send the request, returning error if errors are encountered. 446// 447// Send will sign the request prior to sending. All Send Handlers will 448// be executed in the order they were set. 449// 450// Canceling a request is non-deterministic. If a request has been canceled, 451// then the transport will choose, randomly, one of the state channels during 452// reads or getting the connection. 453// 454// readLoop() and getConn(req *Request, cm connectMethod) 455// https://github.com/golang/go/blob/master/src/net/http/transport.go 456// 457// Send will not close the request.Request's body. 458func (r *Request) Send() error { 459 defer func() { 460 // Regardless of success or failure of the request trigger the Complete 461 // request handlers. 462 r.Handlers.Complete.Run(r) 463 }() 464 465 for { 466 r.AttemptTime = time.Now() 467 if aws.BoolValue(r.Retryable) { 468 if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { 469 r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", 470 r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) 471 } 472 473 // The previous http.Request will have a reference to the r.Body 474 // and the HTTP Client's Transport may still be reading from 475 // the request's body even though the Client's Do returned. 476 r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) 477 r.ResetBody() 478 479 // Closing response body to ensure that no response body is leaked 480 // between retry attempts. 481 if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { 482 r.HTTPResponse.Body.Close() 483 } 484 } 485 486 r.Sign() 487 if r.Error != nil { 488 return r.Error 489 } 490 491 r.Retryable = nil 492 493 r.Handlers.Send.Run(r) 494 if r.Error != nil { 495 if !shouldRetryCancel(r) { 496 return r.Error 497 } 498 499 err := r.Error 500 r.Handlers.Retry.Run(r) 501 r.Handlers.AfterRetry.Run(r) 502 if r.Error != nil { 503 debugLogReqError(r, "Send Request", false, err) 504 return r.Error 505 } 506 debugLogReqError(r, "Send Request", true, err) 507 continue 508 } 509 r.Handlers.UnmarshalMeta.Run(r) 510 r.Handlers.ValidateResponse.Run(r) 511 if r.Error != nil { 512 r.Handlers.UnmarshalError.Run(r) 513 err := r.Error 514 515 r.Handlers.Retry.Run(r) 516 r.Handlers.AfterRetry.Run(r) 517 if r.Error != nil { 518 debugLogReqError(r, "Validate Response", false, err) 519 return r.Error 520 } 521 debugLogReqError(r, "Validate Response", true, err) 522 continue 523 } 524 525 r.Handlers.Unmarshal.Run(r) 526 if r.Error != nil { 527 err := r.Error 528 r.Handlers.Retry.Run(r) 529 r.Handlers.AfterRetry.Run(r) 530 if r.Error != nil { 531 debugLogReqError(r, "Unmarshal Response", false, err) 532 return r.Error 533 } 534 debugLogReqError(r, "Unmarshal Response", true, err) 535 continue 536 } 537 538 break 539 } 540 541 return nil 542} 543 544// copy will copy a request which will allow for local manipulation of the 545// request. 546func (r *Request) copy() *Request { 547 req := &Request{} 548 *req = *r 549 req.Handlers = r.Handlers.Copy() 550 op := *r.Operation 551 req.Operation = &op 552 return req 553} 554 555// AddToUserAgent adds the string to the end of the request's current user agent. 556func AddToUserAgent(r *Request, s string) { 557 curUA := r.HTTPRequest.Header.Get("User-Agent") 558 if len(curUA) > 0 { 559 s = curUA + " " + s 560 } 561 r.HTTPRequest.Header.Set("User-Agent", s) 562} 563 564func shouldRetryCancel(r *Request) bool { 565 awsErr, ok := r.Error.(awserr.Error) 566 timeoutErr := false 567 errStr := r.Error.Error() 568 if ok { 569 if awsErr.Code() == CanceledErrorCode { 570 return false 571 } 572 err := awsErr.OrigErr() 573 netErr, netOK := err.(net.Error) 574 timeoutErr = netOK && netErr.Temporary() 575 if urlErr, ok := err.(*url.Error); !timeoutErr && ok { 576 errStr = urlErr.Err.Error() 577 } 578 } 579 580 // There can be two types of canceled errors here. 581 // The first being a net.Error and the other being an error. 582 // If the request was timed out, we want to continue the retry 583 // process. Otherwise, return the canceled error. 584 return timeoutErr || 585 (errStr != "net/http: request canceled" && 586 errStr != "net/http: request canceled while waiting for connection") 587 588} 589 590// SanitizeHostForHeader removes default port from host and updates request.Host 591func SanitizeHostForHeader(r *http.Request) { 592 host := getHost(r) 593 port := portOnly(host) 594 if port != "" && isDefaultPort(r.URL.Scheme, port) { 595 r.Host = stripPort(host) 596 } 597} 598 599// Returns host from request 600func getHost(r *http.Request) string { 601 if r.Host != "" { 602 return r.Host 603 } 604 605 return r.URL.Host 606} 607 608// Hostname returns u.Host, without any port number. 609// 610// If Host is an IPv6 literal with a port number, Hostname returns the 611// IPv6 literal without the square brackets. IPv6 literals may include 612// a zone identifier. 613// 614// Copied from the Go 1.8 standard library (net/url) 615func stripPort(hostport string) string { 616 colon := strings.IndexByte(hostport, ':') 617 if colon == -1 { 618 return hostport 619 } 620 if i := strings.IndexByte(hostport, ']'); i != -1 { 621 return strings.TrimPrefix(hostport[:i], "[") 622 } 623 return hostport[:colon] 624} 625 626// Port returns the port part of u.Host, without the leading colon. 627// If u.Host doesn't contain a port, Port returns an empty string. 628// 629// Copied from the Go 1.8 standard library (net/url) 630func portOnly(hostport string) string { 631 colon := strings.IndexByte(hostport, ':') 632 if colon == -1 { 633 return "" 634 } 635 if i := strings.Index(hostport, "]:"); i != -1 { 636 return hostport[i+len("]:"):] 637 } 638 if strings.Contains(hostport, "]") { 639 return "" 640 } 641 return hostport[colon+len(":"):] 642} 643 644// Returns true if the specified URI is using the standard port 645// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs) 646func isDefaultPort(scheme, port string) bool { 647 if port == "" { 648 return true 649 } 650 651 lowerCaseScheme := strings.ToLower(scheme) 652 if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") { 653 return true 654 } 655 656 return false 657} 658