1/* 2 * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 * Copyright 2015-2018 Minio, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package minio 19 20import ( 21 "bytes" 22 "context" 23 "crypto/md5" 24 "crypto/sha256" 25 "errors" 26 "fmt" 27 "hash" 28 "io" 29 "io/ioutil" 30 "math/rand" 31 "net" 32 "net/http" 33 "net/http/cookiejar" 34 "net/http/httputil" 35 "net/url" 36 "os" 37 "runtime" 38 "strings" 39 "sync" 40 "time" 41 42 "golang.org/x/net/publicsuffix" 43 44 "github.com/minio/minio-go/pkg/credentials" 45 "github.com/minio/minio-go/pkg/s3signer" 46 "github.com/minio/minio-go/pkg/s3utils" 47) 48 49// Client implements Amazon S3 compatible methods. 50type Client struct { 51 /// Standard options. 52 53 // Parsed endpoint url provided by the user. 54 endpointURL *url.URL 55 56 // Holds various credential providers. 57 credsProvider *credentials.Credentials 58 59 // Custom signerType value overrides all credentials. 60 overrideSignerType credentials.SignatureType 61 62 // User supplied. 63 appInfo struct { 64 appName string 65 appVersion string 66 } 67 68 // Indicate whether we are using https or not 69 secure bool 70 71 // Needs allocation. 72 httpClient *http.Client 73 bucketLocCache *bucketLocationCache 74 75 // Advanced functionality. 76 isTraceEnabled bool 77 traceOutput io.Writer 78 79 // S3 specific accelerated endpoint. 80 s3AccelerateEndpoint string 81 82 // Region endpoint 83 region string 84 85 // Random seed. 86 random *rand.Rand 87 88 // lookup indicates type of url lookup supported by server. If not specified, 89 // default to Auto. 90 lookup BucketLookupType 91} 92 93// Options for New method 94type Options struct { 95 Creds *credentials.Credentials 96 Secure bool 97 Region string 98 BucketLookup BucketLookupType 99 // Add future fields here 100} 101 102// Global constants. 103const ( 104 libraryName = "minio-go" 105 libraryVersion = "v6.0.14" 106) 107 108// User Agent should always following the below style. 109// Please open an issue to discuss any new changes here. 110// 111// Minio (OS; ARCH) LIB/VER APP/VER 112const ( 113 libraryUserAgentPrefix = "Minio (" + runtime.GOOS + "; " + runtime.GOARCH + ") " 114 libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion 115) 116 117// BucketLookupType is type of url lookup supported by server. 118type BucketLookupType int 119 120// Different types of url lookup supported by the server.Initialized to BucketLookupAuto 121const ( 122 BucketLookupAuto BucketLookupType = iota 123 BucketLookupDNS 124 BucketLookupPath 125) 126 127// NewV2 - instantiate minio client with Amazon S3 signature version 128// '2' compatibility. 129func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { 130 creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") 131 clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) 132 if err != nil { 133 return nil, err 134 } 135 clnt.overrideSignerType = credentials.SignatureV2 136 return clnt, nil 137} 138 139// NewV4 - instantiate minio client with Amazon S3 signature version 140// '4' compatibility. 141func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { 142 creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") 143 clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) 144 if err != nil { 145 return nil, err 146 } 147 clnt.overrideSignerType = credentials.SignatureV4 148 return clnt, nil 149} 150 151// New - instantiate minio client, adds automatic verification of signature. 152func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { 153 creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") 154 clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) 155 if err != nil { 156 return nil, err 157 } 158 // Google cloud storage should be set to signature V2, force it if not. 159 if s3utils.IsGoogleEndpoint(*clnt.endpointURL) { 160 clnt.overrideSignerType = credentials.SignatureV2 161 } 162 // If Amazon S3 set to signature v4. 163 if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { 164 clnt.overrideSignerType = credentials.SignatureV4 165 } 166 return clnt, nil 167} 168 169// NewWithCredentials - instantiate minio client with credentials provider 170// for retrieving credentials from various credentials provider such as 171// IAM, File, Env etc. 172func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { 173 return privateNew(endpoint, creds, secure, region, BucketLookupAuto) 174} 175 176// NewWithRegion - instantiate minio client, with region configured. Unlike New(), 177// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. 178// Use this function when if your application deals with single region. 179func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { 180 creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") 181 return privateNew(endpoint, creds, secure, region, BucketLookupAuto) 182} 183 184// NewWithOptions - instantiate minio client with options 185func NewWithOptions(endpoint string, opts *Options) (*Client, error) { 186 return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) 187} 188 189// lockedRandSource provides protected rand source, implements rand.Source interface. 190type lockedRandSource struct { 191 lk sync.Mutex 192 src rand.Source 193} 194 195// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. 196func (r *lockedRandSource) Int63() (n int64) { 197 r.lk.Lock() 198 n = r.src.Int63() 199 r.lk.Unlock() 200 return 201} 202 203// Seed uses the provided seed value to initialize the generator to a 204// deterministic state. 205func (r *lockedRandSource) Seed(seed int64) { 206 r.lk.Lock() 207 r.src.Seed(seed) 208 r.lk.Unlock() 209} 210 211// Redirect requests by re signing the request. 212func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { 213 if len(via) >= 5 { 214 return errors.New("stopped after 5 redirects") 215 } 216 if len(via) == 0 { 217 return nil 218 } 219 lastRequest := via[len(via)-1] 220 var reAuth bool 221 for attr, val := range lastRequest.Header { 222 // if hosts do not match do not copy Authorization header 223 if attr == "Authorization" && req.Host != lastRequest.Host { 224 reAuth = true 225 continue 226 } 227 if _, ok := req.Header[attr]; !ok { 228 req.Header[attr] = val 229 } 230 } 231 232 *c.endpointURL = *req.URL 233 234 value, err := c.credsProvider.Get() 235 if err != nil { 236 return err 237 } 238 var ( 239 signerType = value.SignerType 240 accessKeyID = value.AccessKeyID 241 secretAccessKey = value.SecretAccessKey 242 sessionToken = value.SessionToken 243 region = c.region 244 ) 245 246 // Custom signer set then override the behavior. 247 if c.overrideSignerType != credentials.SignatureDefault { 248 signerType = c.overrideSignerType 249 } 250 251 // If signerType returned by credentials helper is anonymous, 252 // then do not sign regardless of signerType override. 253 if value.SignerType == credentials.SignatureAnonymous { 254 signerType = credentials.SignatureAnonymous 255 } 256 257 if reAuth { 258 // Check if there is no region override, if not get it from the URL if possible. 259 if region == "" { 260 region = s3utils.GetRegionFromURL(*c.endpointURL) 261 } 262 switch { 263 case signerType.IsV2(): 264 return errors.New("signature V2 cannot support redirection") 265 case signerType.IsV4(): 266 req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) 267 } 268 } 269 return nil 270} 271 272func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) { 273 // construct endpoint. 274 endpointURL, err := getEndpointURL(endpoint, secure) 275 if err != nil { 276 return nil, err 277 } 278 279 // Initialize cookies to preserve server sent cookies if any and replay 280 // them upon each request. 281 jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) 282 if err != nil { 283 return nil, err 284 } 285 286 // instantiate new Client. 287 clnt := new(Client) 288 289 // Save the credentials. 290 clnt.credsProvider = creds 291 292 // Remember whether we are using https or not 293 clnt.secure = secure 294 295 // Save endpoint URL, user agent for future uses. 296 clnt.endpointURL = endpointURL 297 298 // Instantiate http client and bucket location cache. 299 clnt.httpClient = &http.Client{ 300 Jar: jar, 301 Transport: DefaultTransport, 302 CheckRedirect: clnt.redirectHeaders, 303 } 304 305 // Sets custom region, if region is empty bucket location cache is used automatically. 306 if region == "" { 307 region = s3utils.GetRegionFromURL(*clnt.endpointURL) 308 } 309 clnt.region = region 310 311 // Instantiate bucket location cache. 312 clnt.bucketLocCache = newBucketLocationCache() 313 314 // Introduce a new locked random seed. 315 clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}) 316 317 // Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined 318 // by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints. 319 clnt.lookup = lookup 320 // Return. 321 return clnt, nil 322} 323 324// SetAppInfo - add application details to user agent. 325func (c *Client) SetAppInfo(appName string, appVersion string) { 326 // if app name and version not set, we do not set a new user agent. 327 if appName != "" && appVersion != "" { 328 c.appInfo = struct { 329 appName string 330 appVersion string 331 }{} 332 c.appInfo.appName = appName 333 c.appInfo.appVersion = appVersion 334 } 335} 336 337// SetCustomTransport - set new custom transport. 338func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { 339 // Set this to override default transport 340 // ``http.DefaultTransport``. 341 // 342 // This transport is usually needed for debugging OR to add your 343 // own custom TLS certificates on the client transport, for custom 344 // CA's and certs which are not part of standard certificate 345 // authority follow this example :- 346 // 347 // tr := &http.Transport{ 348 // TLSClientConfig: &tls.Config{RootCAs: pool}, 349 // DisableCompression: true, 350 // } 351 // api.SetCustomTransport(tr) 352 // 353 if c.httpClient != nil { 354 c.httpClient.Transport = customHTTPTransport 355 } 356} 357 358// TraceOn - enable HTTP tracing. 359func (c *Client) TraceOn(outputStream io.Writer) { 360 // if outputStream is nil then default to os.Stdout. 361 if outputStream == nil { 362 outputStream = os.Stdout 363 } 364 // Sets a new output stream. 365 c.traceOutput = outputStream 366 367 // Enable tracing. 368 c.isTraceEnabled = true 369} 370 371// TraceOff - disable HTTP tracing. 372func (c *Client) TraceOff() { 373 // Disable tracing. 374 c.isTraceEnabled = false 375} 376 377// SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your 378// requests. This feature is only specific to S3 for all other endpoints this 379// function does nothing. To read further details on s3 transfer acceleration 380// please vist - 381// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html 382func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { 383 if s3utils.IsAmazonEndpoint(*c.endpointURL) { 384 c.s3AccelerateEndpoint = accelerateEndpoint 385 } 386} 387 388// Hash materials provides relevant initialized hash algo writers 389// based on the expected signature type. 390// 391// - For signature v4 request if the connection is insecure compute only sha256. 392// - For signature v4 request if the connection is secure compute only md5. 393// - For anonymous request compute md5. 394func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { 395 hashSums = make(map[string][]byte) 396 hashAlgos = make(map[string]hash.Hash) 397 if c.overrideSignerType.IsV4() { 398 if c.secure { 399 hashAlgos["md5"] = md5.New() 400 } else { 401 hashAlgos["sha256"] = sha256.New() 402 } 403 } else { 404 if c.overrideSignerType.IsAnonymous() { 405 hashAlgos["md5"] = md5.New() 406 } 407 } 408 return hashAlgos, hashSums 409} 410 411// requestMetadata - is container for all the values to make a request. 412type requestMetadata struct { 413 // If set newRequest presigns the URL. 414 presignURL bool 415 416 // User supplied. 417 bucketName string 418 objectName string 419 queryValues url.Values 420 customHeader http.Header 421 expires int64 422 423 // Generated by our internal code. 424 bucketLocation string 425 contentBody io.Reader 426 contentLength int64 427 contentMD5Base64 string // carries base64 encoded md5sum 428 contentSHA256Hex string // carries hex encoded sha256sum 429} 430 431// dumpHTTP - dump HTTP request and response. 432func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { 433 // Starts http dump. 434 _, err := fmt.Fprintln(c.traceOutput, "---------START-HTTP---------") 435 if err != nil { 436 return err 437 } 438 439 // Filter out Signature field from Authorization header. 440 origAuth := req.Header.Get("Authorization") 441 if origAuth != "" { 442 req.Header.Set("Authorization", redactSignature(origAuth)) 443 } 444 445 // Only display request header. 446 reqTrace, err := httputil.DumpRequestOut(req, false) 447 if err != nil { 448 return err 449 } 450 451 // Write request to trace output. 452 _, err = fmt.Fprint(c.traceOutput, string(reqTrace)) 453 if err != nil { 454 return err 455 } 456 457 // Only display response header. 458 var respTrace []byte 459 460 // For errors we make sure to dump response body as well. 461 if resp.StatusCode != http.StatusOK && 462 resp.StatusCode != http.StatusPartialContent && 463 resp.StatusCode != http.StatusNoContent { 464 respTrace, err = httputil.DumpResponse(resp, true) 465 if err != nil { 466 return err 467 } 468 } else { 469 respTrace, err = httputil.DumpResponse(resp, false) 470 if err != nil { 471 return err 472 } 473 } 474 475 // Write response to trace output. 476 _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) 477 if err != nil { 478 return err 479 } 480 481 // Ends the http dump. 482 _, err = fmt.Fprintln(c.traceOutput, "---------END-HTTP---------") 483 if err != nil { 484 return err 485 } 486 487 // Returns success. 488 return nil 489} 490 491// do - execute http request. 492func (c Client) do(req *http.Request) (*http.Response, error) { 493 resp, err := c.httpClient.Do(req) 494 if err != nil { 495 // Handle this specifically for now until future Golang versions fix this issue properly. 496 if urlErr, ok := err.(*url.Error); ok { 497 if strings.Contains(urlErr.Err.Error(), "EOF") { 498 return nil, &url.Error{ 499 Op: urlErr.Op, 500 URL: urlErr.URL, 501 Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), 502 } 503 } 504 } 505 return nil, err 506 } 507 508 // Response cannot be non-nil, report error if thats the case. 509 if resp == nil { 510 msg := "Response is empty. " + reportIssue 511 return nil, ErrInvalidArgument(msg) 512 } 513 514 // If trace is enabled, dump http request and response. 515 if c.isTraceEnabled { 516 err = c.dumpHTTP(req, resp) 517 if err != nil { 518 return nil, err 519 } 520 } 521 522 return resp, nil 523} 524 525// List of success status. 526var successStatus = []int{ 527 http.StatusOK, 528 http.StatusNoContent, 529 http.StatusPartialContent, 530} 531 532// executeMethod - instantiates a given method, and retries the 533// request upon any error up to maxRetries attempts in a binomially 534// delayed manner using a standard back off algorithm. 535func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { 536 var isRetryable bool // Indicates if request can be retried. 537 var bodySeeker io.Seeker // Extracted seeker from io.Reader. 538 var reqRetry = MaxRetry // Indicates how many times we can retry the request 539 540 if metadata.contentBody != nil { 541 // Check if body is seekable then it is retryable. 542 bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) 543 switch bodySeeker { 544 case os.Stdin, os.Stdout, os.Stderr: 545 isRetryable = false 546 } 547 // Retry only when reader is seekable 548 if !isRetryable { 549 reqRetry = 1 550 } 551 552 // Figure out if the body can be closed - if yes 553 // we will definitely close it upon the function 554 // return. 555 bodyCloser, ok := metadata.contentBody.(io.Closer) 556 if ok { 557 defer bodyCloser.Close() 558 } 559 } 560 561 // Create a done channel to control 'newRetryTimer' go routine. 562 doneCh := make(chan struct{}, 1) 563 564 // Indicate to our routine to exit cleanly upon return. 565 defer close(doneCh) 566 567 // Blank indentifier is kept here on purpose since 'range' without 568 // blank identifiers is only supported since go1.4 569 // https://golang.org/doc/go1.4#forrange. 570 for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { 571 // Retry executes the following function body if request has an 572 // error until maxRetries have been exhausted, retry attempts are 573 // performed after waiting for a given period of time in a 574 // binomial fashion. 575 if isRetryable { 576 // Seek back to beginning for each attempt. 577 if _, err = bodySeeker.Seek(0, 0); err != nil { 578 // If seek failed, no need to retry. 579 return nil, err 580 } 581 } 582 583 // Instantiate a new request. 584 var req *http.Request 585 req, err = c.newRequest(method, metadata) 586 if err != nil { 587 errResponse := ToErrorResponse(err) 588 if isS3CodeRetryable(errResponse.Code) { 589 continue // Retry. 590 } 591 return nil, err 592 } 593 594 // Add context to request 595 req = req.WithContext(ctx) 596 597 // Initiate the request. 598 res, err = c.do(req) 599 if err != nil { 600 // For supported http requests errors verify. 601 if isHTTPReqErrorRetryable(err) { 602 continue // Retry. 603 } 604 // For other errors, return here no need to retry. 605 return nil, err 606 } 607 608 // For any known successful http status, return quickly. 609 for _, httpStatus := range successStatus { 610 if httpStatus == res.StatusCode { 611 return res, nil 612 } 613 } 614 615 // Read the body to be saved later. 616 errBodyBytes, err := ioutil.ReadAll(res.Body) 617 // res.Body should be closed 618 closeResponse(res) 619 if err != nil { 620 return nil, err 621 } 622 623 // Save the body. 624 errBodySeeker := bytes.NewReader(errBodyBytes) 625 res.Body = ioutil.NopCloser(errBodySeeker) 626 627 // For errors verify if its retryable otherwise fail quickly. 628 errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)) 629 630 // Save the body back again. 631 errBodySeeker.Seek(0, 0) // Seek back to starting point. 632 res.Body = ioutil.NopCloser(errBodySeeker) 633 634 // Bucket region if set in error response and the error 635 // code dictates invalid region, we can retry the request 636 // with the new region. 637 // 638 // Additionally we should only retry if bucketLocation and custom 639 // region is empty. 640 if metadata.bucketLocation == "" && c.region == "" { 641 if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { 642 if metadata.bucketName != "" && errResponse.Region != "" { 643 // Gather Cached location only if bucketName is present. 644 if _, cachedLocationError := c.bucketLocCache.Get(metadata.bucketName); cachedLocationError != false { 645 c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) 646 continue // Retry. 647 } 648 } 649 } 650 } 651 652 // Verify if error response code is retryable. 653 if isS3CodeRetryable(errResponse.Code) { 654 continue // Retry. 655 } 656 657 // Verify if http status code is retryable. 658 if isHTTPStatusRetryable(res.StatusCode) { 659 continue // Retry. 660 } 661 662 // For all other cases break out of the retry loop. 663 break 664 } 665 return res, err 666} 667 668// newRequest - instantiate a new HTTP request for a given method. 669func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) { 670 // If no method is supplied default to 'POST'. 671 if method == "" { 672 method = "POST" 673 } 674 675 location := metadata.bucketLocation 676 if location == "" { 677 if metadata.bucketName != "" { 678 // Gather location only if bucketName is present. 679 location, err = c.getBucketLocation(metadata.bucketName) 680 if err != nil { 681 if ToErrorResponse(err).Code != "AccessDenied" { 682 return nil, err 683 } 684 } 685 // Upon AccessDenied error on fetching bucket location, default 686 // to possible locations based on endpoint URL. This can usually 687 // happen when GetBucketLocation() is disabled using IAM policies. 688 } 689 if location == "" { 690 location = getDefaultLocation(*c.endpointURL, c.region) 691 } 692 } 693 694 // Look if target url supports virtual host. 695 isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) 696 697 // Construct a new target URL. 698 targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) 699 if err != nil { 700 return nil, err 701 } 702 703 // Initialize a new HTTP request for the method. 704 req, err = http.NewRequest(method, targetURL.String(), nil) 705 if err != nil { 706 return nil, err 707 } 708 709 // Get credentials from the configured credentials provider. 710 value, err := c.credsProvider.Get() 711 if err != nil { 712 return nil, err 713 } 714 715 var ( 716 signerType = value.SignerType 717 accessKeyID = value.AccessKeyID 718 secretAccessKey = value.SecretAccessKey 719 sessionToken = value.SessionToken 720 ) 721 722 // Custom signer set then override the behavior. 723 if c.overrideSignerType != credentials.SignatureDefault { 724 signerType = c.overrideSignerType 725 } 726 727 // If signerType returned by credentials helper is anonymous, 728 // then do not sign regardless of signerType override. 729 if value.SignerType == credentials.SignatureAnonymous { 730 signerType = credentials.SignatureAnonymous 731 } 732 733 // Generate presign url if needed, return right here. 734 if metadata.expires != 0 && metadata.presignURL { 735 if signerType.IsAnonymous() { 736 return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") 737 } 738 if signerType.IsV2() { 739 // Presign URL with signature v2. 740 req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) 741 } else if signerType.IsV4() { 742 // Presign URL with signature v4. 743 req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) 744 } 745 return req, nil 746 } 747 748 // Set 'User-Agent' header for the request. 749 c.setUserAgent(req) 750 751 // Set all headers. 752 for k, v := range metadata.customHeader { 753 req.Header.Set(k, v[0]) 754 } 755 756 // Go net/http notoriously closes the request body. 757 // - The request Body, if non-nil, will be closed by the underlying Transport, even on errors. 758 // This can cause underlying *os.File seekers to fail, avoid that 759 // by making sure to wrap the closer as a nop. 760 if metadata.contentLength == 0 { 761 req.Body = nil 762 } else { 763 req.Body = ioutil.NopCloser(metadata.contentBody) 764 } 765 766 // Set incoming content-length. 767 req.ContentLength = metadata.contentLength 768 if req.ContentLength <= -1 { 769 // For unknown content length, we upload using transfer-encoding: chunked. 770 req.TransferEncoding = []string{"chunked"} 771 } 772 773 // set md5Sum for content protection. 774 if len(metadata.contentMD5Base64) > 0 { 775 req.Header.Set("Content-Md5", metadata.contentMD5Base64) 776 } 777 778 // For anonymous requests just return. 779 if signerType.IsAnonymous() { 780 return req, nil 781 } 782 783 switch { 784 case signerType.IsV2(): 785 // Add signature version '2' authorization header. 786 req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) 787 case metadata.objectName != "" && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: 788 // Streaming signature is used by default for a PUT object request. Additionally we also 789 // look if the initialized client is secure, if yes then we don't need to perform 790 // streaming signature. 791 req = s3signer.StreamingSignV4(req, accessKeyID, 792 secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) 793 default: 794 // Set sha256 sum for signature calculation only with signature version '4'. 795 shaHeader := unsignedPayload 796 if metadata.contentSHA256Hex != "" { 797 shaHeader = metadata.contentSHA256Hex 798 } 799 req.Header.Set("X-Amz-Content-Sha256", shaHeader) 800 801 // Add signature version '4' authorization header. 802 req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) 803 } 804 805 // Return request. 806 return req, nil 807} 808 809// set User agent. 810func (c Client) setUserAgent(req *http.Request) { 811 req.Header.Set("User-Agent", libraryUserAgent) 812 if c.appInfo.appName != "" && c.appInfo.appVersion != "" { 813 req.Header.Set("User-Agent", libraryUserAgent+" "+c.appInfo.appName+"/"+c.appInfo.appVersion) 814 } 815} 816 817// makeTargetURL make a new target url. 818func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) { 819 host := c.endpointURL.Host 820 // For Amazon S3 endpoint, try to fetch location based endpoint. 821 if s3utils.IsAmazonEndpoint(*c.endpointURL) { 822 if c.s3AccelerateEndpoint != "" && bucketName != "" { 823 // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html 824 // Disable transfer acceleration for non-compliant bucket names. 825 if strings.Contains(bucketName, ".") { 826 return nil, ErrTransferAccelerationBucket(bucketName) 827 } 828 // If transfer acceleration is requested set new host. 829 // For more details about enabling transfer acceleration read here. 830 // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html 831 host = c.s3AccelerateEndpoint 832 } else { 833 // Do not change the host if the endpoint URL is a FIPS S3 endpoint. 834 if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) { 835 // Fetch new host based on the bucket location. 836 host = getS3Endpoint(bucketLocation) 837 } 838 } 839 } 840 841 // Save scheme. 842 scheme := c.endpointURL.Scheme 843 844 // Strip port 80 and 443 so we won't send these ports in Host header. 845 // The reason is that browsers and curl automatically remove :80 and :443 846 // with the generated presigned urls, then a signature mismatch error. 847 if h, p, err := net.SplitHostPort(host); err == nil { 848 if scheme == "http" && p == "80" || scheme == "https" && p == "443" { 849 host = h 850 } 851 } 852 853 urlStr := scheme + "://" + host + "/" 854 // Make URL only if bucketName is available, otherwise use the 855 // endpoint URL. 856 if bucketName != "" { 857 // If endpoint supports virtual host style use that always. 858 // Currently only S3 and Google Cloud Storage would support 859 // virtual host style. 860 if isVirtualHostStyle { 861 urlStr = scheme + "://" + bucketName + "." + host + "/" 862 if objectName != "" { 863 urlStr = urlStr + s3utils.EncodePath(objectName) 864 } 865 } else { 866 // If not fall back to using path style. 867 urlStr = urlStr + bucketName + "/" 868 if objectName != "" { 869 urlStr = urlStr + s3utils.EncodePath(objectName) 870 } 871 } 872 } 873 874 // If there are any query values, add them to the end. 875 if len(queryValues) > 0 { 876 urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) 877 } 878 879 return url.Parse(urlStr) 880} 881 882// returns true if virtual hosted style requests are to be used. 883func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool { 884 if bucketName == "" { 885 return false 886 } 887 888 if c.lookup == BucketLookupDNS { 889 return true 890 } 891 if c.lookup == BucketLookupPath { 892 return false 893 } 894 895 // default to virtual only for Amazon/Google storage. In all other cases use 896 // path style requests 897 return s3utils.IsVirtualHostSupported(url, bucketName) 898} 899