1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "context" 20 "crypto" 21 "crypto/rand" 22 "crypto/rsa" 23 "crypto/sha256" 24 "crypto/x509" 25 "encoding/base64" 26 "encoding/hex" 27 "encoding/pem" 28 "errors" 29 "fmt" 30 "net/http" 31 "net/url" 32 "os" 33 "reflect" 34 "regexp" 35 "sort" 36 "strconv" 37 "strings" 38 "time" 39 "unicode/utf8" 40 41 "cloud.google.com/go/internal/optional" 42 "cloud.google.com/go/internal/trace" 43 "cloud.google.com/go/internal/version" 44 "google.golang.org/api/googleapi" 45 "google.golang.org/api/option" 46 raw "google.golang.org/api/storage/v1" 47 htransport "google.golang.org/api/transport/http" 48) 49 50// Methods which can be used in signed URLs. 51var signedURLMethods = map[string]bool{"DELETE": true, "GET": true, "HEAD": true, "POST": true, "PUT": true} 52 53var ( 54 // ErrBucketNotExist indicates that the bucket does not exist. 55 ErrBucketNotExist = errors.New("storage: bucket doesn't exist") 56 // ErrObjectNotExist indicates that the object does not exist. 57 ErrObjectNotExist = errors.New("storage: object doesn't exist") 58 // errMethodNotValid indicates that given HTTP method is not valid. 59 errMethodNotValid = fmt.Errorf("storage: HTTP method should be one of %v", reflect.ValueOf(signedURLMethods).MapKeys()) 60) 61 62var userAgent = fmt.Sprintf("gcloud-golang-storage/%s", version.Repo) 63 64const ( 65 // ScopeFullControl grants permissions to manage your 66 // data and permissions in Google Cloud Storage. 67 ScopeFullControl = raw.DevstorageFullControlScope 68 69 // ScopeReadOnly grants permissions to 70 // view your data in Google Cloud Storage. 71 ScopeReadOnly = raw.DevstorageReadOnlyScope 72 73 // ScopeReadWrite grants permissions to manage your 74 // data in Google Cloud Storage. 75 ScopeReadWrite = raw.DevstorageReadWriteScope 76) 77 78var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) 79 80func setClientHeader(headers http.Header) { 81 headers.Set("x-goog-api-client", xGoogHeader) 82} 83 84// Client is a client for interacting with Google Cloud Storage. 85// 86// Clients should be reused instead of created as needed. 87// The methods of Client are safe for concurrent use by multiple goroutines. 88type Client struct { 89 hc *http.Client 90 raw *raw.Service 91 // Scheme describes the scheme under the current host. 92 scheme string 93 // EnvHost is the host set on the STORAGE_EMULATOR_HOST variable. 94 envHost string 95 // ReadHost is the default host used on the reader. 96 readHost string 97} 98 99// NewClient creates a new Google Cloud Storage client. 100// The default scope is ScopeFullControl. To use a different scope, like ScopeReadOnly, use option.WithScopes. 101func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 102 var host, readHost, scheme string 103 104 if host = os.Getenv("STORAGE_EMULATOR_HOST"); host == "" { 105 scheme = "https" 106 readHost = "storage.googleapis.com" 107 108 // Prepend default options to avoid overriding options passed by the user. 109 opts = append([]option.ClientOption{option.WithScopes(ScopeFullControl), option.WithUserAgent(userAgent)}, opts...) 110 } else { 111 scheme = "http" 112 readHost = host 113 114 opts = append([]option.ClientOption{option.WithoutAuthentication()}, opts...) 115 } 116 117 hc, ep, err := htransport.NewClient(ctx, opts...) 118 if err != nil { 119 return nil, fmt.Errorf("dialing: %v", err) 120 } 121 rawService, err := raw.NewService(ctx, option.WithHTTPClient(hc)) 122 if err != nil { 123 return nil, fmt.Errorf("storage client: %v", err) 124 } 125 if ep == "" { 126 // Override the default value for BasePath from the raw client. 127 // TODO: remove when the raw client uses this endpoint as its default (~end of 2020) 128 rawService.BasePath = "https://storage.googleapis.com/storage/v1/" 129 } else { 130 // If the endpoint has been set explicitly, use this for the BasePath 131 // as well as readHost 132 rawService.BasePath = ep 133 u, err := url.Parse(ep) 134 if err != nil { 135 return nil, fmt.Errorf("supplied endpoint %v is not valid: %v", ep, err) 136 } 137 readHost = u.Host 138 } 139 140 return &Client{ 141 hc: hc, 142 raw: rawService, 143 scheme: scheme, 144 envHost: host, 145 readHost: readHost, 146 }, nil 147} 148 149// Close closes the Client. 150// 151// Close need not be called at program exit. 152func (c *Client) Close() error { 153 // Set fields to nil so that subsequent uses will panic. 154 c.hc = nil 155 c.raw = nil 156 return nil 157} 158 159// SigningScheme determines the API version to use when signing URLs. 160type SigningScheme int 161 162const ( 163 // SigningSchemeDefault is presently V2 and will change to V4 in the future. 164 SigningSchemeDefault SigningScheme = iota 165 166 // SigningSchemeV2 uses the V2 scheme to sign URLs. 167 SigningSchemeV2 168 169 // SigningSchemeV4 uses the V4 scheme to sign URLs. 170 SigningSchemeV4 171) 172 173// URLStyle determines the style to use for the signed URL. pathStyle is the 174// default. All non-default options work with V4 scheme only. See 175// https://cloud.google.com/storage/docs/request-endpoints for details. 176type URLStyle interface { 177 // host should return the host portion of the signed URL, not including 178 // the scheme (e.g. storage.googleapis.com). 179 host(bucket string) string 180 181 // path should return the path portion of the signed URL, which may include 182 // both the bucket and object name or only the object name depending on the 183 // style. 184 path(bucket, object string) string 185} 186 187type pathStyle struct{} 188 189type virtualHostedStyle struct{} 190 191type bucketBoundHostname struct { 192 hostname string 193} 194 195func (s pathStyle) host(bucket string) string { 196 return "storage.googleapis.com" 197} 198 199func (s virtualHostedStyle) host(bucket string) string { 200 return bucket + ".storage.googleapis.com" 201} 202 203func (s bucketBoundHostname) host(bucket string) string { 204 return s.hostname 205} 206 207func (s pathStyle) path(bucket, object string) string { 208 p := bucket 209 if object != "" { 210 p += "/" + object 211 } 212 return p 213} 214 215func (s virtualHostedStyle) path(bucket, object string) string { 216 return object 217} 218 219func (s bucketBoundHostname) path(bucket, object string) string { 220 return object 221} 222 223// PathStyle is the default style, and will generate a URL of the form 224// "storage.googleapis.com/<bucket-name>/<object-name>". 225func PathStyle() URLStyle { 226 return pathStyle{} 227} 228 229// VirtualHostedStyle generates a URL relative to the bucket's virtual 230// hostname, e.g. "<bucket-name>.storage.googleapis.com/<object-name>". 231func VirtualHostedStyle() URLStyle { 232 return virtualHostedStyle{} 233} 234 235// BucketBoundHostname generates a URL with a custom hostname tied to a 236// specific GCS bucket. The desired hostname should be passed in using the 237// hostname argument. Generated urls will be of the form 238// "<bucket-bound-hostname>/<object-name>". See 239// https://cloud.google.com/storage/docs/request-endpoints#cname and 240// https://cloud.google.com/load-balancing/docs/https/adding-backend-buckets-to-load-balancers 241// for details. Note that for CNAMEs, only HTTP is supported, so Insecure must 242// be set to true. 243func BucketBoundHostname(hostname string) URLStyle { 244 return bucketBoundHostname{hostname: hostname} 245} 246 247// SignedURLOptions allows you to restrict the access to the signed URL. 248type SignedURLOptions struct { 249 // GoogleAccessID represents the authorizer of the signed URL generation. 250 // It is typically the Google service account client email address from 251 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 252 // Required. 253 GoogleAccessID string 254 255 // PrivateKey is the Google service account private key. It is obtainable 256 // from the Google Developers Console. 257 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 258 // create a service account client ID or reuse one of your existing service account 259 // credentials. Click on the "Generate new P12 key" to generate and download 260 // a new private key. Once you download the P12 file, use the following command 261 // to convert it into a PEM file. 262 // 263 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 264 // 265 // Provide the contents of the PEM file as a byte slice. 266 // Exactly one of PrivateKey or SignBytes must be non-nil. 267 PrivateKey []byte 268 269 // SignBytes is a function for implementing custom signing. For example, if 270 // your application is running on Google App Engine, you can use 271 // appengine's internal signing function: 272 // ctx := appengine.NewContext(request) 273 // acc, _ := appengine.ServiceAccount(ctx) 274 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 275 // GoogleAccessID: acc, 276 // SignBytes: func(b []byte) ([]byte, error) { 277 // _, signedBytes, err := appengine.SignBytes(ctx, b) 278 // return signedBytes, err 279 // }, 280 // // etc. 281 // }) 282 // 283 // Exactly one of PrivateKey or SignBytes must be non-nil. 284 SignBytes func([]byte) ([]byte, error) 285 286 // Method is the HTTP method to be used with the signed URL. 287 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. 288 // Required. 289 Method string 290 291 // Expires is the expiration time on the signed URL. It must be 292 // a datetime in the future. For SigningSchemeV4, the expiration may be no 293 // more than seven days in the future. 294 // Required. 295 Expires time.Time 296 297 // ContentType is the content type header the client must provide 298 // to use the generated signed URL. 299 // Optional. 300 ContentType string 301 302 // Headers is a list of extension headers the client must provide 303 // in order to use the generated signed URL. Each must be a string of the 304 // form "key:values", with multiple values separated by a semicolon. 305 // Optional. 306 Headers []string 307 308 // QueryParameters is a map of additional query parameters. When 309 // SigningScheme is V4, this is used in computing the signature, and the 310 // client must use the same query parameters when using the generated signed 311 // URL. 312 // Optional. 313 QueryParameters url.Values 314 315 // MD5 is the base64 encoded MD5 checksum of the file. 316 // If provided, the client should provide the exact value on the request 317 // header in order to use the signed URL. 318 // Optional. 319 MD5 string 320 321 // Style provides options for the type of URL to use. Options are 322 // PathStyle (default), BucketBoundHostname, and VirtualHostedStyle. See 323 // https://cloud.google.com/storage/docs/request-endpoints for details. 324 // Only supported for V4 signing. 325 // Optional. 326 Style URLStyle 327 328 // Insecure determines whether the signed URL should use HTTPS (default) or 329 // HTTP. 330 // Only supported for V4 signing. 331 // Optional. 332 Insecure bool 333 334 // Scheme determines the version of URL signing to use. Default is 335 // SigningSchemeV2. 336 Scheme SigningScheme 337} 338 339var ( 340 tabRegex = regexp.MustCompile(`[\t]+`) 341 // I was tempted to call this spacex. :) 342 spaceRegex = regexp.MustCompile(` +`) 343 344 canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) 345 excludedCanonicalHeaders = map[string]bool{ 346 "x-goog-encryption-key": true, 347 "x-goog-encryption-key-sha256": true, 348 } 349) 350 351// v2SanitizeHeaders applies the specifications for canonical extension headers at 352// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. 353func v2SanitizeHeaders(hdrs []string) []string { 354 headerMap := map[string][]string{} 355 for _, hdr := range hdrs { 356 // No leading or trailing whitespaces. 357 sanitizedHeader := strings.TrimSpace(hdr) 358 359 var header, value string 360 // Only keep canonical headers, discard any others. 361 headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) 362 if len(headerMatches) == 0 { 363 continue 364 } 365 header = headerMatches[1] 366 value = headerMatches[2] 367 368 header = strings.ToLower(strings.TrimSpace(header)) 369 value = strings.TrimSpace(value) 370 371 if excludedCanonicalHeaders[header] { 372 // Do not keep any deliberately excluded canonical headers when signing. 373 continue 374 } 375 376 if len(value) > 0 { 377 // Remove duplicate headers by appending the values of duplicates 378 // in their order of appearance. 379 headerMap[header] = append(headerMap[header], value) 380 } 381 } 382 383 var sanitizedHeaders []string 384 for header, values := range headerMap { 385 // There should be no spaces around the colon separating the header name 386 // from the header value or around the values themselves. The values 387 // should be separated by commas. 388 // 389 // NOTE: The semantics for headers without a value are not clear. 390 // However from specifications these should be edge-cases anyway and we 391 // should assume that there will be no canonical headers using empty 392 // values. Any such headers are discarded at the regexp stage above. 393 sanitizedHeaders = append(sanitizedHeaders, fmt.Sprintf("%s:%s", header, strings.Join(values, ","))) 394 } 395 sort.Strings(sanitizedHeaders) 396 return sanitizedHeaders 397} 398 399// v4SanitizeHeaders applies the specifications for canonical extension headers 400// at https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. 401// 402// V4 does a couple things differently from V2: 403// - Headers get sorted by key, instead of by key:value. We do this in 404// signedURLV4. 405// - There's no canonical regexp: we simply split headers on :. 406// - We don't exclude canonical headers. 407// - We replace leading and trailing spaces in header values, like v2, but also 408// all intermediate space duplicates get stripped. That is, there's only ever 409// a single consecutive space. 410func v4SanitizeHeaders(hdrs []string) []string { 411 headerMap := map[string][]string{} 412 for _, hdr := range hdrs { 413 // No leading or trailing whitespaces. 414 sanitizedHeader := strings.TrimSpace(hdr) 415 416 var key, value string 417 headerMatches := strings.Split(sanitizedHeader, ":") 418 if len(headerMatches) < 2 { 419 continue 420 } 421 422 key = headerMatches[0] 423 value = headerMatches[1] 424 425 key = strings.ToLower(strings.TrimSpace(key)) 426 value = strings.TrimSpace(value) 427 value = string(spaceRegex.ReplaceAll([]byte(value), []byte(" "))) 428 value = string(tabRegex.ReplaceAll([]byte(value), []byte("\t"))) 429 430 if len(value) > 0 { 431 // Remove duplicate headers by appending the values of duplicates 432 // in their order of appearance. 433 headerMap[key] = append(headerMap[key], value) 434 } 435 } 436 437 var sanitizedHeaders []string 438 for header, values := range headerMap { 439 // There should be no spaces around the colon separating the header name 440 // from the header value or around the values themselves. The values 441 // should be separated by commas. 442 // 443 // NOTE: The semantics for headers without a value are not clear. 444 // However from specifications these should be edge-cases anyway and we 445 // should assume that there will be no canonical headers using empty 446 // values. Any such headers are discarded at the regexp stage above. 447 sanitizedHeaders = append(sanitizedHeaders, fmt.Sprintf("%s:%s", header, strings.Join(values, ","))) 448 } 449 return sanitizedHeaders 450} 451 452// SignedURL returns a URL for the specified object. Signed URLs allow 453// the users access to a restricted resource for a limited time without having a 454// Google account or signing in. For more information about the signed 455// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. 456func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { 457 now := utcNow() 458 if err := validateOptions(opts, now); err != nil { 459 return "", err 460 } 461 462 switch opts.Scheme { 463 case SigningSchemeV2: 464 opts.Headers = v2SanitizeHeaders(opts.Headers) 465 return signedURLV2(bucket, name, opts) 466 case SigningSchemeV4: 467 opts.Headers = v4SanitizeHeaders(opts.Headers) 468 return signedURLV4(bucket, name, opts, now) 469 default: // SigningSchemeDefault 470 opts.Headers = v2SanitizeHeaders(opts.Headers) 471 return signedURLV2(bucket, name, opts) 472 } 473} 474 475func validateOptions(opts *SignedURLOptions, now time.Time) error { 476 if opts == nil { 477 return errors.New("storage: missing required SignedURLOptions") 478 } 479 if opts.GoogleAccessID == "" { 480 return errors.New("storage: missing required GoogleAccessID") 481 } 482 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) { 483 return errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 484 } 485 opts.Method = strings.ToUpper(opts.Method) 486 if _, ok := signedURLMethods[opts.Method]; !ok { 487 return errMethodNotValid 488 } 489 if opts.Expires.IsZero() { 490 return errors.New("storage: missing required expires option") 491 } 492 if opts.MD5 != "" { 493 md5, err := base64.StdEncoding.DecodeString(opts.MD5) 494 if err != nil || len(md5) != 16 { 495 return errors.New("storage: invalid MD5 checksum") 496 } 497 } 498 if opts.Style == nil { 499 opts.Style = PathStyle() 500 } 501 if _, ok := opts.Style.(pathStyle); !ok && opts.Scheme == SigningSchemeV2 { 502 return errors.New("storage: only path-style URLs are permitted with SigningSchemeV2") 503 } 504 if opts.Scheme == SigningSchemeV4 { 505 cutoff := now.Add(604801 * time.Second) // 7 days + 1 second 506 if !opts.Expires.Before(cutoff) { 507 return errors.New("storage: expires must be within seven days from now") 508 } 509 } 510 return nil 511} 512 513const ( 514 iso8601 = "20060102T150405Z" 515 yearMonthDay = "20060102" 516) 517 518// utcNow returns the current time in UTC and is a variable to allow for 519// reassignment in tests to provide deterministic signed URL values. 520var utcNow = func() time.Time { 521 return time.Now().UTC() 522} 523 524// extractHeaderNames takes in a series of key:value headers and returns the 525// header names only. 526func extractHeaderNames(kvs []string) []string { 527 var res []string 528 for _, header := range kvs { 529 nameValue := strings.Split(header, ":") 530 res = append(res, nameValue[0]) 531 } 532 return res 533} 534 535// pathEncodeV4 creates an encoded string that matches the v4 signature spec. 536// Following the spec precisely is necessary in order to ensure that the URL 537// and signing string are correctly formed, and Go's url.PathEncode and 538// url.QueryEncode don't generate an exact match without some additional logic. 539func pathEncodeV4(path string) string { 540 segments := strings.Split(path, "/") 541 var encodedSegments []string 542 for _, s := range segments { 543 encodedSegments = append(encodedSegments, url.QueryEscape(s)) 544 } 545 encodedStr := strings.Join(encodedSegments, "/") 546 encodedStr = strings.Replace(encodedStr, "+", "%20", -1) 547 return encodedStr 548} 549 550// signedURLV4 creates a signed URL using the sigV4 algorithm. 551func signedURLV4(bucket, name string, opts *SignedURLOptions, now time.Time) (string, error) { 552 buf := &bytes.Buffer{} 553 fmt.Fprintf(buf, "%s\n", opts.Method) 554 555 u := &url.URL{Path: opts.Style.path(bucket, name)} 556 u.RawPath = pathEncodeV4(u.Path) 557 558 // Note: we have to add a / here because GCS does so auto-magically, despite 559 // our encoding not doing so (and we have to exactly match their 560 // canonical query). 561 fmt.Fprintf(buf, "/%s\n", u.RawPath) 562 563 headerNames := append(extractHeaderNames(opts.Headers), "host") 564 if opts.ContentType != "" { 565 headerNames = append(headerNames, "content-type") 566 } 567 if opts.MD5 != "" { 568 headerNames = append(headerNames, "content-md5") 569 } 570 sort.Strings(headerNames) 571 signedHeaders := strings.Join(headerNames, ";") 572 timestamp := now.Format(iso8601) 573 credentialScope := fmt.Sprintf("%s/auto/storage/goog4_request", now.Format(yearMonthDay)) 574 canonicalQueryString := url.Values{ 575 "X-Goog-Algorithm": {"GOOG4-RSA-SHA256"}, 576 "X-Goog-Credential": {fmt.Sprintf("%s/%s", opts.GoogleAccessID, credentialScope)}, 577 "X-Goog-Date": {timestamp}, 578 "X-Goog-Expires": {fmt.Sprintf("%d", int(opts.Expires.Sub(now).Seconds()))}, 579 "X-Goog-SignedHeaders": {signedHeaders}, 580 } 581 // Add user-supplied query parameters to the canonical query string. For V4, 582 // it's necessary to include these. 583 for k, v := range opts.QueryParameters { 584 canonicalQueryString[k] = append(canonicalQueryString[k], v...) 585 } 586 587 fmt.Fprintf(buf, "%s\n", canonicalQueryString.Encode()) 588 589 // Fill in the hostname based on the desired URL style. 590 u.Host = opts.Style.host(bucket) 591 592 // Fill in the URL scheme. 593 if opts.Insecure { 594 u.Scheme = "http" 595 } else { 596 u.Scheme = "https" 597 } 598 599 var headersWithValue []string 600 headersWithValue = append(headersWithValue, "host:"+u.Host) 601 headersWithValue = append(headersWithValue, opts.Headers...) 602 if opts.ContentType != "" { 603 headersWithValue = append(headersWithValue, "content-type:"+opts.ContentType) 604 } 605 if opts.MD5 != "" { 606 headersWithValue = append(headersWithValue, "content-md5:"+opts.MD5) 607 } 608 // Trim extra whitespace from headers and replace with a single space. 609 var trimmedHeaders []string 610 for _, h := range headersWithValue { 611 trimmedHeaders = append(trimmedHeaders, strings.Join(strings.Fields(h), " ")) 612 } 613 canonicalHeaders := strings.Join(sortHeadersByKey(trimmedHeaders), "\n") 614 fmt.Fprintf(buf, "%s\n\n", canonicalHeaders) 615 fmt.Fprintf(buf, "%s\n", signedHeaders) 616 617 // If the user provides a value for X-Goog-Content-SHA256, we must use 618 // that value in the request string. If not, we use UNSIGNED-PAYLOAD. 619 sha256Header := false 620 for _, h := range trimmedHeaders { 621 if strings.HasPrefix(strings.ToLower(h), "x-goog-content-sha256") && strings.Contains(h, ":") { 622 sha256Header = true 623 fmt.Fprintf(buf, "%s", strings.SplitN(h, ":", 2)[1]) 624 break 625 } 626 } 627 if !sha256Header { 628 fmt.Fprint(buf, "UNSIGNED-PAYLOAD") 629 } 630 631 sum := sha256.Sum256(buf.Bytes()) 632 hexDigest := hex.EncodeToString(sum[:]) 633 signBuf := &bytes.Buffer{} 634 fmt.Fprint(signBuf, "GOOG4-RSA-SHA256\n") 635 fmt.Fprintf(signBuf, "%s\n", timestamp) 636 fmt.Fprintf(signBuf, "%s\n", credentialScope) 637 fmt.Fprintf(signBuf, "%s", hexDigest) 638 639 signBytes := opts.SignBytes 640 if opts.PrivateKey != nil { 641 key, err := parseKey(opts.PrivateKey) 642 if err != nil { 643 return "", err 644 } 645 signBytes = func(b []byte) ([]byte, error) { 646 sum := sha256.Sum256(b) 647 return rsa.SignPKCS1v15( 648 rand.Reader, 649 key, 650 crypto.SHA256, 651 sum[:], 652 ) 653 } 654 } 655 b, err := signBytes(signBuf.Bytes()) 656 if err != nil { 657 return "", err 658 } 659 signature := hex.EncodeToString(b) 660 canonicalQueryString.Set("X-Goog-Signature", string(signature)) 661 u.RawQuery = canonicalQueryString.Encode() 662 return u.String(), nil 663} 664 665// takes a list of headerKey:headervalue1,headervalue2,etc and sorts by header 666// key. 667func sortHeadersByKey(hdrs []string) []string { 668 headersMap := map[string]string{} 669 var headersKeys []string 670 for _, h := range hdrs { 671 parts := strings.Split(h, ":") 672 k := parts[0] 673 v := parts[1] 674 headersMap[k] = v 675 headersKeys = append(headersKeys, k) 676 } 677 sort.Strings(headersKeys) 678 var sorted []string 679 for _, k := range headersKeys { 680 v := headersMap[k] 681 sorted = append(sorted, fmt.Sprintf("%s:%s", k, v)) 682 } 683 return sorted 684} 685 686func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) { 687 signBytes := opts.SignBytes 688 if opts.PrivateKey != nil { 689 key, err := parseKey(opts.PrivateKey) 690 if err != nil { 691 return "", err 692 } 693 signBytes = func(b []byte) ([]byte, error) { 694 sum := sha256.Sum256(b) 695 return rsa.SignPKCS1v15( 696 rand.Reader, 697 key, 698 crypto.SHA256, 699 sum[:], 700 ) 701 } 702 } 703 704 u := &url.URL{ 705 Path: fmt.Sprintf("/%s/%s", bucket, name), 706 } 707 708 buf := &bytes.Buffer{} 709 fmt.Fprintf(buf, "%s\n", opts.Method) 710 fmt.Fprintf(buf, "%s\n", opts.MD5) 711 fmt.Fprintf(buf, "%s\n", opts.ContentType) 712 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) 713 if len(opts.Headers) > 0 { 714 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) 715 } 716 fmt.Fprintf(buf, "%s", u.String()) 717 718 b, err := signBytes(buf.Bytes()) 719 if err != nil { 720 return "", err 721 } 722 encoded := base64.StdEncoding.EncodeToString(b) 723 u.Scheme = "https" 724 u.Host = "storage.googleapis.com" 725 q := u.Query() 726 q.Set("GoogleAccessId", opts.GoogleAccessID) 727 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) 728 q.Set("Signature", string(encoded)) 729 u.RawQuery = q.Encode() 730 return u.String(), nil 731} 732 733// ObjectHandle provides operations on an object in a Google Cloud Storage bucket. 734// Use BucketHandle.Object to get a handle. 735type ObjectHandle struct { 736 c *Client 737 bucket string 738 object string 739 acl ACLHandle 740 gen int64 // a negative value indicates latest 741 conds *Conditions 742 encryptionKey []byte // AES-256 key 743 userProject string // for requester-pays buckets 744 readCompressed bool // Accept-Encoding: gzip 745} 746 747// ACL provides access to the object's access control list. 748// This controls who can read and write this object. 749// This call does not perform any network operations. 750func (o *ObjectHandle) ACL() *ACLHandle { 751 return &o.acl 752} 753 754// Generation returns a new ObjectHandle that operates on a specific generation 755// of the object. 756// By default, the handle operates on the latest generation. Not 757// all operations work when given a specific generation; check the API 758// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. 759func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { 760 o2 := *o 761 o2.gen = gen 762 return &o2 763} 764 765// If returns a new ObjectHandle that applies a set of preconditions. 766// Preconditions already set on the ObjectHandle are ignored. 767// Operations on the new handle will return an error if the preconditions are not 768// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions 769// for more details. 770func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { 771 o2 := *o 772 o2.conds = &conds 773 return &o2 774} 775 776// Key returns a new ObjectHandle that uses the supplied encryption 777// key to encrypt and decrypt the object's contents. 778// 779// Encryption key must be a 32-byte AES-256 key. 780// See https://cloud.google.com/storage/docs/encryption for details. 781func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { 782 o2 := *o 783 o2.encryptionKey = encryptionKey 784 return &o2 785} 786 787// Attrs returns meta information about the object. 788// ErrObjectNotExist will be returned if the object is not found. 789func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { 790 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") 791 defer func() { trace.EndSpan(ctx, err) }() 792 793 if err := o.validate(); err != nil { 794 return nil, err 795 } 796 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) 797 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { 798 return nil, err 799 } 800 if o.userProject != "" { 801 call.UserProject(o.userProject) 802 } 803 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 804 return nil, err 805 } 806 var obj *raw.Object 807 setClientHeader(call.Header()) 808 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 809 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 810 return nil, ErrObjectNotExist 811 } 812 if err != nil { 813 return nil, err 814 } 815 return newObject(obj), nil 816} 817 818// Update updates an object with the provided attributes. 819// All zero-value attributes are ignored. 820// ErrObjectNotExist will be returned if the object is not found. 821func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { 822 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") 823 defer func() { trace.EndSpan(ctx, err) }() 824 825 if err := o.validate(); err != nil { 826 return nil, err 827 } 828 var attrs ObjectAttrs 829 // Lists of fields to send, and set to null, in the JSON. 830 var forceSendFields, nullFields []string 831 if uattrs.ContentType != nil { 832 attrs.ContentType = optional.ToString(uattrs.ContentType) 833 // For ContentType, sending the empty string is a no-op. 834 // Instead we send a null. 835 if attrs.ContentType == "" { 836 nullFields = append(nullFields, "ContentType") 837 } else { 838 forceSendFields = append(forceSendFields, "ContentType") 839 } 840 } 841 if uattrs.ContentLanguage != nil { 842 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) 843 // For ContentLanguage it's an error to send the empty string. 844 // Instead we send a null. 845 if attrs.ContentLanguage == "" { 846 nullFields = append(nullFields, "ContentLanguage") 847 } else { 848 forceSendFields = append(forceSendFields, "ContentLanguage") 849 } 850 } 851 if uattrs.ContentEncoding != nil { 852 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) 853 forceSendFields = append(forceSendFields, "ContentEncoding") 854 } 855 if uattrs.ContentDisposition != nil { 856 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) 857 forceSendFields = append(forceSendFields, "ContentDisposition") 858 } 859 if uattrs.CacheControl != nil { 860 attrs.CacheControl = optional.ToString(uattrs.CacheControl) 861 forceSendFields = append(forceSendFields, "CacheControl") 862 } 863 if uattrs.EventBasedHold != nil { 864 attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold) 865 forceSendFields = append(forceSendFields, "EventBasedHold") 866 } 867 if uattrs.TemporaryHold != nil { 868 attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) 869 forceSendFields = append(forceSendFields, "TemporaryHold") 870 } 871 if uattrs.Metadata != nil { 872 attrs.Metadata = uattrs.Metadata 873 if len(attrs.Metadata) == 0 { 874 // Sending the empty map is a no-op. We send null instead. 875 nullFields = append(nullFields, "Metadata") 876 } else { 877 forceSendFields = append(forceSendFields, "Metadata") 878 } 879 } 880 if uattrs.ACL != nil { 881 attrs.ACL = uattrs.ACL 882 // It's an error to attempt to delete the ACL, so 883 // we don't append to nullFields here. 884 forceSendFields = append(forceSendFields, "Acl") 885 } 886 rawObj := attrs.toRawObject(o.bucket) 887 rawObj.ForceSendFields = forceSendFields 888 rawObj.NullFields = nullFields 889 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) 890 if err := applyConds("Update", o.gen, o.conds, call); err != nil { 891 return nil, err 892 } 893 if o.userProject != "" { 894 call.UserProject(o.userProject) 895 } 896 if uattrs.PredefinedACL != "" { 897 call.PredefinedAcl(uattrs.PredefinedACL) 898 } 899 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 900 return nil, err 901 } 902 var obj *raw.Object 903 setClientHeader(call.Header()) 904 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 905 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 906 return nil, ErrObjectNotExist 907 } 908 if err != nil { 909 return nil, err 910 } 911 return newObject(obj), nil 912} 913 914// BucketName returns the name of the bucket. 915func (o *ObjectHandle) BucketName() string { 916 return o.bucket 917} 918 919// ObjectName returns the name of the object. 920func (o *ObjectHandle) ObjectName() string { 921 return o.object 922} 923 924// ObjectAttrsToUpdate is used to update the attributes of an object. 925// Only fields set to non-nil values will be updated. 926// Set a field to its zero value to delete it. 927// 928// For example, to change ContentType and delete ContentEncoding and 929// Metadata, use 930// ObjectAttrsToUpdate{ 931// ContentType: "text/html", 932// ContentEncoding: "", 933// Metadata: map[string]string{}, 934// } 935type ObjectAttrsToUpdate struct { 936 EventBasedHold optional.Bool 937 TemporaryHold optional.Bool 938 ContentType optional.String 939 ContentLanguage optional.String 940 ContentEncoding optional.String 941 ContentDisposition optional.String 942 CacheControl optional.String 943 Metadata map[string]string // set to map[string]string{} to delete 944 ACL []ACLRule 945 946 // If not empty, applies a predefined set of access controls. ACL must be nil. 947 // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch. 948 PredefinedACL string 949} 950 951// Delete deletes the single specified object. 952func (o *ObjectHandle) Delete(ctx context.Context) error { 953 if err := o.validate(); err != nil { 954 return err 955 } 956 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) 957 if err := applyConds("Delete", o.gen, o.conds, call); err != nil { 958 return err 959 } 960 if o.userProject != "" { 961 call.UserProject(o.userProject) 962 } 963 // Encryption doesn't apply to Delete. 964 setClientHeader(call.Header()) 965 err := runWithRetry(ctx, func() error { return call.Do() }) 966 switch e := err.(type) { 967 case nil: 968 return nil 969 case *googleapi.Error: 970 if e.Code == http.StatusNotFound { 971 return ErrObjectNotExist 972 } 973 } 974 return err 975} 976 977// ReadCompressed when true causes the read to happen without decompressing. 978func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { 979 o2 := *o 980 o2.readCompressed = compressed 981 return &o2 982} 983 984// NewWriter returns a storage Writer that writes to the GCS object 985// associated with this ObjectHandle. 986// 987// A new object will be created unless an object with this name already exists. 988// Otherwise any previous object with the same name will be replaced. 989// The object will not be available (and any previous object will remain) 990// until Close has been called. 991// 992// Attributes can be set on the object by modifying the returned Writer's 993// ObjectAttrs field before the first call to Write. If no ContentType 994// attribute is specified, the content type will be automatically sniffed 995// using net/http.DetectContentType. 996// 997// It is the caller's responsibility to call Close when writing is done. To 998// stop writing without saving the data, cancel the context. 999func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { 1000 return &Writer{ 1001 ctx: ctx, 1002 o: o, 1003 donec: make(chan struct{}), 1004 ObjectAttrs: ObjectAttrs{Name: o.object}, 1005 ChunkSize: googleapi.DefaultUploadChunkSize, 1006 } 1007} 1008 1009func (o *ObjectHandle) validate() error { 1010 if o.bucket == "" { 1011 return errors.New("storage: bucket name is empty") 1012 } 1013 if o.object == "" { 1014 return errors.New("storage: object name is empty") 1015 } 1016 if !utf8.ValidString(o.object) { 1017 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) 1018 } 1019 return nil 1020} 1021 1022// parseKey converts the binary contents of a private key file to an 1023// *rsa.PrivateKey. It detects whether the private key is in a PEM container or 1024// not. If so, it extracts the private key from PEM container before 1025// conversion. It only supports PEM containers with no passphrase. 1026func parseKey(key []byte) (*rsa.PrivateKey, error) { 1027 if block, _ := pem.Decode(key); block != nil { 1028 key = block.Bytes 1029 } 1030 parsedKey, err := x509.ParsePKCS8PrivateKey(key) 1031 if err != nil { 1032 parsedKey, err = x509.ParsePKCS1PrivateKey(key) 1033 if err != nil { 1034 return nil, err 1035 } 1036 } 1037 parsed, ok := parsedKey.(*rsa.PrivateKey) 1038 if !ok { 1039 return nil, errors.New("oauth2: private key is invalid") 1040 } 1041 return parsed, nil 1042} 1043 1044// toRawObject copies the editable attributes from o to the raw library's Object type. 1045func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { 1046 var ret string 1047 if !o.RetentionExpirationTime.IsZero() { 1048 ret = o.RetentionExpirationTime.Format(time.RFC3339) 1049 } 1050 return &raw.Object{ 1051 Bucket: bucket, 1052 Name: o.Name, 1053 EventBasedHold: o.EventBasedHold, 1054 TemporaryHold: o.TemporaryHold, 1055 RetentionExpirationTime: ret, 1056 ContentType: o.ContentType, 1057 ContentEncoding: o.ContentEncoding, 1058 ContentLanguage: o.ContentLanguage, 1059 CacheControl: o.CacheControl, 1060 ContentDisposition: o.ContentDisposition, 1061 StorageClass: o.StorageClass, 1062 Acl: toRawObjectACL(o.ACL), 1063 Metadata: o.Metadata, 1064 } 1065} 1066 1067// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. 1068type ObjectAttrs struct { 1069 // Bucket is the name of the bucket containing this GCS object. 1070 // This field is read-only. 1071 Bucket string 1072 1073 // Name is the name of the object within the bucket. 1074 // This field is read-only. 1075 Name string 1076 1077 // ContentType is the MIME type of the object's content. 1078 ContentType string 1079 1080 // ContentLanguage is the content language of the object's content. 1081 ContentLanguage string 1082 1083 // CacheControl is the Cache-Control header to be sent in the response 1084 // headers when serving the object data. 1085 CacheControl string 1086 1087 // EventBasedHold specifies whether an object is under event-based hold. New 1088 // objects created in a bucket whose DefaultEventBasedHold is set will 1089 // default to that value. 1090 EventBasedHold bool 1091 1092 // TemporaryHold specifies whether an object is under temporary hold. While 1093 // this flag is set to true, the object is protected against deletion and 1094 // overwrites. 1095 TemporaryHold bool 1096 1097 // RetentionExpirationTime is a server-determined value that specifies the 1098 // earliest time that the object's retention period expires. 1099 // This is a read-only field. 1100 RetentionExpirationTime time.Time 1101 1102 // ACL is the list of access control rules for the object. 1103 ACL []ACLRule 1104 1105 // If not empty, applies a predefined set of access controls. It should be set 1106 // only when writing, copying or composing an object. When copying or composing, 1107 // it acts as the destinationPredefinedAcl parameter. 1108 // PredefinedACL is always empty for ObjectAttrs returned from the service. 1109 // See https://cloud.google.com/storage/docs/json_api/v1/objects/insert 1110 // for valid values. 1111 PredefinedACL string 1112 1113 // Owner is the owner of the object. This field is read-only. 1114 // 1115 // If non-zero, it is in the form of "user-<userId>". 1116 Owner string 1117 1118 // Size is the length of the object's content. This field is read-only. 1119 Size int64 1120 1121 // ContentEncoding is the encoding of the object's content. 1122 ContentEncoding string 1123 1124 // ContentDisposition is the optional Content-Disposition header of the object 1125 // sent in the response headers. 1126 ContentDisposition string 1127 1128 // MD5 is the MD5 hash of the object's content. This field is read-only, 1129 // except when used from a Writer. If set on a Writer, the uploaded 1130 // data is rejected if its MD5 hash does not match this field. 1131 MD5 []byte 1132 1133 // CRC32C is the CRC32 checksum of the object's content using the Castagnoli93 1134 // polynomial. This field is read-only, except when used from a Writer or 1135 // Composer. In those cases, if the SendCRC32C field in the Writer or Composer 1136 // is set to is true, the uploaded data is rejected if its CRC32C hash does 1137 // not match this field. 1138 CRC32C uint32 1139 1140 // MediaLink is an URL to the object's content. This field is read-only. 1141 MediaLink string 1142 1143 // Metadata represents user-provided metadata, in key/value pairs. 1144 // It can be nil if no metadata is provided. 1145 Metadata map[string]string 1146 1147 // Generation is the generation number of the object's content. 1148 // This field is read-only. 1149 Generation int64 1150 1151 // Metageneration is the version of the metadata for this 1152 // object at this generation. This field is used for preconditions 1153 // and for detecting changes in metadata. A metageneration number 1154 // is only meaningful in the context of a particular generation 1155 // of a particular object. This field is read-only. 1156 Metageneration int64 1157 1158 // StorageClass is the storage class of the object. This defines 1159 // how objects are stored and determines the SLA and the cost of storage. 1160 // Typical values are "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE". 1161 // Defaults to "STANDARD". 1162 // See https://cloud.google.com/storage/docs/storage-classes for all 1163 // valid values. 1164 StorageClass string 1165 1166 // Created is the time the object was created. This field is read-only. 1167 Created time.Time 1168 1169 // Deleted is the time the object was deleted. 1170 // If not deleted, it is the zero value. This field is read-only. 1171 Deleted time.Time 1172 1173 // Updated is the creation or modification time of the object. 1174 // For buckets with versioning enabled, changing an object's 1175 // metadata does not change this property. This field is read-only. 1176 Updated time.Time 1177 1178 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the 1179 // customer-supplied encryption key for the object. It is empty if there is 1180 // no customer-supplied encryption key. 1181 // See // https://cloud.google.com/storage/docs/encryption for more about 1182 // encryption in Google Cloud Storage. 1183 CustomerKeySHA256 string 1184 1185 // Cloud KMS key name, in the form 1186 // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, 1187 // if the object is encrypted by such a key. 1188 // 1189 // Providing both a KMSKeyName and a customer-supplied encryption key (via 1190 // ObjectHandle.Key) will result in an error when writing an object. 1191 KMSKeyName string 1192 1193 // Prefix is set only for ObjectAttrs which represent synthetic "directory 1194 // entries" when iterating over buckets using Query.Delimiter. See 1195 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be 1196 // populated. 1197 Prefix string 1198 1199 // Etag is the HTTP/1.1 Entity tag for the object. 1200 // This field is read-only. 1201 Etag string 1202} 1203 1204// convertTime converts a time in RFC3339 format to time.Time. 1205// If any error occurs in parsing, the zero-value time.Time is silently returned. 1206func convertTime(t string) time.Time { 1207 var r time.Time 1208 if t != "" { 1209 r, _ = time.Parse(time.RFC3339, t) 1210 } 1211 return r 1212} 1213 1214func newObject(o *raw.Object) *ObjectAttrs { 1215 if o == nil { 1216 return nil 1217 } 1218 owner := "" 1219 if o.Owner != nil { 1220 owner = o.Owner.Entity 1221 } 1222 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) 1223 crc32c, _ := decodeUint32(o.Crc32c) 1224 var sha256 string 1225 if o.CustomerEncryption != nil { 1226 sha256 = o.CustomerEncryption.KeySha256 1227 } 1228 return &ObjectAttrs{ 1229 Bucket: o.Bucket, 1230 Name: o.Name, 1231 ContentType: o.ContentType, 1232 ContentLanguage: o.ContentLanguage, 1233 CacheControl: o.CacheControl, 1234 EventBasedHold: o.EventBasedHold, 1235 TemporaryHold: o.TemporaryHold, 1236 RetentionExpirationTime: convertTime(o.RetentionExpirationTime), 1237 ACL: toObjectACLRules(o.Acl), 1238 Owner: owner, 1239 ContentEncoding: o.ContentEncoding, 1240 ContentDisposition: o.ContentDisposition, 1241 Size: int64(o.Size), 1242 MD5: md5, 1243 CRC32C: crc32c, 1244 MediaLink: o.MediaLink, 1245 Metadata: o.Metadata, 1246 Generation: o.Generation, 1247 Metageneration: o.Metageneration, 1248 StorageClass: o.StorageClass, 1249 CustomerKeySHA256: sha256, 1250 KMSKeyName: o.KmsKeyName, 1251 Created: convertTime(o.TimeCreated), 1252 Deleted: convertTime(o.TimeDeleted), 1253 Updated: convertTime(o.Updated), 1254 Etag: o.Etag, 1255 } 1256} 1257 1258// Decode a uint32 encoded in Base64 in big-endian byte order. 1259func decodeUint32(b64 string) (uint32, error) { 1260 d, err := base64.StdEncoding.DecodeString(b64) 1261 if err != nil { 1262 return 0, err 1263 } 1264 if len(d) != 4 { 1265 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) 1266 } 1267 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil 1268} 1269 1270// Encode a uint32 as Base64 in big-endian byte order. 1271func encodeUint32(u uint32) string { 1272 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} 1273 return base64.StdEncoding.EncodeToString(b) 1274} 1275 1276// Query represents a query to filter objects from a bucket. 1277type Query struct { 1278 // Delimiter returns results in a directory-like fashion. 1279 // Results will contain only objects whose names, aside from the 1280 // prefix, do not contain delimiter. Objects whose names, 1281 // aside from the prefix, contain delimiter will have their name, 1282 // truncated after the delimiter, returned in prefixes. 1283 // Duplicate prefixes are omitted. 1284 // Optional. 1285 Delimiter string 1286 1287 // Prefix is the prefix filter to query objects 1288 // whose names begin with this prefix. 1289 // Optional. 1290 Prefix string 1291 1292 // Versions indicates whether multiple versions of the same 1293 // object will be included in the results. 1294 Versions bool 1295 1296 // fieldSelection is used to select only specific fields to be returned by 1297 // the query. It's used internally and is populated for the user by 1298 // calling Query.SetAttrSelection 1299 fieldSelection string 1300} 1301 1302// attrToFieldMap maps the field names of ObjectAttrs to the underlying field 1303// names in the API call. Only the ObjectAttrs field names are visible to users 1304// because they are already part of the public API of the package. 1305var attrToFieldMap = map[string]string{ 1306 "Bucket": "bucket", 1307 "Name": "name", 1308 "ContentType": "contentType", 1309 "ContentLanguage": "contentLanguage", 1310 "CacheControl": "cacheControl", 1311 "EventBasedHold": "eventBasedHold", 1312 "TemporaryHold": "temporaryHold", 1313 "RetentionExpirationTime": "retentionExpirationTime", 1314 "ACL": "acl", 1315 "Owner": "owner", 1316 "ContentEncoding": "contentEncoding", 1317 "ContentDisposition": "contentDisposition", 1318 "Size": "size", 1319 "MD5": "md5Hash", 1320 "CRC32C": "crc32c", 1321 "MediaLink": "mediaLink", 1322 "Metadata": "metadata", 1323 "Generation": "generation", 1324 "Metageneration": "metageneration", 1325 "StorageClass": "storageClass", 1326 "CustomerKeySHA256": "customerEncryption", 1327 "KMSKeyName": "kmsKeyName", 1328 "Created": "timeCreated", 1329 "Deleted": "timeDeleted", 1330 "Updated": "updated", 1331 "Etag": "etag", 1332} 1333 1334// SetAttrSelection makes the query populate only specific attributes of 1335// objects. When iterating over objects, if you only need each object's name 1336// and size, pass []string{"Name", "Size"} to this method. Only these fields 1337// will be fetched for each object across the network; the other fields of 1338// ObjectAttr will remain at their default values. This is a performance 1339// optimization; for more information, see 1340// https://cloud.google.com/storage/docs/json_api/v1/how-tos/performance 1341func (q *Query) SetAttrSelection(attrs []string) error { 1342 fieldSet := make(map[string]bool) 1343 1344 for _, attr := range attrs { 1345 field, ok := attrToFieldMap[attr] 1346 if !ok { 1347 return fmt.Errorf("storage: attr %v is not valid", attr) 1348 } 1349 fieldSet[field] = true 1350 } 1351 1352 if len(fieldSet) > 0 { 1353 var b bytes.Buffer 1354 b.WriteString("items(") 1355 first := true 1356 for field := range fieldSet { 1357 if !first { 1358 b.WriteString(",") 1359 } 1360 first = false 1361 b.WriteString(field) 1362 } 1363 b.WriteString(")") 1364 q.fieldSelection = b.String() 1365 } 1366 return nil 1367} 1368 1369// Conditions constrain methods to act on specific generations of 1370// objects. 1371// 1372// The zero value is an empty set of constraints. Not all conditions or 1373// combinations of conditions are applicable to all methods. 1374// See https://cloud.google.com/storage/docs/generations-preconditions 1375// for details on how these operate. 1376type Conditions struct { 1377 // Generation constraints. 1378 // At most one of the following can be set to a non-zero value. 1379 1380 // GenerationMatch specifies that the object must have the given generation 1381 // for the operation to occur. 1382 // If GenerationMatch is zero, it has no effect. 1383 // Use DoesNotExist to specify that the object does not exist in the bucket. 1384 GenerationMatch int64 1385 1386 // GenerationNotMatch specifies that the object must not have the given 1387 // generation for the operation to occur. 1388 // If GenerationNotMatch is zero, it has no effect. 1389 GenerationNotMatch int64 1390 1391 // DoesNotExist specifies that the object must not exist in the bucket for 1392 // the operation to occur. 1393 // If DoesNotExist is false, it has no effect. 1394 DoesNotExist bool 1395 1396 // Metadata generation constraints. 1397 // At most one of the following can be set to a non-zero value. 1398 1399 // MetagenerationMatch specifies that the object must have the given 1400 // metageneration for the operation to occur. 1401 // If MetagenerationMatch is zero, it has no effect. 1402 MetagenerationMatch int64 1403 1404 // MetagenerationNotMatch specifies that the object must not have the given 1405 // metageneration for the operation to occur. 1406 // If MetagenerationNotMatch is zero, it has no effect. 1407 MetagenerationNotMatch int64 1408} 1409 1410func (c *Conditions) validate(method string) error { 1411 if *c == (Conditions{}) { 1412 return fmt.Errorf("storage: %s: empty conditions", method) 1413 } 1414 if !c.isGenerationValid() { 1415 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) 1416 } 1417 if !c.isMetagenerationValid() { 1418 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 1419 } 1420 return nil 1421} 1422 1423func (c *Conditions) isGenerationValid() bool { 1424 n := 0 1425 if c.GenerationMatch != 0 { 1426 n++ 1427 } 1428 if c.GenerationNotMatch != 0 { 1429 n++ 1430 } 1431 if c.DoesNotExist { 1432 n++ 1433 } 1434 return n <= 1 1435} 1436 1437func (c *Conditions) isMetagenerationValid() bool { 1438 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 1439} 1440 1441// applyConds modifies the provided call using the conditions in conds. 1442// call is something that quacks like a *raw.WhateverCall. 1443func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { 1444 cval := reflect.ValueOf(call) 1445 if gen >= 0 { 1446 if !setConditionField(cval, "Generation", gen) { 1447 return fmt.Errorf("storage: %s: generation not supported", method) 1448 } 1449 } 1450 if conds == nil { 1451 return nil 1452 } 1453 if err := conds.validate(method); err != nil { 1454 return err 1455 } 1456 switch { 1457 case conds.GenerationMatch != 0: 1458 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { 1459 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) 1460 } 1461 case conds.GenerationNotMatch != 0: 1462 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { 1463 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) 1464 } 1465 case conds.DoesNotExist: 1466 if !setConditionField(cval, "IfGenerationMatch", int64(0)) { 1467 return fmt.Errorf("storage: %s: DoesNotExist not supported", method) 1468 } 1469 } 1470 switch { 1471 case conds.MetagenerationMatch != 0: 1472 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 1473 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 1474 } 1475 case conds.MetagenerationNotMatch != 0: 1476 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 1477 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 1478 } 1479 } 1480 return nil 1481} 1482 1483func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { 1484 if gen >= 0 { 1485 call.SourceGeneration(gen) 1486 } 1487 if conds == nil { 1488 return nil 1489 } 1490 if err := conds.validate("CopyTo source"); err != nil { 1491 return err 1492 } 1493 switch { 1494 case conds.GenerationMatch != 0: 1495 call.IfSourceGenerationMatch(conds.GenerationMatch) 1496 case conds.GenerationNotMatch != 0: 1497 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) 1498 case conds.DoesNotExist: 1499 call.IfSourceGenerationMatch(0) 1500 } 1501 switch { 1502 case conds.MetagenerationMatch != 0: 1503 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) 1504 case conds.MetagenerationNotMatch != 0: 1505 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) 1506 } 1507 return nil 1508} 1509 1510// setConditionField sets a field on a *raw.WhateverCall. 1511// We can't use anonymous interfaces because the return type is 1512// different, since the field setters are builders. 1513func setConditionField(call reflect.Value, name string, value interface{}) bool { 1514 m := call.MethodByName(name) 1515 if !m.IsValid() { 1516 return false 1517 } 1518 m.Call([]reflect.Value{reflect.ValueOf(value)}) 1519 return true 1520} 1521 1522// conditionsQuery returns the generation and conditions as a URL query 1523// string suitable for URL.RawQuery. It assumes that the conditions 1524// have been validated. 1525func conditionsQuery(gen int64, conds *Conditions) string { 1526 // URL escapes are elided because integer strings are URL-safe. 1527 var buf []byte 1528 1529 appendParam := func(s string, n int64) { 1530 if len(buf) > 0 { 1531 buf = append(buf, '&') 1532 } 1533 buf = append(buf, s...) 1534 buf = strconv.AppendInt(buf, n, 10) 1535 } 1536 1537 if gen >= 0 { 1538 appendParam("generation=", gen) 1539 } 1540 if conds == nil { 1541 return string(buf) 1542 } 1543 switch { 1544 case conds.GenerationMatch != 0: 1545 appendParam("ifGenerationMatch=", conds.GenerationMatch) 1546 case conds.GenerationNotMatch != 0: 1547 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) 1548 case conds.DoesNotExist: 1549 appendParam("ifGenerationMatch=", 0) 1550 } 1551 switch { 1552 case conds.MetagenerationMatch != 0: 1553 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) 1554 case conds.MetagenerationNotMatch != 0: 1555 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) 1556 } 1557 return string(buf) 1558} 1559 1560// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods 1561// that modifyCall searches for by name. 1562type composeSourceObj struct { 1563 src *raw.ComposeRequestSourceObjects 1564} 1565 1566func (c composeSourceObj) Generation(gen int64) { 1567 c.src.Generation = gen 1568} 1569 1570func (c composeSourceObj) IfGenerationMatch(gen int64) { 1571 // It's safe to overwrite ObjectPreconditions, since its only field is 1572 // IfGenerationMatch. 1573 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{ 1574 IfGenerationMatch: gen, 1575 } 1576} 1577 1578func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error { 1579 if key == nil { 1580 return nil 1581 } 1582 // TODO(jbd): Ask the API team to return a more user-friendly error 1583 // and avoid doing this check at the client level. 1584 if len(key) != 32 { 1585 return errors.New("storage: not a 32-byte AES-256 key") 1586 } 1587 var cs string 1588 if copySource { 1589 cs = "copy-source-" 1590 } 1591 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") 1592 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) 1593 keyHash := sha256.Sum256(key) 1594 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) 1595 return nil 1596} 1597 1598// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. 1599func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { 1600 r := c.raw.Projects.ServiceAccount.Get(projectID) 1601 res, err := r.Context(ctx).Do() 1602 if err != nil { 1603 return "", err 1604 } 1605 return res.EmailAddress, nil 1606} 1607