1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "context" 20 "crypto" 21 "crypto/rand" 22 "crypto/rsa" 23 "crypto/sha256" 24 "crypto/x509" 25 "encoding/base64" 26 "encoding/hex" 27 "encoding/pem" 28 "errors" 29 "fmt" 30 "net/http" 31 "net/url" 32 "os" 33 "reflect" 34 "regexp" 35 "sort" 36 "strconv" 37 "strings" 38 "time" 39 "unicode/utf8" 40 41 "cloud.google.com/go/internal/optional" 42 "cloud.google.com/go/internal/trace" 43 "cloud.google.com/go/internal/version" 44 "google.golang.org/api/googleapi" 45 "google.golang.org/api/option" 46 raw "google.golang.org/api/storage/v1" 47 htransport "google.golang.org/api/transport/http" 48) 49 50// Methods which can be used in signed URLs. 51var signedURLMethods = map[string]bool{"DELETE": true, "GET": true, "HEAD": true, "POST": true, "PUT": true} 52 53var ( 54 // ErrBucketNotExist indicates that the bucket does not exist. 55 ErrBucketNotExist = errors.New("storage: bucket doesn't exist") 56 // ErrObjectNotExist indicates that the object does not exist. 57 ErrObjectNotExist = errors.New("storage: object doesn't exist") 58 // errMethodNotValid indicates that given HTTP method is not valid. 59 errMethodNotValid = fmt.Errorf("storage: HTTP method should be one of %v", reflect.ValueOf(signedURLMethods).MapKeys()) 60) 61 62var userAgent = fmt.Sprintf("gcloud-golang-storage/%s", version.Repo) 63 64const ( 65 // ScopeFullControl grants permissions to manage your 66 // data and permissions in Google Cloud Storage. 67 ScopeFullControl = raw.DevstorageFullControlScope 68 69 // ScopeReadOnly grants permissions to 70 // view your data in Google Cloud Storage. 71 ScopeReadOnly = raw.DevstorageReadOnlyScope 72 73 // ScopeReadWrite grants permissions to manage your 74 // data in Google Cloud Storage. 75 ScopeReadWrite = raw.DevstorageReadWriteScope 76) 77 78var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) 79 80func setClientHeader(headers http.Header) { 81 headers.Set("x-goog-api-client", xGoogHeader) 82} 83 84// Client is a client for interacting with Google Cloud Storage. 85// 86// Clients should be reused instead of created as needed. 87// The methods of Client are safe for concurrent use by multiple goroutines. 88type Client struct { 89 hc *http.Client 90 raw *raw.Service 91 // Scheme describes the scheme under the current host. 92 scheme string 93 // EnvHost is the host set on the STORAGE_EMULATOR_HOST variable. 94 envHost string 95 // ReadHost is the default host used on the reader. 96 readHost string 97} 98 99// NewClient creates a new Google Cloud Storage client. 100// The default scope is ScopeFullControl. To use a different scope, like 101// ScopeReadOnly, use option.WithScopes. 102// 103// Clients should be reused instead of created as needed. The methods of Client 104// are safe for concurrent use by multiple goroutines. 105func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 106 var host, readHost, scheme string 107 108 if host = os.Getenv("STORAGE_EMULATOR_HOST"); host == "" { 109 scheme = "https" 110 readHost = "storage.googleapis.com" 111 112 // Prepend default options to avoid overriding options passed by the user. 113 opts = append([]option.ClientOption{option.WithScopes(ScopeFullControl), option.WithUserAgent(userAgent)}, opts...) 114 } else { 115 scheme = "http" 116 readHost = host 117 118 opts = append([]option.ClientOption{option.WithoutAuthentication()}, opts...) 119 } 120 121 hc, ep, err := htransport.NewClient(ctx, opts...) 122 if err != nil { 123 return nil, fmt.Errorf("dialing: %v", err) 124 } 125 rawService, err := raw.NewService(ctx, option.WithHTTPClient(hc)) 126 if err != nil { 127 return nil, fmt.Errorf("storage client: %v", err) 128 } 129 if ep == "" { 130 // Override the default value for BasePath from the raw client. 131 // TODO: remove when the raw client uses this endpoint as its default (~end of 2020) 132 rawService.BasePath = "https://storage.googleapis.com/storage/v1/" 133 } else { 134 // If the endpoint has been set explicitly, use this for the BasePath 135 // as well as readHost 136 rawService.BasePath = ep 137 u, err := url.Parse(ep) 138 if err != nil { 139 return nil, fmt.Errorf("supplied endpoint %v is not valid: %v", ep, err) 140 } 141 readHost = u.Host 142 } 143 144 return &Client{ 145 hc: hc, 146 raw: rawService, 147 scheme: scheme, 148 envHost: host, 149 readHost: readHost, 150 }, nil 151} 152 153// Close closes the Client. 154// 155// Close need not be called at program exit. 156func (c *Client) Close() error { 157 // Set fields to nil so that subsequent uses will panic. 158 c.hc = nil 159 c.raw = nil 160 return nil 161} 162 163// SigningScheme determines the API version to use when signing URLs. 164type SigningScheme int 165 166const ( 167 // SigningSchemeDefault is presently V2 and will change to V4 in the future. 168 SigningSchemeDefault SigningScheme = iota 169 170 // SigningSchemeV2 uses the V2 scheme to sign URLs. 171 SigningSchemeV2 172 173 // SigningSchemeV4 uses the V4 scheme to sign URLs. 174 SigningSchemeV4 175) 176 177// URLStyle determines the style to use for the signed URL. pathStyle is the 178// default. All non-default options work with V4 scheme only. See 179// https://cloud.google.com/storage/docs/request-endpoints for details. 180type URLStyle interface { 181 // host should return the host portion of the signed URL, not including 182 // the scheme (e.g. storage.googleapis.com). 183 host(bucket string) string 184 185 // path should return the path portion of the signed URL, which may include 186 // both the bucket and object name or only the object name depending on the 187 // style. 188 path(bucket, object string) string 189} 190 191type pathStyle struct{} 192 193type virtualHostedStyle struct{} 194 195type bucketBoundHostname struct { 196 hostname string 197} 198 199func (s pathStyle) host(bucket string) string { 200 return "storage.googleapis.com" 201} 202 203func (s virtualHostedStyle) host(bucket string) string { 204 return bucket + ".storage.googleapis.com" 205} 206 207func (s bucketBoundHostname) host(bucket string) string { 208 return s.hostname 209} 210 211func (s pathStyle) path(bucket, object string) string { 212 p := bucket 213 if object != "" { 214 p += "/" + object 215 } 216 return p 217} 218 219func (s virtualHostedStyle) path(bucket, object string) string { 220 return object 221} 222 223func (s bucketBoundHostname) path(bucket, object string) string { 224 return object 225} 226 227// PathStyle is the default style, and will generate a URL of the form 228// "storage.googleapis.com/<bucket-name>/<object-name>". 229func PathStyle() URLStyle { 230 return pathStyle{} 231} 232 233// VirtualHostedStyle generates a URL relative to the bucket's virtual 234// hostname, e.g. "<bucket-name>.storage.googleapis.com/<object-name>". 235func VirtualHostedStyle() URLStyle { 236 return virtualHostedStyle{} 237} 238 239// BucketBoundHostname generates a URL with a custom hostname tied to a 240// specific GCS bucket. The desired hostname should be passed in using the 241// hostname argument. Generated urls will be of the form 242// "<bucket-bound-hostname>/<object-name>". See 243// https://cloud.google.com/storage/docs/request-endpoints#cname and 244// https://cloud.google.com/load-balancing/docs/https/adding-backend-buckets-to-load-balancers 245// for details. Note that for CNAMEs, only HTTP is supported, so Insecure must 246// be set to true. 247func BucketBoundHostname(hostname string) URLStyle { 248 return bucketBoundHostname{hostname: hostname} 249} 250 251// SignedURLOptions allows you to restrict the access to the signed URL. 252type SignedURLOptions struct { 253 // GoogleAccessID represents the authorizer of the signed URL generation. 254 // It is typically the Google service account client email address from 255 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 256 // Required. 257 GoogleAccessID string 258 259 // PrivateKey is the Google service account private key. It is obtainable 260 // from the Google Developers Console. 261 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 262 // create a service account client ID or reuse one of your existing service account 263 // credentials. Click on the "Generate new P12 key" to generate and download 264 // a new private key. Once you download the P12 file, use the following command 265 // to convert it into a PEM file. 266 // 267 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 268 // 269 // Provide the contents of the PEM file as a byte slice. 270 // Exactly one of PrivateKey or SignBytes must be non-nil. 271 PrivateKey []byte 272 273 // SignBytes is a function for implementing custom signing. For example, if 274 // your application is running on Google App Engine, you can use 275 // appengine's internal signing function: 276 // ctx := appengine.NewContext(request) 277 // acc, _ := appengine.ServiceAccount(ctx) 278 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 279 // GoogleAccessID: acc, 280 // SignBytes: func(b []byte) ([]byte, error) { 281 // _, signedBytes, err := appengine.SignBytes(ctx, b) 282 // return signedBytes, err 283 // }, 284 // // etc. 285 // }) 286 // 287 // Exactly one of PrivateKey or SignBytes must be non-nil. 288 SignBytes func([]byte) ([]byte, error) 289 290 // Method is the HTTP method to be used with the signed URL. 291 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. 292 // Required. 293 Method string 294 295 // Expires is the expiration time on the signed URL. It must be 296 // a datetime in the future. For SigningSchemeV4, the expiration may be no 297 // more than seven days in the future. 298 // Required. 299 Expires time.Time 300 301 // ContentType is the content type header the client must provide 302 // to use the generated signed URL. 303 // Optional. 304 ContentType string 305 306 // Headers is a list of extension headers the client must provide 307 // in order to use the generated signed URL. Each must be a string of the 308 // form "key:values", with multiple values separated by a semicolon. 309 // Optional. 310 Headers []string 311 312 // QueryParameters is a map of additional query parameters. When 313 // SigningScheme is V4, this is used in computing the signature, and the 314 // client must use the same query parameters when using the generated signed 315 // URL. 316 // Optional. 317 QueryParameters url.Values 318 319 // MD5 is the base64 encoded MD5 checksum of the file. 320 // If provided, the client should provide the exact value on the request 321 // header in order to use the signed URL. 322 // Optional. 323 MD5 string 324 325 // Style provides options for the type of URL to use. Options are 326 // PathStyle (default), BucketBoundHostname, and VirtualHostedStyle. See 327 // https://cloud.google.com/storage/docs/request-endpoints for details. 328 // Only supported for V4 signing. 329 // Optional. 330 Style URLStyle 331 332 // Insecure determines whether the signed URL should use HTTPS (default) or 333 // HTTP. 334 // Only supported for V4 signing. 335 // Optional. 336 Insecure bool 337 338 // Scheme determines the version of URL signing to use. Default is 339 // SigningSchemeV2. 340 Scheme SigningScheme 341} 342 343var ( 344 tabRegex = regexp.MustCompile(`[\t]+`) 345 // I was tempted to call this spacex. :) 346 spaceRegex = regexp.MustCompile(` +`) 347 348 canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) 349 excludedCanonicalHeaders = map[string]bool{ 350 "x-goog-encryption-key": true, 351 "x-goog-encryption-key-sha256": true, 352 } 353) 354 355// v2SanitizeHeaders applies the specifications for canonical extension headers at 356// https://cloud.google.com/storage/docs/access-control/signed-urls-v2#about-canonical-extension-headers 357func v2SanitizeHeaders(hdrs []string) []string { 358 headerMap := map[string][]string{} 359 for _, hdr := range hdrs { 360 // No leading or trailing whitespaces. 361 sanitizedHeader := strings.TrimSpace(hdr) 362 363 var header, value string 364 // Only keep canonical headers, discard any others. 365 headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) 366 if len(headerMatches) == 0 { 367 continue 368 } 369 header = headerMatches[1] 370 value = headerMatches[2] 371 372 header = strings.ToLower(strings.TrimSpace(header)) 373 value = strings.TrimSpace(value) 374 375 if excludedCanonicalHeaders[header] { 376 // Do not keep any deliberately excluded canonical headers when signing. 377 continue 378 } 379 380 if len(value) > 0 { 381 // Remove duplicate headers by appending the values of duplicates 382 // in their order of appearance. 383 headerMap[header] = append(headerMap[header], value) 384 } 385 } 386 387 var sanitizedHeaders []string 388 for header, values := range headerMap { 389 // There should be no spaces around the colon separating the header name 390 // from the header value or around the values themselves. The values 391 // should be separated by commas. 392 // 393 // NOTE: The semantics for headers without a value are not clear. 394 // However from specifications these should be edge-cases anyway and we 395 // should assume that there will be no canonical headers using empty 396 // values. Any such headers are discarded at the regexp stage above. 397 sanitizedHeaders = append(sanitizedHeaders, fmt.Sprintf("%s:%s", header, strings.Join(values, ","))) 398 } 399 sort.Strings(sanitizedHeaders) 400 return sanitizedHeaders 401} 402 403// v4SanitizeHeaders applies the specifications for canonical extension headers 404// at https://cloud.google.com/storage/docs/authentication/canonical-requests#about-headers. 405// 406// V4 does a couple things differently from V2: 407// - Headers get sorted by key, instead of by key:value. We do this in 408// signedURLV4. 409// - There's no canonical regexp: we simply split headers on :. 410// - We don't exclude canonical headers. 411// - We replace leading and trailing spaces in header values, like v2, but also 412// all intermediate space duplicates get stripped. That is, there's only ever 413// a single consecutive space. 414func v4SanitizeHeaders(hdrs []string) []string { 415 headerMap := map[string][]string{} 416 for _, hdr := range hdrs { 417 // No leading or trailing whitespaces. 418 sanitizedHeader := strings.TrimSpace(hdr) 419 420 var key, value string 421 headerMatches := strings.Split(sanitizedHeader, ":") 422 if len(headerMatches) < 2 { 423 continue 424 } 425 426 key = headerMatches[0] 427 value = headerMatches[1] 428 429 key = strings.ToLower(strings.TrimSpace(key)) 430 value = strings.TrimSpace(value) 431 value = string(spaceRegex.ReplaceAll([]byte(value), []byte(" "))) 432 value = string(tabRegex.ReplaceAll([]byte(value), []byte("\t"))) 433 434 if len(value) > 0 { 435 // Remove duplicate headers by appending the values of duplicates 436 // in their order of appearance. 437 headerMap[key] = append(headerMap[key], value) 438 } 439 } 440 441 var sanitizedHeaders []string 442 for header, values := range headerMap { 443 // There should be no spaces around the colon separating the header name 444 // from the header value or around the values themselves. The values 445 // should be separated by commas. 446 // 447 // NOTE: The semantics for headers without a value are not clear. 448 // However from specifications these should be edge-cases anyway and we 449 // should assume that there will be no canonical headers using empty 450 // values. Any such headers are discarded at the regexp stage above. 451 sanitizedHeaders = append(sanitizedHeaders, fmt.Sprintf("%s:%s", header, strings.Join(values, ","))) 452 } 453 return sanitizedHeaders 454} 455 456// SignedURL returns a URL for the specified object. Signed URLs allow 457// the users access to a restricted resource for a limited time without having a 458// Google account or signing in. For more information about the signed 459// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. 460func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { 461 now := utcNow() 462 if err := validateOptions(opts, now); err != nil { 463 return "", err 464 } 465 466 switch opts.Scheme { 467 case SigningSchemeV2: 468 opts.Headers = v2SanitizeHeaders(opts.Headers) 469 return signedURLV2(bucket, name, opts) 470 case SigningSchemeV4: 471 opts.Headers = v4SanitizeHeaders(opts.Headers) 472 return signedURLV4(bucket, name, opts, now) 473 default: // SigningSchemeDefault 474 opts.Headers = v2SanitizeHeaders(opts.Headers) 475 return signedURLV2(bucket, name, opts) 476 } 477} 478 479func validateOptions(opts *SignedURLOptions, now time.Time) error { 480 if opts == nil { 481 return errors.New("storage: missing required SignedURLOptions") 482 } 483 if opts.GoogleAccessID == "" { 484 return errors.New("storage: missing required GoogleAccessID") 485 } 486 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) { 487 return errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 488 } 489 opts.Method = strings.ToUpper(opts.Method) 490 if _, ok := signedURLMethods[opts.Method]; !ok { 491 return errMethodNotValid 492 } 493 if opts.Expires.IsZero() { 494 return errors.New("storage: missing required expires option") 495 } 496 if opts.MD5 != "" { 497 md5, err := base64.StdEncoding.DecodeString(opts.MD5) 498 if err != nil || len(md5) != 16 { 499 return errors.New("storage: invalid MD5 checksum") 500 } 501 } 502 if opts.Style == nil { 503 opts.Style = PathStyle() 504 } 505 if _, ok := opts.Style.(pathStyle); !ok && opts.Scheme == SigningSchemeV2 { 506 return errors.New("storage: only path-style URLs are permitted with SigningSchemeV2") 507 } 508 if opts.Scheme == SigningSchemeV4 { 509 cutoff := now.Add(604801 * time.Second) // 7 days + 1 second 510 if !opts.Expires.Before(cutoff) { 511 return errors.New("storage: expires must be within seven days from now") 512 } 513 } 514 return nil 515} 516 517const ( 518 iso8601 = "20060102T150405Z" 519 yearMonthDay = "20060102" 520) 521 522// utcNow returns the current time in UTC and is a variable to allow for 523// reassignment in tests to provide deterministic signed URL values. 524var utcNow = func() time.Time { 525 return time.Now().UTC() 526} 527 528// extractHeaderNames takes in a series of key:value headers and returns the 529// header names only. 530func extractHeaderNames(kvs []string) []string { 531 var res []string 532 for _, header := range kvs { 533 nameValue := strings.Split(header, ":") 534 res = append(res, nameValue[0]) 535 } 536 return res 537} 538 539// pathEncodeV4 creates an encoded string that matches the v4 signature spec. 540// Following the spec precisely is necessary in order to ensure that the URL 541// and signing string are correctly formed, and Go's url.PathEncode and 542// url.QueryEncode don't generate an exact match without some additional logic. 543func pathEncodeV4(path string) string { 544 segments := strings.Split(path, "/") 545 var encodedSegments []string 546 for _, s := range segments { 547 encodedSegments = append(encodedSegments, url.QueryEscape(s)) 548 } 549 encodedStr := strings.Join(encodedSegments, "/") 550 encodedStr = strings.Replace(encodedStr, "+", "%20", -1) 551 return encodedStr 552} 553 554// signedURLV4 creates a signed URL using the sigV4 algorithm. 555func signedURLV4(bucket, name string, opts *SignedURLOptions, now time.Time) (string, error) { 556 buf := &bytes.Buffer{} 557 fmt.Fprintf(buf, "%s\n", opts.Method) 558 559 u := &url.URL{Path: opts.Style.path(bucket, name)} 560 u.RawPath = pathEncodeV4(u.Path) 561 562 // Note: we have to add a / here because GCS does so auto-magically, despite 563 // our encoding not doing so (and we have to exactly match their 564 // canonical query). 565 fmt.Fprintf(buf, "/%s\n", u.RawPath) 566 567 headerNames := append(extractHeaderNames(opts.Headers), "host") 568 if opts.ContentType != "" { 569 headerNames = append(headerNames, "content-type") 570 } 571 if opts.MD5 != "" { 572 headerNames = append(headerNames, "content-md5") 573 } 574 sort.Strings(headerNames) 575 signedHeaders := strings.Join(headerNames, ";") 576 timestamp := now.Format(iso8601) 577 credentialScope := fmt.Sprintf("%s/auto/storage/goog4_request", now.Format(yearMonthDay)) 578 canonicalQueryString := url.Values{ 579 "X-Goog-Algorithm": {"GOOG4-RSA-SHA256"}, 580 "X-Goog-Credential": {fmt.Sprintf("%s/%s", opts.GoogleAccessID, credentialScope)}, 581 "X-Goog-Date": {timestamp}, 582 "X-Goog-Expires": {fmt.Sprintf("%d", int(opts.Expires.Sub(now).Seconds()))}, 583 "X-Goog-SignedHeaders": {signedHeaders}, 584 } 585 // Add user-supplied query parameters to the canonical query string. For V4, 586 // it's necessary to include these. 587 for k, v := range opts.QueryParameters { 588 canonicalQueryString[k] = append(canonicalQueryString[k], v...) 589 } 590 // url.Values.Encode escaping is correct, except that a space must be replaced 591 // by `%20` rather than `+`. 592 escapedQuery := strings.Replace(canonicalQueryString.Encode(), "+", "%20", -1) 593 fmt.Fprintf(buf, "%s\n", escapedQuery) 594 595 // Fill in the hostname based on the desired URL style. 596 u.Host = opts.Style.host(bucket) 597 598 // Fill in the URL scheme. 599 if opts.Insecure { 600 u.Scheme = "http" 601 } else { 602 u.Scheme = "https" 603 } 604 605 var headersWithValue []string 606 headersWithValue = append(headersWithValue, "host:"+u.Host) 607 headersWithValue = append(headersWithValue, opts.Headers...) 608 if opts.ContentType != "" { 609 headersWithValue = append(headersWithValue, "content-type:"+opts.ContentType) 610 } 611 if opts.MD5 != "" { 612 headersWithValue = append(headersWithValue, "content-md5:"+opts.MD5) 613 } 614 // Trim extra whitespace from headers and replace with a single space. 615 var trimmedHeaders []string 616 for _, h := range headersWithValue { 617 trimmedHeaders = append(trimmedHeaders, strings.Join(strings.Fields(h), " ")) 618 } 619 canonicalHeaders := strings.Join(sortHeadersByKey(trimmedHeaders), "\n") 620 fmt.Fprintf(buf, "%s\n\n", canonicalHeaders) 621 fmt.Fprintf(buf, "%s\n", signedHeaders) 622 623 // If the user provides a value for X-Goog-Content-SHA256, we must use 624 // that value in the request string. If not, we use UNSIGNED-PAYLOAD. 625 sha256Header := false 626 for _, h := range trimmedHeaders { 627 if strings.HasPrefix(strings.ToLower(h), "x-goog-content-sha256") && strings.Contains(h, ":") { 628 sha256Header = true 629 fmt.Fprintf(buf, "%s", strings.SplitN(h, ":", 2)[1]) 630 break 631 } 632 } 633 if !sha256Header { 634 fmt.Fprint(buf, "UNSIGNED-PAYLOAD") 635 } 636 637 sum := sha256.Sum256(buf.Bytes()) 638 hexDigest := hex.EncodeToString(sum[:]) 639 signBuf := &bytes.Buffer{} 640 fmt.Fprint(signBuf, "GOOG4-RSA-SHA256\n") 641 fmt.Fprintf(signBuf, "%s\n", timestamp) 642 fmt.Fprintf(signBuf, "%s\n", credentialScope) 643 fmt.Fprintf(signBuf, "%s", hexDigest) 644 645 signBytes := opts.SignBytes 646 if opts.PrivateKey != nil { 647 key, err := parseKey(opts.PrivateKey) 648 if err != nil { 649 return "", err 650 } 651 signBytes = func(b []byte) ([]byte, error) { 652 sum := sha256.Sum256(b) 653 return rsa.SignPKCS1v15( 654 rand.Reader, 655 key, 656 crypto.SHA256, 657 sum[:], 658 ) 659 } 660 } 661 b, err := signBytes(signBuf.Bytes()) 662 if err != nil { 663 return "", err 664 } 665 signature := hex.EncodeToString(b) 666 canonicalQueryString.Set("X-Goog-Signature", string(signature)) 667 u.RawQuery = canonicalQueryString.Encode() 668 return u.String(), nil 669} 670 671// takes a list of headerKey:headervalue1,headervalue2,etc and sorts by header 672// key. 673func sortHeadersByKey(hdrs []string) []string { 674 headersMap := map[string]string{} 675 var headersKeys []string 676 for _, h := range hdrs { 677 parts := strings.Split(h, ":") 678 k := parts[0] 679 v := parts[1] 680 headersMap[k] = v 681 headersKeys = append(headersKeys, k) 682 } 683 sort.Strings(headersKeys) 684 var sorted []string 685 for _, k := range headersKeys { 686 v := headersMap[k] 687 sorted = append(sorted, fmt.Sprintf("%s:%s", k, v)) 688 } 689 return sorted 690} 691 692func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) { 693 signBytes := opts.SignBytes 694 if opts.PrivateKey != nil { 695 key, err := parseKey(opts.PrivateKey) 696 if err != nil { 697 return "", err 698 } 699 signBytes = func(b []byte) ([]byte, error) { 700 sum := sha256.Sum256(b) 701 return rsa.SignPKCS1v15( 702 rand.Reader, 703 key, 704 crypto.SHA256, 705 sum[:], 706 ) 707 } 708 } 709 710 u := &url.URL{ 711 Path: fmt.Sprintf("/%s/%s", bucket, name), 712 } 713 714 buf := &bytes.Buffer{} 715 fmt.Fprintf(buf, "%s\n", opts.Method) 716 fmt.Fprintf(buf, "%s\n", opts.MD5) 717 fmt.Fprintf(buf, "%s\n", opts.ContentType) 718 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) 719 if len(opts.Headers) > 0 { 720 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) 721 } 722 fmt.Fprintf(buf, "%s", u.String()) 723 724 b, err := signBytes(buf.Bytes()) 725 if err != nil { 726 return "", err 727 } 728 encoded := base64.StdEncoding.EncodeToString(b) 729 u.Scheme = "https" 730 u.Host = "storage.googleapis.com" 731 q := u.Query() 732 q.Set("GoogleAccessId", opts.GoogleAccessID) 733 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) 734 q.Set("Signature", string(encoded)) 735 u.RawQuery = q.Encode() 736 return u.String(), nil 737} 738 739// ObjectHandle provides operations on an object in a Google Cloud Storage bucket. 740// Use BucketHandle.Object to get a handle. 741type ObjectHandle struct { 742 c *Client 743 bucket string 744 object string 745 acl ACLHandle 746 gen int64 // a negative value indicates latest 747 conds *Conditions 748 encryptionKey []byte // AES-256 key 749 userProject string // for requester-pays buckets 750 readCompressed bool // Accept-Encoding: gzip 751} 752 753// ACL provides access to the object's access control list. 754// This controls who can read and write this object. 755// This call does not perform any network operations. 756func (o *ObjectHandle) ACL() *ACLHandle { 757 return &o.acl 758} 759 760// Generation returns a new ObjectHandle that operates on a specific generation 761// of the object. 762// By default, the handle operates on the latest generation. Not 763// all operations work when given a specific generation; check the API 764// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. 765func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { 766 o2 := *o 767 o2.gen = gen 768 return &o2 769} 770 771// If returns a new ObjectHandle that applies a set of preconditions. 772// Preconditions already set on the ObjectHandle are ignored. 773// Operations on the new handle will return an error if the preconditions are not 774// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions 775// for more details. 776func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { 777 o2 := *o 778 o2.conds = &conds 779 return &o2 780} 781 782// Key returns a new ObjectHandle that uses the supplied encryption 783// key to encrypt and decrypt the object's contents. 784// 785// Encryption key must be a 32-byte AES-256 key. 786// See https://cloud.google.com/storage/docs/encryption for details. 787func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { 788 o2 := *o 789 o2.encryptionKey = encryptionKey 790 return &o2 791} 792 793// Attrs returns meta information about the object. 794// ErrObjectNotExist will be returned if the object is not found. 795func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { 796 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") 797 defer func() { trace.EndSpan(ctx, err) }() 798 799 if err := o.validate(); err != nil { 800 return nil, err 801 } 802 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) 803 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { 804 return nil, err 805 } 806 if o.userProject != "" { 807 call.UserProject(o.userProject) 808 } 809 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 810 return nil, err 811 } 812 var obj *raw.Object 813 setClientHeader(call.Header()) 814 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 815 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 816 return nil, ErrObjectNotExist 817 } 818 if err != nil { 819 return nil, err 820 } 821 return newObject(obj), nil 822} 823 824// Update updates an object with the provided attributes. 825// All zero-value attributes are ignored. 826// ErrObjectNotExist will be returned if the object is not found. 827func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { 828 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") 829 defer func() { trace.EndSpan(ctx, err) }() 830 831 if err := o.validate(); err != nil { 832 return nil, err 833 } 834 var attrs ObjectAttrs 835 // Lists of fields to send, and set to null, in the JSON. 836 var forceSendFields, nullFields []string 837 if uattrs.ContentType != nil { 838 attrs.ContentType = optional.ToString(uattrs.ContentType) 839 // For ContentType, sending the empty string is a no-op. 840 // Instead we send a null. 841 if attrs.ContentType == "" { 842 nullFields = append(nullFields, "ContentType") 843 } else { 844 forceSendFields = append(forceSendFields, "ContentType") 845 } 846 } 847 if uattrs.ContentLanguage != nil { 848 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) 849 // For ContentLanguage it's an error to send the empty string. 850 // Instead we send a null. 851 if attrs.ContentLanguage == "" { 852 nullFields = append(nullFields, "ContentLanguage") 853 } else { 854 forceSendFields = append(forceSendFields, "ContentLanguage") 855 } 856 } 857 if uattrs.ContentEncoding != nil { 858 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) 859 forceSendFields = append(forceSendFields, "ContentEncoding") 860 } 861 if uattrs.ContentDisposition != nil { 862 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) 863 forceSendFields = append(forceSendFields, "ContentDisposition") 864 } 865 if uattrs.CacheControl != nil { 866 attrs.CacheControl = optional.ToString(uattrs.CacheControl) 867 forceSendFields = append(forceSendFields, "CacheControl") 868 } 869 if uattrs.EventBasedHold != nil { 870 attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold) 871 forceSendFields = append(forceSendFields, "EventBasedHold") 872 } 873 if uattrs.TemporaryHold != nil { 874 attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) 875 forceSendFields = append(forceSendFields, "TemporaryHold") 876 } 877 if !uattrs.CustomTime.IsZero() { 878 attrs.CustomTime = uattrs.CustomTime 879 forceSendFields = append(forceSendFields, "CustomTime") 880 } 881 if uattrs.Metadata != nil { 882 attrs.Metadata = uattrs.Metadata 883 if len(attrs.Metadata) == 0 { 884 // Sending the empty map is a no-op. We send null instead. 885 nullFields = append(nullFields, "Metadata") 886 } else { 887 forceSendFields = append(forceSendFields, "Metadata") 888 } 889 } 890 if uattrs.ACL != nil { 891 attrs.ACL = uattrs.ACL 892 // It's an error to attempt to delete the ACL, so 893 // we don't append to nullFields here. 894 forceSendFields = append(forceSendFields, "Acl") 895 } 896 rawObj := attrs.toRawObject(o.bucket) 897 rawObj.ForceSendFields = forceSendFields 898 rawObj.NullFields = nullFields 899 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) 900 if err := applyConds("Update", o.gen, o.conds, call); err != nil { 901 return nil, err 902 } 903 if o.userProject != "" { 904 call.UserProject(o.userProject) 905 } 906 if uattrs.PredefinedACL != "" { 907 call.PredefinedAcl(uattrs.PredefinedACL) 908 } 909 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 910 return nil, err 911 } 912 var obj *raw.Object 913 setClientHeader(call.Header()) 914 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 915 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 916 return nil, ErrObjectNotExist 917 } 918 if err != nil { 919 return nil, err 920 } 921 return newObject(obj), nil 922} 923 924// BucketName returns the name of the bucket. 925func (o *ObjectHandle) BucketName() string { 926 return o.bucket 927} 928 929// ObjectName returns the name of the object. 930func (o *ObjectHandle) ObjectName() string { 931 return o.object 932} 933 934// ObjectAttrsToUpdate is used to update the attributes of an object. 935// Only fields set to non-nil values will be updated. 936// Set a field to its zero value to delete it. 937// 938// For example, to change ContentType and delete ContentEncoding and 939// Metadata, use 940// ObjectAttrsToUpdate{ 941// ContentType: "text/html", 942// ContentEncoding: "", 943// Metadata: map[string]string{}, 944// } 945type ObjectAttrsToUpdate struct { 946 EventBasedHold optional.Bool 947 TemporaryHold optional.Bool 948 ContentType optional.String 949 ContentLanguage optional.String 950 ContentEncoding optional.String 951 ContentDisposition optional.String 952 CacheControl optional.String 953 CustomTime time.Time 954 Metadata map[string]string // set to map[string]string{} to delete 955 ACL []ACLRule 956 957 // If not empty, applies a predefined set of access controls. ACL must be nil. 958 // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch. 959 PredefinedACL string 960} 961 962// Delete deletes the single specified object. 963func (o *ObjectHandle) Delete(ctx context.Context) error { 964 if err := o.validate(); err != nil { 965 return err 966 } 967 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) 968 if err := applyConds("Delete", o.gen, o.conds, call); err != nil { 969 return err 970 } 971 if o.userProject != "" { 972 call.UserProject(o.userProject) 973 } 974 // Encryption doesn't apply to Delete. 975 setClientHeader(call.Header()) 976 err := runWithRetry(ctx, func() error { return call.Do() }) 977 switch e := err.(type) { 978 case nil: 979 return nil 980 case *googleapi.Error: 981 if e.Code == http.StatusNotFound { 982 return ErrObjectNotExist 983 } 984 } 985 return err 986} 987 988// ReadCompressed when true causes the read to happen without decompressing. 989func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { 990 o2 := *o 991 o2.readCompressed = compressed 992 return &o2 993} 994 995// NewWriter returns a storage Writer that writes to the GCS object 996// associated with this ObjectHandle. 997// 998// A new object will be created unless an object with this name already exists. 999// Otherwise any previous object with the same name will be replaced. 1000// The object will not be available (and any previous object will remain) 1001// until Close has been called. 1002// 1003// Attributes can be set on the object by modifying the returned Writer's 1004// ObjectAttrs field before the first call to Write. If no ContentType 1005// attribute is specified, the content type will be automatically sniffed 1006// using net/http.DetectContentType. 1007// 1008// It is the caller's responsibility to call Close when writing is done. To 1009// stop writing without saving the data, cancel the context. 1010func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { 1011 return &Writer{ 1012 ctx: ctx, 1013 o: o, 1014 donec: make(chan struct{}), 1015 ObjectAttrs: ObjectAttrs{Name: o.object}, 1016 ChunkSize: googleapi.DefaultUploadChunkSize, 1017 } 1018} 1019 1020func (o *ObjectHandle) validate() error { 1021 if o.bucket == "" { 1022 return errors.New("storage: bucket name is empty") 1023 } 1024 if o.object == "" { 1025 return errors.New("storage: object name is empty") 1026 } 1027 if !utf8.ValidString(o.object) { 1028 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) 1029 } 1030 return nil 1031} 1032 1033// parseKey converts the binary contents of a private key file to an 1034// *rsa.PrivateKey. It detects whether the private key is in a PEM container or 1035// not. If so, it extracts the private key from PEM container before 1036// conversion. It only supports PEM containers with no passphrase. 1037func parseKey(key []byte) (*rsa.PrivateKey, error) { 1038 if block, _ := pem.Decode(key); block != nil { 1039 key = block.Bytes 1040 } 1041 parsedKey, err := x509.ParsePKCS8PrivateKey(key) 1042 if err != nil { 1043 parsedKey, err = x509.ParsePKCS1PrivateKey(key) 1044 if err != nil { 1045 return nil, err 1046 } 1047 } 1048 parsed, ok := parsedKey.(*rsa.PrivateKey) 1049 if !ok { 1050 return nil, errors.New("oauth2: private key is invalid") 1051 } 1052 return parsed, nil 1053} 1054 1055// toRawObject copies the editable attributes from o to the raw library's Object type. 1056func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { 1057 var ret string 1058 if !o.RetentionExpirationTime.IsZero() { 1059 ret = o.RetentionExpirationTime.Format(time.RFC3339) 1060 } 1061 var ct string 1062 if !o.CustomTime.IsZero() { 1063 ct = o.CustomTime.Format(time.RFC3339) 1064 } 1065 return &raw.Object{ 1066 Bucket: bucket, 1067 Name: o.Name, 1068 EventBasedHold: o.EventBasedHold, 1069 TemporaryHold: o.TemporaryHold, 1070 RetentionExpirationTime: ret, 1071 ContentType: o.ContentType, 1072 ContentEncoding: o.ContentEncoding, 1073 ContentLanguage: o.ContentLanguage, 1074 CacheControl: o.CacheControl, 1075 ContentDisposition: o.ContentDisposition, 1076 StorageClass: o.StorageClass, 1077 Acl: toRawObjectACL(o.ACL), 1078 Metadata: o.Metadata, 1079 CustomTime: ct, 1080 } 1081} 1082 1083// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. 1084type ObjectAttrs struct { 1085 // Bucket is the name of the bucket containing this GCS object. 1086 // This field is read-only. 1087 Bucket string 1088 1089 // Name is the name of the object within the bucket. 1090 // This field is read-only. 1091 Name string 1092 1093 // ContentType is the MIME type of the object's content. 1094 ContentType string 1095 1096 // ContentLanguage is the content language of the object's content. 1097 ContentLanguage string 1098 1099 // CacheControl is the Cache-Control header to be sent in the response 1100 // headers when serving the object data. 1101 CacheControl string 1102 1103 // EventBasedHold specifies whether an object is under event-based hold. New 1104 // objects created in a bucket whose DefaultEventBasedHold is set will 1105 // default to that value. 1106 EventBasedHold bool 1107 1108 // TemporaryHold specifies whether an object is under temporary hold. While 1109 // this flag is set to true, the object is protected against deletion and 1110 // overwrites. 1111 TemporaryHold bool 1112 1113 // RetentionExpirationTime is a server-determined value that specifies the 1114 // earliest time that the object's retention period expires. 1115 // This is a read-only field. 1116 RetentionExpirationTime time.Time 1117 1118 // ACL is the list of access control rules for the object. 1119 ACL []ACLRule 1120 1121 // If not empty, applies a predefined set of access controls. It should be set 1122 // only when writing, copying or composing an object. When copying or composing, 1123 // it acts as the destinationPredefinedAcl parameter. 1124 // PredefinedACL is always empty for ObjectAttrs returned from the service. 1125 // See https://cloud.google.com/storage/docs/json_api/v1/objects/insert 1126 // for valid values. 1127 PredefinedACL string 1128 1129 // Owner is the owner of the object. This field is read-only. 1130 // 1131 // If non-zero, it is in the form of "user-<userId>". 1132 Owner string 1133 1134 // Size is the length of the object's content. This field is read-only. 1135 Size int64 1136 1137 // ContentEncoding is the encoding of the object's content. 1138 ContentEncoding string 1139 1140 // ContentDisposition is the optional Content-Disposition header of the object 1141 // sent in the response headers. 1142 ContentDisposition string 1143 1144 // MD5 is the MD5 hash of the object's content. This field is read-only, 1145 // except when used from a Writer. If set on a Writer, the uploaded 1146 // data is rejected if its MD5 hash does not match this field. 1147 MD5 []byte 1148 1149 // CRC32C is the CRC32 checksum of the object's content using the Castagnoli93 1150 // polynomial. This field is read-only, except when used from a Writer or 1151 // Composer. In those cases, if the SendCRC32C field in the Writer or Composer 1152 // is set to is true, the uploaded data is rejected if its CRC32C hash does 1153 // not match this field. 1154 CRC32C uint32 1155 1156 // MediaLink is an URL to the object's content. This field is read-only. 1157 MediaLink string 1158 1159 // Metadata represents user-provided metadata, in key/value pairs. 1160 // It can be nil if no metadata is provided. 1161 Metadata map[string]string 1162 1163 // Generation is the generation number of the object's content. 1164 // This field is read-only. 1165 Generation int64 1166 1167 // Metageneration is the version of the metadata for this 1168 // object at this generation. This field is used for preconditions 1169 // and for detecting changes in metadata. A metageneration number 1170 // is only meaningful in the context of a particular generation 1171 // of a particular object. This field is read-only. 1172 Metageneration int64 1173 1174 // StorageClass is the storage class of the object. This defines 1175 // how objects are stored and determines the SLA and the cost of storage. 1176 // Typical values are "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE". 1177 // Defaults to "STANDARD". 1178 // See https://cloud.google.com/storage/docs/storage-classes for all 1179 // valid values. 1180 StorageClass string 1181 1182 // Created is the time the object was created. This field is read-only. 1183 Created time.Time 1184 1185 // Deleted is the time the object was deleted. 1186 // If not deleted, it is the zero value. This field is read-only. 1187 Deleted time.Time 1188 1189 // Updated is the creation or modification time of the object. 1190 // For buckets with versioning enabled, changing an object's 1191 // metadata does not change this property. This field is read-only. 1192 Updated time.Time 1193 1194 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the 1195 // customer-supplied encryption key for the object. It is empty if there is 1196 // no customer-supplied encryption key. 1197 // See // https://cloud.google.com/storage/docs/encryption for more about 1198 // encryption in Google Cloud Storage. 1199 CustomerKeySHA256 string 1200 1201 // Cloud KMS key name, in the form 1202 // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, 1203 // if the object is encrypted by such a key. 1204 // 1205 // Providing both a KMSKeyName and a customer-supplied encryption key (via 1206 // ObjectHandle.Key) will result in an error when writing an object. 1207 KMSKeyName string 1208 1209 // Prefix is set only for ObjectAttrs which represent synthetic "directory 1210 // entries" when iterating over buckets using Query.Delimiter. See 1211 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be 1212 // populated. 1213 Prefix string 1214 1215 // Etag is the HTTP/1.1 Entity tag for the object. 1216 // This field is read-only. 1217 Etag string 1218 1219 // A user-specified timestamp which can be applied to an object. This is 1220 // typically set in order to use the CustomTimeBefore and DaysSinceCustomTime 1221 // LifecycleConditions to manage object lifecycles. 1222 // 1223 // CustomTime cannot be removed once set on an object. It can be updated to a 1224 // later value but not to an earlier one. 1225 CustomTime time.Time 1226} 1227 1228// convertTime converts a time in RFC3339 format to time.Time. 1229// If any error occurs in parsing, the zero-value time.Time is silently returned. 1230func convertTime(t string) time.Time { 1231 var r time.Time 1232 if t != "" { 1233 r, _ = time.Parse(time.RFC3339, t) 1234 } 1235 return r 1236} 1237 1238func newObject(o *raw.Object) *ObjectAttrs { 1239 if o == nil { 1240 return nil 1241 } 1242 owner := "" 1243 if o.Owner != nil { 1244 owner = o.Owner.Entity 1245 } 1246 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) 1247 crc32c, _ := decodeUint32(o.Crc32c) 1248 var sha256 string 1249 if o.CustomerEncryption != nil { 1250 sha256 = o.CustomerEncryption.KeySha256 1251 } 1252 return &ObjectAttrs{ 1253 Bucket: o.Bucket, 1254 Name: o.Name, 1255 ContentType: o.ContentType, 1256 ContentLanguage: o.ContentLanguage, 1257 CacheControl: o.CacheControl, 1258 EventBasedHold: o.EventBasedHold, 1259 TemporaryHold: o.TemporaryHold, 1260 RetentionExpirationTime: convertTime(o.RetentionExpirationTime), 1261 ACL: toObjectACLRules(o.Acl), 1262 Owner: owner, 1263 ContentEncoding: o.ContentEncoding, 1264 ContentDisposition: o.ContentDisposition, 1265 Size: int64(o.Size), 1266 MD5: md5, 1267 CRC32C: crc32c, 1268 MediaLink: o.MediaLink, 1269 Metadata: o.Metadata, 1270 Generation: o.Generation, 1271 Metageneration: o.Metageneration, 1272 StorageClass: o.StorageClass, 1273 CustomerKeySHA256: sha256, 1274 KMSKeyName: o.KmsKeyName, 1275 Created: convertTime(o.TimeCreated), 1276 Deleted: convertTime(o.TimeDeleted), 1277 Updated: convertTime(o.Updated), 1278 Etag: o.Etag, 1279 CustomTime: convertTime(o.CustomTime), 1280 } 1281} 1282 1283// Decode a uint32 encoded in Base64 in big-endian byte order. 1284func decodeUint32(b64 string) (uint32, error) { 1285 d, err := base64.StdEncoding.DecodeString(b64) 1286 if err != nil { 1287 return 0, err 1288 } 1289 if len(d) != 4 { 1290 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) 1291 } 1292 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil 1293} 1294 1295// Encode a uint32 as Base64 in big-endian byte order. 1296func encodeUint32(u uint32) string { 1297 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} 1298 return base64.StdEncoding.EncodeToString(b) 1299} 1300 1301// Query represents a query to filter objects from a bucket. 1302type Query struct { 1303 // Delimiter returns results in a directory-like fashion. 1304 // Results will contain only objects whose names, aside from the 1305 // prefix, do not contain delimiter. Objects whose names, 1306 // aside from the prefix, contain delimiter will have their name, 1307 // truncated after the delimiter, returned in prefixes. 1308 // Duplicate prefixes are omitted. 1309 // Optional. 1310 Delimiter string 1311 1312 // Prefix is the prefix filter to query objects 1313 // whose names begin with this prefix. 1314 // Optional. 1315 Prefix string 1316 1317 // Versions indicates whether multiple versions of the same 1318 // object will be included in the results. 1319 Versions bool 1320 1321 // fieldSelection is used to select only specific fields to be returned by 1322 // the query. It's used internally and is populated for the user by 1323 // calling Query.SetAttrSelection 1324 fieldSelection string 1325 1326 // StartOffset is used to filter results to objects whose names are 1327 // lexicographically equal to or after startOffset. If endOffset is also set, 1328 // the objects listed will have names between startOffset (inclusive) and 1329 // endOffset (exclusive). 1330 StartOffset string 1331 1332 // EndOffset is used to filter results to objects whose names are 1333 // lexicographically before endOffset. If startOffset is also set, the objects 1334 // listed will have names between startOffset (inclusive) and endOffset (exclusive). 1335 EndOffset string 1336} 1337 1338// attrToFieldMap maps the field names of ObjectAttrs to the underlying field 1339// names in the API call. Only the ObjectAttrs field names are visible to users 1340// because they are already part of the public API of the package. 1341var attrToFieldMap = map[string]string{ 1342 "Bucket": "bucket", 1343 "Name": "name", 1344 "ContentType": "contentType", 1345 "ContentLanguage": "contentLanguage", 1346 "CacheControl": "cacheControl", 1347 "EventBasedHold": "eventBasedHold", 1348 "TemporaryHold": "temporaryHold", 1349 "RetentionExpirationTime": "retentionExpirationTime", 1350 "ACL": "acl", 1351 "Owner": "owner", 1352 "ContentEncoding": "contentEncoding", 1353 "ContentDisposition": "contentDisposition", 1354 "Size": "size", 1355 "MD5": "md5Hash", 1356 "CRC32C": "crc32c", 1357 "MediaLink": "mediaLink", 1358 "Metadata": "metadata", 1359 "Generation": "generation", 1360 "Metageneration": "metageneration", 1361 "StorageClass": "storageClass", 1362 "CustomerKeySHA256": "customerEncryption", 1363 "KMSKeyName": "kmsKeyName", 1364 "Created": "timeCreated", 1365 "Deleted": "timeDeleted", 1366 "Updated": "updated", 1367 "Etag": "etag", 1368 "CustomTime": "customTime", 1369} 1370 1371// SetAttrSelection makes the query populate only specific attributes of 1372// objects. When iterating over objects, if you only need each object's name 1373// and size, pass []string{"Name", "Size"} to this method. Only these fields 1374// will be fetched for each object across the network; the other fields of 1375// ObjectAttr will remain at their default values. This is a performance 1376// optimization; for more information, see 1377// https://cloud.google.com/storage/docs/json_api/v1/how-tos/performance 1378func (q *Query) SetAttrSelection(attrs []string) error { 1379 fieldSet := make(map[string]bool) 1380 1381 for _, attr := range attrs { 1382 field, ok := attrToFieldMap[attr] 1383 if !ok { 1384 return fmt.Errorf("storage: attr %v is not valid", attr) 1385 } 1386 fieldSet[field] = true 1387 } 1388 1389 if len(fieldSet) > 0 { 1390 var b bytes.Buffer 1391 b.WriteString("prefixes,items(") 1392 first := true 1393 for field := range fieldSet { 1394 if !first { 1395 b.WriteString(",") 1396 } 1397 first = false 1398 b.WriteString(field) 1399 } 1400 b.WriteString(")") 1401 q.fieldSelection = b.String() 1402 } 1403 return nil 1404} 1405 1406// Conditions constrain methods to act on specific generations of 1407// objects. 1408// 1409// The zero value is an empty set of constraints. Not all conditions or 1410// combinations of conditions are applicable to all methods. 1411// See https://cloud.google.com/storage/docs/generations-preconditions 1412// for details on how these operate. 1413type Conditions struct { 1414 // Generation constraints. 1415 // At most one of the following can be set to a non-zero value. 1416 1417 // GenerationMatch specifies that the object must have the given generation 1418 // for the operation to occur. 1419 // If GenerationMatch is zero, it has no effect. 1420 // Use DoesNotExist to specify that the object does not exist in the bucket. 1421 GenerationMatch int64 1422 1423 // GenerationNotMatch specifies that the object must not have the given 1424 // generation for the operation to occur. 1425 // If GenerationNotMatch is zero, it has no effect. 1426 GenerationNotMatch int64 1427 1428 // DoesNotExist specifies that the object must not exist in the bucket for 1429 // the operation to occur. 1430 // If DoesNotExist is false, it has no effect. 1431 DoesNotExist bool 1432 1433 // Metadata generation constraints. 1434 // At most one of the following can be set to a non-zero value. 1435 1436 // MetagenerationMatch specifies that the object must have the given 1437 // metageneration for the operation to occur. 1438 // If MetagenerationMatch is zero, it has no effect. 1439 MetagenerationMatch int64 1440 1441 // MetagenerationNotMatch specifies that the object must not have the given 1442 // metageneration for the operation to occur. 1443 // If MetagenerationNotMatch is zero, it has no effect. 1444 MetagenerationNotMatch int64 1445} 1446 1447func (c *Conditions) validate(method string) error { 1448 if *c == (Conditions{}) { 1449 return fmt.Errorf("storage: %s: empty conditions", method) 1450 } 1451 if !c.isGenerationValid() { 1452 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) 1453 } 1454 if !c.isMetagenerationValid() { 1455 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 1456 } 1457 return nil 1458} 1459 1460func (c *Conditions) isGenerationValid() bool { 1461 n := 0 1462 if c.GenerationMatch != 0 { 1463 n++ 1464 } 1465 if c.GenerationNotMatch != 0 { 1466 n++ 1467 } 1468 if c.DoesNotExist { 1469 n++ 1470 } 1471 return n <= 1 1472} 1473 1474func (c *Conditions) isMetagenerationValid() bool { 1475 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 1476} 1477 1478// applyConds modifies the provided call using the conditions in conds. 1479// call is something that quacks like a *raw.WhateverCall. 1480func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { 1481 cval := reflect.ValueOf(call) 1482 if gen >= 0 { 1483 if !setConditionField(cval, "Generation", gen) { 1484 return fmt.Errorf("storage: %s: generation not supported", method) 1485 } 1486 } 1487 if conds == nil { 1488 return nil 1489 } 1490 if err := conds.validate(method); err != nil { 1491 return err 1492 } 1493 switch { 1494 case conds.GenerationMatch != 0: 1495 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { 1496 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) 1497 } 1498 case conds.GenerationNotMatch != 0: 1499 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { 1500 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) 1501 } 1502 case conds.DoesNotExist: 1503 if !setConditionField(cval, "IfGenerationMatch", int64(0)) { 1504 return fmt.Errorf("storage: %s: DoesNotExist not supported", method) 1505 } 1506 } 1507 switch { 1508 case conds.MetagenerationMatch != 0: 1509 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 1510 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 1511 } 1512 case conds.MetagenerationNotMatch != 0: 1513 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 1514 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 1515 } 1516 } 1517 return nil 1518} 1519 1520func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { 1521 if gen >= 0 { 1522 call.SourceGeneration(gen) 1523 } 1524 if conds == nil { 1525 return nil 1526 } 1527 if err := conds.validate("CopyTo source"); err != nil { 1528 return err 1529 } 1530 switch { 1531 case conds.GenerationMatch != 0: 1532 call.IfSourceGenerationMatch(conds.GenerationMatch) 1533 case conds.GenerationNotMatch != 0: 1534 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) 1535 case conds.DoesNotExist: 1536 call.IfSourceGenerationMatch(0) 1537 } 1538 switch { 1539 case conds.MetagenerationMatch != 0: 1540 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) 1541 case conds.MetagenerationNotMatch != 0: 1542 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) 1543 } 1544 return nil 1545} 1546 1547// setConditionField sets a field on a *raw.WhateverCall. 1548// We can't use anonymous interfaces because the return type is 1549// different, since the field setters are builders. 1550func setConditionField(call reflect.Value, name string, value interface{}) bool { 1551 m := call.MethodByName(name) 1552 if !m.IsValid() { 1553 return false 1554 } 1555 m.Call([]reflect.Value{reflect.ValueOf(value)}) 1556 return true 1557} 1558 1559// conditionsQuery returns the generation and conditions as a URL query 1560// string suitable for URL.RawQuery. It assumes that the conditions 1561// have been validated. 1562func conditionsQuery(gen int64, conds *Conditions) string { 1563 // URL escapes are elided because integer strings are URL-safe. 1564 var buf []byte 1565 1566 appendParam := func(s string, n int64) { 1567 if len(buf) > 0 { 1568 buf = append(buf, '&') 1569 } 1570 buf = append(buf, s...) 1571 buf = strconv.AppendInt(buf, n, 10) 1572 } 1573 1574 if gen >= 0 { 1575 appendParam("generation=", gen) 1576 } 1577 if conds == nil { 1578 return string(buf) 1579 } 1580 switch { 1581 case conds.GenerationMatch != 0: 1582 appendParam("ifGenerationMatch=", conds.GenerationMatch) 1583 case conds.GenerationNotMatch != 0: 1584 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) 1585 case conds.DoesNotExist: 1586 appendParam("ifGenerationMatch=", 0) 1587 } 1588 switch { 1589 case conds.MetagenerationMatch != 0: 1590 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) 1591 case conds.MetagenerationNotMatch != 0: 1592 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) 1593 } 1594 return string(buf) 1595} 1596 1597// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods 1598// that modifyCall searches for by name. 1599type composeSourceObj struct { 1600 src *raw.ComposeRequestSourceObjects 1601} 1602 1603func (c composeSourceObj) Generation(gen int64) { 1604 c.src.Generation = gen 1605} 1606 1607func (c composeSourceObj) IfGenerationMatch(gen int64) { 1608 // It's safe to overwrite ObjectPreconditions, since its only field is 1609 // IfGenerationMatch. 1610 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{ 1611 IfGenerationMatch: gen, 1612 } 1613} 1614 1615func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error { 1616 if key == nil { 1617 return nil 1618 } 1619 // TODO(jbd): Ask the API team to return a more user-friendly error 1620 // and avoid doing this check at the client level. 1621 if len(key) != 32 { 1622 return errors.New("storage: not a 32-byte AES-256 key") 1623 } 1624 var cs string 1625 if copySource { 1626 cs = "copy-source-" 1627 } 1628 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") 1629 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) 1630 keyHash := sha256.Sum256(key) 1631 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) 1632 return nil 1633} 1634 1635// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. 1636func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { 1637 r := c.raw.Projects.ServiceAccount.Get(projectID) 1638 res, err := r.Context(ctx).Do() 1639 if err != nil { 1640 return "", err 1641 } 1642 return res.EmailAddress, nil 1643} 1644