1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "context" 20 "crypto" 21 "crypto/rand" 22 "crypto/rsa" 23 "crypto/sha256" 24 "crypto/x509" 25 "encoding/base64" 26 "encoding/pem" 27 "errors" 28 "fmt" 29 "io" 30 "net/http" 31 "net/url" 32 "reflect" 33 "regexp" 34 "sort" 35 "strconv" 36 "strings" 37 "time" 38 "unicode/utf8" 39 40 "cloud.google.com/go/internal/optional" 41 "cloud.google.com/go/internal/trace" 42 "cloud.google.com/go/internal/version" 43 "google.golang.org/api/googleapi" 44 "google.golang.org/api/option" 45 raw "google.golang.org/api/storage/v1" 46 htransport "google.golang.org/api/transport/http" 47) 48 49var ( 50 // ErrBucketNotExist indicates that the bucket does not exist. 51 ErrBucketNotExist = errors.New("storage: bucket doesn't exist") 52 // ErrObjectNotExist indicates that the object does not exist. 53 ErrObjectNotExist = errors.New("storage: object doesn't exist") 54) 55 56const userAgent = "gcloud-golang-storage/20151204" 57 58const ( 59 // ScopeFullControl grants permissions to manage your 60 // data and permissions in Google Cloud Storage. 61 ScopeFullControl = raw.DevstorageFullControlScope 62 63 // ScopeReadOnly grants permissions to 64 // view your data in Google Cloud Storage. 65 ScopeReadOnly = raw.DevstorageReadOnlyScope 66 67 // ScopeReadWrite grants permissions to manage your 68 // data in Google Cloud Storage. 69 ScopeReadWrite = raw.DevstorageReadWriteScope 70) 71 72var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) 73 74func setClientHeader(headers http.Header) { 75 headers.Set("x-goog-api-client", xGoogHeader) 76} 77 78// Client is a client for interacting with Google Cloud Storage. 79// 80// Clients should be reused instead of created as needed. 81// The methods of Client are safe for concurrent use by multiple goroutines. 82type Client struct { 83 hc *http.Client 84 raw *raw.Service 85} 86 87// NewClient creates a new Google Cloud Storage client. 88// The default scope is ScopeFullControl. To use a different scope, like ScopeReadOnly, use option.WithScopes. 89func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 90 o := []option.ClientOption{ 91 option.WithScopes(ScopeFullControl), 92 option.WithUserAgent(userAgent), 93 } 94 opts = append(o, opts...) 95 hc, ep, err := htransport.NewClient(ctx, opts...) 96 if err != nil { 97 return nil, fmt.Errorf("dialing: %v", err) 98 } 99 rawService, err := raw.New(hc) 100 if err != nil { 101 return nil, fmt.Errorf("storage client: %v", err) 102 } 103 if ep != "" { 104 rawService.BasePath = ep 105 } 106 return &Client{ 107 hc: hc, 108 raw: rawService, 109 }, nil 110} 111 112// Close closes the Client. 113// 114// Close need not be called at program exit. 115func (c *Client) Close() error { 116 // Set fields to nil so that subsequent uses will panic. 117 c.hc = nil 118 c.raw = nil 119 return nil 120} 121 122// SignedURLOptions allows you to restrict the access to the signed URL. 123type SignedURLOptions struct { 124 // GoogleAccessID represents the authorizer of the signed URL generation. 125 // It is typically the Google service account client email address from 126 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 127 // Required. 128 GoogleAccessID string 129 130 // PrivateKey is the Google service account private key. It is obtainable 131 // from the Google Developers Console. 132 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 133 // create a service account client ID or reuse one of your existing service account 134 // credentials. Click on the "Generate new P12 key" to generate and download 135 // a new private key. Once you download the P12 file, use the following command 136 // to convert it into a PEM file. 137 // 138 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 139 // 140 // Provide the contents of the PEM file as a byte slice. 141 // Exactly one of PrivateKey or SignBytes must be non-nil. 142 PrivateKey []byte 143 144 // SignBytes is a function for implementing custom signing. 145 // If your application is running on Google App Engine, you can use appengine's internal signing function: 146 // ctx := appengine.NewContext(request) 147 // acc, _ := appengine.ServiceAccount(ctx) 148 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 149 // GoogleAccessID: acc, 150 // SignBytes: func(b []byte) ([]byte, error) { 151 // _, signedBytes, err := appengine.SignBytes(ctx, b) 152 // return signedBytes, err 153 // }, 154 // // etc. 155 // }) 156 // 157 // Exactly one of PrivateKey or SignBytes must be non-nil. 158 SignBytes func([]byte) ([]byte, error) 159 160 // Method is the HTTP method to be used with the signed URL. 161 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. 162 // Required. 163 Method string 164 165 // Expires is the expiration time on the signed URL. It must be 166 // a datetime in the future. 167 // Required. 168 Expires time.Time 169 170 // ContentType is the content type header the client must provide 171 // to use the generated signed URL. 172 // Optional. 173 ContentType string 174 175 // Headers is a list of extension headers the client must provide 176 // in order to use the generated signed URL. 177 // Optional. 178 Headers []string 179 180 // MD5 is the base64 encoded MD5 checksum of the file. 181 // If provided, the client should provide the exact value on the request 182 // header in order to use the signed URL. 183 // Optional. 184 MD5 string 185} 186 187var ( 188 canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) 189 excludedCanonicalHeaders = map[string]bool{ 190 "x-goog-encryption-key": true, 191 "x-goog-encryption-key-sha256": true, 192 } 193) 194 195// sanitizeHeaders applies the specifications for canonical extension headers at 196// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. 197func sanitizeHeaders(hdrs []string) []string { 198 headerMap := map[string][]string{} 199 for _, hdr := range hdrs { 200 // No leading or trailing whitespaces. 201 sanitizedHeader := strings.TrimSpace(hdr) 202 203 // Only keep canonical headers, discard any others. 204 headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) 205 if len(headerMatches) == 0 { 206 continue 207 } 208 209 header := strings.ToLower(strings.TrimSpace(headerMatches[1])) 210 if excludedCanonicalHeaders[headerMatches[1]] { 211 // Do not keep any deliberately excluded canonical headers when signing. 212 continue 213 } 214 value := strings.TrimSpace(headerMatches[2]) 215 if len(value) > 0 { 216 // Remove duplicate headers by appending the values of duplicates 217 // in their order of appearance. 218 headerMap[header] = append(headerMap[header], value) 219 } 220 } 221 222 var sanitizedHeaders []string 223 for header, values := range headerMap { 224 // There should be no spaces around the colon separating the 225 // header name from the header value or around the values 226 // themselves. The values should be separated by commas. 227 // NOTE: The semantics for headers without a value are not clear. 228 // However from specifications these should be edge-cases 229 // anyway and we should assume that there will be no 230 // canonical headers using empty values. Any such headers 231 // are discarded at the regexp stage above. 232 sanitizedHeaders = append( 233 sanitizedHeaders, 234 fmt.Sprintf("%s:%s", header, strings.Join(values, ",")), 235 ) 236 } 237 sort.Strings(sanitizedHeaders) 238 return sanitizedHeaders 239} 240 241// SignedURL returns a URL for the specified object. Signed URLs allow 242// the users access to a restricted resource for a limited time without having a 243// Google account or signing in. For more information about the signed 244// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. 245func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { 246 if opts == nil { 247 return "", errors.New("storage: missing required SignedURLOptions") 248 } 249 if opts.GoogleAccessID == "" { 250 return "", errors.New("storage: missing required GoogleAccessID") 251 } 252 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) { 253 return "", errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 254 } 255 if opts.Method == "" { 256 return "", errors.New("storage: missing required method option") 257 } 258 if opts.Expires.IsZero() { 259 return "", errors.New("storage: missing required expires option") 260 } 261 if opts.MD5 != "" { 262 md5, err := base64.StdEncoding.DecodeString(opts.MD5) 263 if err != nil || len(md5) != 16 { 264 return "", errors.New("storage: invalid MD5 checksum") 265 } 266 } 267 opts.Headers = sanitizeHeaders(opts.Headers) 268 269 signBytes := opts.SignBytes 270 if opts.PrivateKey != nil { 271 key, err := parseKey(opts.PrivateKey) 272 if err != nil { 273 return "", err 274 } 275 signBytes = func(b []byte) ([]byte, error) { 276 sum := sha256.Sum256(b) 277 return rsa.SignPKCS1v15( 278 rand.Reader, 279 key, 280 crypto.SHA256, 281 sum[:], 282 ) 283 } 284 } 285 286 u := &url.URL{ 287 Path: fmt.Sprintf("/%s/%s", bucket, name), 288 } 289 290 buf := &bytes.Buffer{} 291 fmt.Fprintf(buf, "%s\n", opts.Method) 292 fmt.Fprintf(buf, "%s\n", opts.MD5) 293 fmt.Fprintf(buf, "%s\n", opts.ContentType) 294 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) 295 if len(opts.Headers) > 0 { 296 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) 297 } 298 fmt.Fprintf(buf, "%s", u.String()) 299 300 b, err := signBytes(buf.Bytes()) 301 if err != nil { 302 return "", err 303 } 304 encoded := base64.StdEncoding.EncodeToString(b) 305 u.Scheme = "https" 306 u.Host = "storage.googleapis.com" 307 q := u.Query() 308 q.Set("GoogleAccessId", opts.GoogleAccessID) 309 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) 310 q.Set("Signature", string(encoded)) 311 u.RawQuery = q.Encode() 312 return u.String(), nil 313} 314 315// ObjectHandle provides operations on an object in a Google Cloud Storage bucket. 316// Use BucketHandle.Object to get a handle. 317type ObjectHandle struct { 318 c *Client 319 bucket string 320 object string 321 acl ACLHandle 322 gen int64 // a negative value indicates latest 323 conds *Conditions 324 encryptionKey []byte // AES-256 key 325 userProject string // for requester-pays buckets 326 readCompressed bool // Accept-Encoding: gzip 327} 328 329// ACL provides access to the object's access control list. 330// This controls who can read and write this object. 331// This call does not perform any network operations. 332func (o *ObjectHandle) ACL() *ACLHandle { 333 return &o.acl 334} 335 336// Generation returns a new ObjectHandle that operates on a specific generation 337// of the object. 338// By default, the handle operates on the latest generation. Not 339// all operations work when given a specific generation; check the API 340// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. 341func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { 342 o2 := *o 343 o2.gen = gen 344 return &o2 345} 346 347// If returns a new ObjectHandle that applies a set of preconditions. 348// Preconditions already set on the ObjectHandle are ignored. 349// Operations on the new handle will return an error if the preconditions are not 350// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions 351// for more details. 352func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { 353 o2 := *o 354 o2.conds = &conds 355 return &o2 356} 357 358// Key returns a new ObjectHandle that uses the supplied encryption 359// key to encrypt and decrypt the object's contents. 360// 361// Encryption key must be a 32-byte AES-256 key. 362// See https://cloud.google.com/storage/docs/encryption for details. 363func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { 364 o2 := *o 365 o2.encryptionKey = encryptionKey 366 return &o2 367} 368 369// Attrs returns meta information about the object. 370// ErrObjectNotExist will be returned if the object is not found. 371func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { 372 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") 373 defer func() { trace.EndSpan(ctx, err) }() 374 375 if err := o.validate(); err != nil { 376 return nil, err 377 } 378 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) 379 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { 380 return nil, err 381 } 382 if o.userProject != "" { 383 call.UserProject(o.userProject) 384 } 385 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 386 return nil, err 387 } 388 var obj *raw.Object 389 setClientHeader(call.Header()) 390 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 391 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 392 return nil, ErrObjectNotExist 393 } 394 if err != nil { 395 return nil, err 396 } 397 return newObject(obj), nil 398} 399 400// Update updates an object with the provided attributes. 401// All zero-value attributes are ignored. 402// ErrObjectNotExist will be returned if the object is not found. 403func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { 404 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") 405 defer func() { trace.EndSpan(ctx, err) }() 406 407 if err := o.validate(); err != nil { 408 return nil, err 409 } 410 var attrs ObjectAttrs 411 // Lists of fields to send, and set to null, in the JSON. 412 var forceSendFields, nullFields []string 413 if uattrs.ContentType != nil { 414 attrs.ContentType = optional.ToString(uattrs.ContentType) 415 // For ContentType, sending the empty string is a no-op. 416 // Instead we send a null. 417 if attrs.ContentType == "" { 418 nullFields = append(nullFields, "ContentType") 419 } else { 420 forceSendFields = append(forceSendFields, "ContentType") 421 } 422 } 423 if uattrs.ContentLanguage != nil { 424 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) 425 // For ContentLanguage it's an error to send the empty string. 426 // Instead we send a null. 427 if attrs.ContentLanguage == "" { 428 nullFields = append(nullFields, "ContentLanguage") 429 } else { 430 forceSendFields = append(forceSendFields, "ContentLanguage") 431 } 432 } 433 if uattrs.ContentEncoding != nil { 434 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) 435 forceSendFields = append(forceSendFields, "ContentEncoding") 436 } 437 if uattrs.ContentDisposition != nil { 438 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) 439 forceSendFields = append(forceSendFields, "ContentDisposition") 440 } 441 if uattrs.CacheControl != nil { 442 attrs.CacheControl = optional.ToString(uattrs.CacheControl) 443 forceSendFields = append(forceSendFields, "CacheControl") 444 } 445 if uattrs.EventBasedHold != nil { 446 attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold) 447 forceSendFields = append(forceSendFields, "EventBasedHold") 448 } 449 if uattrs.TemporaryHold != nil { 450 attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) 451 forceSendFields = append(forceSendFields, "TemporaryHold") 452 } 453 if uattrs.Metadata != nil { 454 attrs.Metadata = uattrs.Metadata 455 if len(attrs.Metadata) == 0 { 456 // Sending the empty map is a no-op. We send null instead. 457 nullFields = append(nullFields, "Metadata") 458 } else { 459 forceSendFields = append(forceSendFields, "Metadata") 460 } 461 } 462 if uattrs.ACL != nil { 463 attrs.ACL = uattrs.ACL 464 // It's an error to attempt to delete the ACL, so 465 // we don't append to nullFields here. 466 forceSendFields = append(forceSendFields, "Acl") 467 } 468 rawObj := attrs.toRawObject(o.bucket) 469 rawObj.ForceSendFields = forceSendFields 470 rawObj.NullFields = nullFields 471 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) 472 if err := applyConds("Update", o.gen, o.conds, call); err != nil { 473 return nil, err 474 } 475 if o.userProject != "" { 476 call.UserProject(o.userProject) 477 } 478 if uattrs.PredefinedACL != "" { 479 call.PredefinedAcl(uattrs.PredefinedACL) 480 } 481 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 482 return nil, err 483 } 484 var obj *raw.Object 485 setClientHeader(call.Header()) 486 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 487 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 488 return nil, ErrObjectNotExist 489 } 490 if err != nil { 491 return nil, err 492 } 493 return newObject(obj), nil 494} 495 496// BucketName returns the name of the bucket. 497func (o *ObjectHandle) BucketName() string { 498 return o.bucket 499} 500 501// ObjectName returns the name of the object. 502func (o *ObjectHandle) ObjectName() string { 503 return o.object 504} 505 506// ObjectAttrsToUpdate is used to update the attributes of an object. 507// Only fields set to non-nil values will be updated. 508// Set a field to its zero value to delete it. 509// 510// For example, to change ContentType and delete ContentEncoding and 511// Metadata, use 512// ObjectAttrsToUpdate{ 513// ContentType: "text/html", 514// ContentEncoding: "", 515// Metadata: map[string]string{}, 516// } 517type ObjectAttrsToUpdate struct { 518 EventBasedHold optional.Bool 519 TemporaryHold optional.Bool 520 ContentType optional.String 521 ContentLanguage optional.String 522 ContentEncoding optional.String 523 ContentDisposition optional.String 524 CacheControl optional.String 525 Metadata map[string]string // set to map[string]string{} to delete 526 ACL []ACLRule 527 528 // If not empty, applies a predefined set of access controls. ACL must be nil. 529 // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch. 530 PredefinedACL string 531} 532 533// Delete deletes the single specified object. 534func (o *ObjectHandle) Delete(ctx context.Context) error { 535 if err := o.validate(); err != nil { 536 return err 537 } 538 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) 539 if err := applyConds("Delete", o.gen, o.conds, call); err != nil { 540 return err 541 } 542 if o.userProject != "" { 543 call.UserProject(o.userProject) 544 } 545 // Encryption doesn't apply to Delete. 546 setClientHeader(call.Header()) 547 err := runWithRetry(ctx, func() error { return call.Do() }) 548 switch e := err.(type) { 549 case nil: 550 return nil 551 case *googleapi.Error: 552 if e.Code == http.StatusNotFound { 553 return ErrObjectNotExist 554 } 555 } 556 return err 557} 558 559// ReadCompressed when true causes the read to happen without decompressing. 560func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { 561 o2 := *o 562 o2.readCompressed = compressed 563 return &o2 564} 565 566// NewWriter returns a storage Writer that writes to the GCS object 567// associated with this ObjectHandle. 568// 569// A new object will be created unless an object with this name already exists. 570// Otherwise any previous object with the same name will be replaced. 571// The object will not be available (and any previous object will remain) 572// until Close has been called. 573// 574// Attributes can be set on the object by modifying the returned Writer's 575// ObjectAttrs field before the first call to Write. If no ContentType 576// attribute is specified, the content type will be automatically sniffed 577// using net/http.DetectContentType. 578// 579// It is the caller's responsibility to call Close when writing is done. To 580// stop writing without saving the data, cancel the context. 581func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { 582 return &Writer{ 583 ctx: ctx, 584 o: o, 585 donec: make(chan struct{}), 586 ObjectAttrs: ObjectAttrs{Name: o.object}, 587 ChunkSize: googleapi.DefaultUploadChunkSize, 588 } 589} 590 591func (o *ObjectHandle) validate() error { 592 if o.bucket == "" { 593 return errors.New("storage: bucket name is empty") 594 } 595 if o.object == "" { 596 return errors.New("storage: object name is empty") 597 } 598 if !utf8.ValidString(o.object) { 599 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) 600 } 601 return nil 602} 603 604// parseKey converts the binary contents of a private key file to an 605// *rsa.PrivateKey. It detects whether the private key is in a PEM container or 606// not. If so, it extracts the private key from PEM container before 607// conversion. It only supports PEM containers with no passphrase. 608func parseKey(key []byte) (*rsa.PrivateKey, error) { 609 if block, _ := pem.Decode(key); block != nil { 610 key = block.Bytes 611 } 612 parsedKey, err := x509.ParsePKCS8PrivateKey(key) 613 if err != nil { 614 parsedKey, err = x509.ParsePKCS1PrivateKey(key) 615 if err != nil { 616 return nil, err 617 } 618 } 619 parsed, ok := parsedKey.(*rsa.PrivateKey) 620 if !ok { 621 return nil, errors.New("oauth2: private key is invalid") 622 } 623 return parsed, nil 624} 625 626// toRawObject copies the editable attributes from o to the raw library's Object type. 627func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { 628 var ret string 629 if !o.RetentionExpirationTime.IsZero() { 630 ret = o.RetentionExpirationTime.Format(time.RFC3339) 631 } 632 return &raw.Object{ 633 Bucket: bucket, 634 Name: o.Name, 635 EventBasedHold: o.EventBasedHold, 636 TemporaryHold: o.TemporaryHold, 637 RetentionExpirationTime: ret, 638 ContentType: o.ContentType, 639 ContentEncoding: o.ContentEncoding, 640 ContentLanguage: o.ContentLanguage, 641 CacheControl: o.CacheControl, 642 ContentDisposition: o.ContentDisposition, 643 StorageClass: o.StorageClass, 644 Acl: toRawObjectACL(o.ACL), 645 Metadata: o.Metadata, 646 } 647} 648 649// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. 650type ObjectAttrs struct { 651 // Bucket is the name of the bucket containing this GCS object. 652 // This field is read-only. 653 Bucket string 654 655 // Name is the name of the object within the bucket. 656 // This field is read-only. 657 Name string 658 659 // ContentType is the MIME type of the object's content. 660 ContentType string 661 662 // ContentLanguage is the content language of the object's content. 663 ContentLanguage string 664 665 // CacheControl is the Cache-Control header to be sent in the response 666 // headers when serving the object data. 667 CacheControl string 668 669 // EventBasedHold specifies whether an object is under event-based hold. New 670 // objects created in a bucket whose DefaultEventBasedHold is set will 671 // default to that value. 672 EventBasedHold bool 673 674 // TemporaryHold specifies whether an object is under temporary hold. While 675 // this flag is set to true, the object is protected against deletion and 676 // overwrites. 677 TemporaryHold bool 678 679 // RetentionExpirationTime is a server-determined value that specifies the 680 // earliest time that the object's retention period expires. 681 // This is a read-only field. 682 RetentionExpirationTime time.Time 683 684 // ACL is the list of access control rules for the object. 685 ACL []ACLRule 686 687 // If not empty, applies a predefined set of access controls. It should be set 688 // only when writing, copying or composing an object. When copying or composing, 689 // it acts as the destinationPredefinedAcl parameter. 690 // PredefinedACL is always empty for ObjectAttrs returned from the service. 691 // See https://cloud.google.com/storage/docs/json_api/v1/objects/insert 692 // for valid values. 693 PredefinedACL string 694 695 // Owner is the owner of the object. This field is read-only. 696 // 697 // If non-zero, it is in the form of "user-<userId>". 698 Owner string 699 700 // Size is the length of the object's content. This field is read-only. 701 Size int64 702 703 // ContentEncoding is the encoding of the object's content. 704 ContentEncoding string 705 706 // ContentDisposition is the optional Content-Disposition header of the object 707 // sent in the response headers. 708 ContentDisposition string 709 710 // MD5 is the MD5 hash of the object's content. This field is read-only, 711 // except when used from a Writer. If set on a Writer, the uploaded 712 // data is rejected if its MD5 hash does not match this field. 713 MD5 []byte 714 715 // CRC32C is the CRC32 checksum of the object's content using 716 // the Castagnoli93 polynomial. This field is read-only, except when 717 // used from a Writer. If set on a Writer and Writer.SendCRC32C 718 // is true, the uploaded data is rejected if its CRC32c hash does not 719 // match this field. 720 CRC32C uint32 721 722 // MediaLink is an URL to the object's content. This field is read-only. 723 MediaLink string 724 725 // Metadata represents user-provided metadata, in key/value pairs. 726 // It can be nil if no metadata is provided. 727 Metadata map[string]string 728 729 // Generation is the generation number of the object's content. 730 // This field is read-only. 731 Generation int64 732 733 // Metageneration is the version of the metadata for this 734 // object at this generation. This field is used for preconditions 735 // and for detecting changes in metadata. A metageneration number 736 // is only meaningful in the context of a particular generation 737 // of a particular object. This field is read-only. 738 Metageneration int64 739 740 // StorageClass is the storage class of the object. 741 // This value defines how objects in the bucket are stored and 742 // determines the SLA and the cost of storage. Typical values are 743 // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" 744 // and "DURABLE_REDUCED_AVAILABILITY". 745 // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL" 746 // or "REGIONAL" depending on the bucket's location settings. 747 StorageClass string 748 749 // Created is the time the object was created. This field is read-only. 750 Created time.Time 751 752 // Deleted is the time the object was deleted. 753 // If not deleted, it is the zero value. This field is read-only. 754 Deleted time.Time 755 756 // Updated is the creation or modification time of the object. 757 // For buckets with versioning enabled, changing an object's 758 // metadata does not change this property. This field is read-only. 759 Updated time.Time 760 761 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the 762 // customer-supplied encryption key for the object. It is empty if there is 763 // no customer-supplied encryption key. 764 // See // https://cloud.google.com/storage/docs/encryption for more about 765 // encryption in Google Cloud Storage. 766 CustomerKeySHA256 string 767 768 // Cloud KMS key name, in the form 769 // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, 770 // if the object is encrypted by such a key. 771 // 772 // Providing both a KMSKeyName and a customer-supplied encryption key (via 773 // ObjectHandle.Key) will result in an error when writing an object. 774 KMSKeyName string 775 776 // Prefix is set only for ObjectAttrs which represent synthetic "directory 777 // entries" when iterating over buckets using Query.Delimiter. See 778 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be 779 // populated. 780 Prefix string 781} 782 783// convertTime converts a time in RFC3339 format to time.Time. 784// If any error occurs in parsing, the zero-value time.Time is silently returned. 785func convertTime(t string) time.Time { 786 var r time.Time 787 if t != "" { 788 r, _ = time.Parse(time.RFC3339, t) 789 } 790 return r 791} 792 793func newObject(o *raw.Object) *ObjectAttrs { 794 if o == nil { 795 return nil 796 } 797 owner := "" 798 if o.Owner != nil { 799 owner = o.Owner.Entity 800 } 801 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) 802 crc32c, _ := decodeUint32(o.Crc32c) 803 var sha256 string 804 if o.CustomerEncryption != nil { 805 sha256 = o.CustomerEncryption.KeySha256 806 } 807 return &ObjectAttrs{ 808 Bucket: o.Bucket, 809 Name: o.Name, 810 ContentType: o.ContentType, 811 ContentLanguage: o.ContentLanguage, 812 CacheControl: o.CacheControl, 813 EventBasedHold: o.EventBasedHold, 814 TemporaryHold: o.TemporaryHold, 815 RetentionExpirationTime: convertTime(o.RetentionExpirationTime), 816 ACL: toObjectACLRules(o.Acl), 817 Owner: owner, 818 ContentEncoding: o.ContentEncoding, 819 ContentDisposition: o.ContentDisposition, 820 Size: int64(o.Size), 821 MD5: md5, 822 CRC32C: crc32c, 823 MediaLink: o.MediaLink, 824 Metadata: o.Metadata, 825 Generation: o.Generation, 826 Metageneration: o.Metageneration, 827 StorageClass: o.StorageClass, 828 CustomerKeySHA256: sha256, 829 KMSKeyName: o.KmsKeyName, 830 Created: convertTime(o.TimeCreated), 831 Deleted: convertTime(o.TimeDeleted), 832 Updated: convertTime(o.Updated), 833 } 834} 835 836// Decode a uint32 encoded in Base64 in big-endian byte order. 837func decodeUint32(b64 string) (uint32, error) { 838 d, err := base64.StdEncoding.DecodeString(b64) 839 if err != nil { 840 return 0, err 841 } 842 if len(d) != 4 { 843 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) 844 } 845 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil 846} 847 848// Encode a uint32 as Base64 in big-endian byte order. 849func encodeUint32(u uint32) string { 850 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} 851 return base64.StdEncoding.EncodeToString(b) 852} 853 854// Query represents a query to filter objects from a bucket. 855type Query struct { 856 // Delimiter returns results in a directory-like fashion. 857 // Results will contain only objects whose names, aside from the 858 // prefix, do not contain delimiter. Objects whose names, 859 // aside from the prefix, contain delimiter will have their name, 860 // truncated after the delimiter, returned in prefixes. 861 // Duplicate prefixes are omitted. 862 // Optional. 863 Delimiter string 864 865 // Prefix is the prefix filter to query objects 866 // whose names begin with this prefix. 867 // Optional. 868 Prefix string 869 870 // Versions indicates whether multiple versions of the same 871 // object will be included in the results. 872 Versions bool 873} 874 875// contentTyper implements ContentTyper to enable an 876// io.ReadCloser to specify its MIME type. 877type contentTyper struct { 878 io.Reader 879 t string 880} 881 882func (c *contentTyper) ContentType() string { 883 return c.t 884} 885 886// Conditions constrain methods to act on specific generations of 887// objects. 888// 889// The zero value is an empty set of constraints. Not all conditions or 890// combinations of conditions are applicable to all methods. 891// See https://cloud.google.com/storage/docs/generations-preconditions 892// for details on how these operate. 893type Conditions struct { 894 // Generation constraints. 895 // At most one of the following can be set to a non-zero value. 896 897 // GenerationMatch specifies that the object must have the given generation 898 // for the operation to occur. 899 // If GenerationMatch is zero, it has no effect. 900 // Use DoesNotExist to specify that the object does not exist in the bucket. 901 GenerationMatch int64 902 903 // GenerationNotMatch specifies that the object must not have the given 904 // generation for the operation to occur. 905 // If GenerationNotMatch is zero, it has no effect. 906 GenerationNotMatch int64 907 908 // DoesNotExist specifies that the object must not exist in the bucket for 909 // the operation to occur. 910 // If DoesNotExist is false, it has no effect. 911 DoesNotExist bool 912 913 // Metadata generation constraints. 914 // At most one of the following can be set to a non-zero value. 915 916 // MetagenerationMatch specifies that the object must have the given 917 // metageneration for the operation to occur. 918 // If MetagenerationMatch is zero, it has no effect. 919 MetagenerationMatch int64 920 921 // MetagenerationNotMatch specifies that the object must not have the given 922 // metageneration for the operation to occur. 923 // If MetagenerationNotMatch is zero, it has no effect. 924 MetagenerationNotMatch int64 925} 926 927func (c *Conditions) validate(method string) error { 928 if *c == (Conditions{}) { 929 return fmt.Errorf("storage: %s: empty conditions", method) 930 } 931 if !c.isGenerationValid() { 932 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) 933 } 934 if !c.isMetagenerationValid() { 935 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 936 } 937 return nil 938} 939 940func (c *Conditions) isGenerationValid() bool { 941 n := 0 942 if c.GenerationMatch != 0 { 943 n++ 944 } 945 if c.GenerationNotMatch != 0 { 946 n++ 947 } 948 if c.DoesNotExist { 949 n++ 950 } 951 return n <= 1 952} 953 954func (c *Conditions) isMetagenerationValid() bool { 955 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 956} 957 958// applyConds modifies the provided call using the conditions in conds. 959// call is something that quacks like a *raw.WhateverCall. 960func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { 961 cval := reflect.ValueOf(call) 962 if gen >= 0 { 963 if !setConditionField(cval, "Generation", gen) { 964 return fmt.Errorf("storage: %s: generation not supported", method) 965 } 966 } 967 if conds == nil { 968 return nil 969 } 970 if err := conds.validate(method); err != nil { 971 return err 972 } 973 switch { 974 case conds.GenerationMatch != 0: 975 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { 976 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) 977 } 978 case conds.GenerationNotMatch != 0: 979 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { 980 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) 981 } 982 case conds.DoesNotExist: 983 if !setConditionField(cval, "IfGenerationMatch", int64(0)) { 984 return fmt.Errorf("storage: %s: DoesNotExist not supported", method) 985 } 986 } 987 switch { 988 case conds.MetagenerationMatch != 0: 989 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 990 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 991 } 992 case conds.MetagenerationNotMatch != 0: 993 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 994 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 995 } 996 } 997 return nil 998} 999 1000func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { 1001 if gen >= 0 { 1002 call.SourceGeneration(gen) 1003 } 1004 if conds == nil { 1005 return nil 1006 } 1007 if err := conds.validate("CopyTo source"); err != nil { 1008 return err 1009 } 1010 switch { 1011 case conds.GenerationMatch != 0: 1012 call.IfSourceGenerationMatch(conds.GenerationMatch) 1013 case conds.GenerationNotMatch != 0: 1014 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) 1015 case conds.DoesNotExist: 1016 call.IfSourceGenerationMatch(0) 1017 } 1018 switch { 1019 case conds.MetagenerationMatch != 0: 1020 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) 1021 case conds.MetagenerationNotMatch != 0: 1022 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) 1023 } 1024 return nil 1025} 1026 1027// setConditionField sets a field on a *raw.WhateverCall. 1028// We can't use anonymous interfaces because the return type is 1029// different, since the field setters are builders. 1030func setConditionField(call reflect.Value, name string, value interface{}) bool { 1031 m := call.MethodByName(name) 1032 if !m.IsValid() { 1033 return false 1034 } 1035 m.Call([]reflect.Value{reflect.ValueOf(value)}) 1036 return true 1037} 1038 1039// conditionsQuery returns the generation and conditions as a URL query 1040// string suitable for URL.RawQuery. It assumes that the conditions 1041// have been validated. 1042func conditionsQuery(gen int64, conds *Conditions) string { 1043 // URL escapes are elided because integer strings are URL-safe. 1044 var buf []byte 1045 1046 appendParam := func(s string, n int64) { 1047 if len(buf) > 0 { 1048 buf = append(buf, '&') 1049 } 1050 buf = append(buf, s...) 1051 buf = strconv.AppendInt(buf, n, 10) 1052 } 1053 1054 if gen >= 0 { 1055 appendParam("generation=", gen) 1056 } 1057 if conds == nil { 1058 return string(buf) 1059 } 1060 switch { 1061 case conds.GenerationMatch != 0: 1062 appendParam("ifGenerationMatch=", conds.GenerationMatch) 1063 case conds.GenerationNotMatch != 0: 1064 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) 1065 case conds.DoesNotExist: 1066 appendParam("ifGenerationMatch=", 0) 1067 } 1068 switch { 1069 case conds.MetagenerationMatch != 0: 1070 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) 1071 case conds.MetagenerationNotMatch != 0: 1072 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) 1073 } 1074 return string(buf) 1075} 1076 1077// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods 1078// that modifyCall searches for by name. 1079type composeSourceObj struct { 1080 src *raw.ComposeRequestSourceObjects 1081} 1082 1083func (c composeSourceObj) Generation(gen int64) { 1084 c.src.Generation = gen 1085} 1086 1087func (c composeSourceObj) IfGenerationMatch(gen int64) { 1088 // It's safe to overwrite ObjectPreconditions, since its only field is 1089 // IfGenerationMatch. 1090 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{ 1091 IfGenerationMatch: gen, 1092 } 1093} 1094 1095func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error { 1096 if key == nil { 1097 return nil 1098 } 1099 // TODO(jbd): Ask the API team to return a more user-friendly error 1100 // and avoid doing this check at the client level. 1101 if len(key) != 32 { 1102 return errors.New("storage: not a 32-byte AES-256 key") 1103 } 1104 var cs string 1105 if copySource { 1106 cs = "copy-source-" 1107 } 1108 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") 1109 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) 1110 keyHash := sha256.Sum256(key) 1111 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) 1112 return nil 1113} 1114 1115// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. 1116func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { 1117 r := c.raw.Projects.ServiceAccount.Get(projectID) 1118 res, err := r.Context(ctx).Do() 1119 if err != nil { 1120 return "", err 1121 } 1122 return res.EmailAddress, nil 1123} 1124