1// Copyright 2014 Google Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "crypto" 20 "crypto/rand" 21 "crypto/rsa" 22 "crypto/sha256" 23 "crypto/x509" 24 "encoding/base64" 25 "encoding/pem" 26 "errors" 27 "fmt" 28 "io" 29 "io/ioutil" 30 "net/http" 31 "net/url" 32 "reflect" 33 "strconv" 34 "strings" 35 "time" 36 "unicode/utf8" 37 38 "google.golang.org/api/option" 39 htransport "google.golang.org/api/transport/http" 40 41 "cloud.google.com/go/internal/optional" 42 "cloud.google.com/go/internal/version" 43 "golang.org/x/net/context" 44 "google.golang.org/api/googleapi" 45 raw "google.golang.org/api/storage/v1" 46) 47 48var ( 49 ErrBucketNotExist = errors.New("storage: bucket doesn't exist") 50 ErrObjectNotExist = errors.New("storage: object doesn't exist") 51) 52 53const userAgent = "gcloud-golang-storage/20151204" 54 55const ( 56 // ScopeFullControl grants permissions to manage your 57 // data and permissions in Google Cloud Storage. 58 ScopeFullControl = raw.DevstorageFullControlScope 59 60 // ScopeReadOnly grants permissions to 61 // view your data in Google Cloud Storage. 62 ScopeReadOnly = raw.DevstorageReadOnlyScope 63 64 // ScopeReadWrite grants permissions to manage your 65 // data in Google Cloud Storage. 66 ScopeReadWrite = raw.DevstorageReadWriteScope 67) 68 69var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) 70 71func setClientHeader(headers http.Header) { 72 headers.Set("x-goog-api-client", xGoogHeader) 73} 74 75// Client is a client for interacting with Google Cloud Storage. 76// 77// Clients should be reused instead of created as needed. 78// The methods of Client are safe for concurrent use by multiple goroutines. 79type Client struct { 80 hc *http.Client 81 raw *raw.Service 82} 83 84// NewClient creates a new Google Cloud Storage client. 85// The default scope is ScopeFullControl. To use a different scope, like ScopeReadOnly, use option.WithScopes. 86func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 87 o := []option.ClientOption{ 88 option.WithScopes(ScopeFullControl), 89 option.WithUserAgent(userAgent), 90 } 91 opts = append(o, opts...) 92 hc, ep, err := htransport.NewClient(ctx, opts...) 93 if err != nil { 94 return nil, fmt.Errorf("dialing: %v", err) 95 } 96 rawService, err := raw.New(hc) 97 if err != nil { 98 return nil, fmt.Errorf("storage client: %v", err) 99 } 100 if ep != "" { 101 rawService.BasePath = ep 102 } 103 return &Client{ 104 hc: hc, 105 raw: rawService, 106 }, nil 107} 108 109// Close closes the Client. 110// 111// Close need not be called at program exit. 112func (c *Client) Close() error { 113 // Set fields to nil so that subsequent uses 114 // will panic. 115 c.hc = nil 116 c.raw = nil 117 return nil 118} 119 120// SignedURLOptions allows you to restrict the access to the signed URL. 121type SignedURLOptions struct { 122 // GoogleAccessID represents the authorizer of the signed URL generation. 123 // It is typically the Google service account client email address from 124 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 125 // Required. 126 GoogleAccessID string 127 128 // PrivateKey is the Google service account private key. It is obtainable 129 // from the Google Developers Console. 130 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 131 // create a service account client ID or reuse one of your existing service account 132 // credentials. Click on the "Generate new P12 key" to generate and download 133 // a new private key. Once you download the P12 file, use the following command 134 // to convert it into a PEM file. 135 // 136 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 137 // 138 // Provide the contents of the PEM file as a byte slice. 139 // Exactly one of PrivateKey or SignBytes must be non-nil. 140 PrivateKey []byte 141 142 // SignBytes is a function for implementing custom signing. 143 // If your application is running on Google App Engine, you can use appengine's internal signing function: 144 // ctx := appengine.NewContext(request) 145 // acc, _ := appengine.ServiceAccount(ctx) 146 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 147 // GoogleAccessID: acc, 148 // SignBytes: func(b []byte) ([]byte, error) { 149 // _, signedBytes, err := appengine.SignBytes(ctx, b) 150 // return signedBytes, err 151 // }, 152 // // etc. 153 // }) 154 // 155 // Exactly one of PrivateKey or SignBytes must be non-nil. 156 SignBytes func([]byte) ([]byte, error) 157 158 // Method is the HTTP method to be used with the signed URL. 159 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. 160 // Required. 161 Method string 162 163 // Expires is the expiration time on the signed URL. It must be 164 // a datetime in the future. 165 // Required. 166 Expires time.Time 167 168 // ContentType is the content type header the client must provide 169 // to use the generated signed URL. 170 // Optional. 171 ContentType string 172 173 // Headers is a list of extension headers the client must provide 174 // in order to use the generated signed URL. 175 // Optional. 176 Headers []string 177 178 // MD5 is the base64 encoded MD5 checksum of the file. 179 // If provided, the client should provide the exact value on the request 180 // header in order to use the signed URL. 181 // Optional. 182 MD5 string 183} 184 185// SignedURL returns a URL for the specified object. Signed URLs allow 186// the users access to a restricted resource for a limited time without having a 187// Google account or signing in. For more information about the signed 188// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. 189func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { 190 if opts == nil { 191 return "", errors.New("storage: missing required SignedURLOptions") 192 } 193 if opts.GoogleAccessID == "" { 194 return "", errors.New("storage: missing required GoogleAccessID") 195 } 196 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) { 197 return "", errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 198 } 199 if opts.Method == "" { 200 return "", errors.New("storage: missing required method option") 201 } 202 if opts.Expires.IsZero() { 203 return "", errors.New("storage: missing required expires option") 204 } 205 if opts.MD5 != "" { 206 md5, err := base64.StdEncoding.DecodeString(opts.MD5) 207 if err != nil || len(md5) != 16 { 208 return "", errors.New("storage: invalid MD5 checksum") 209 } 210 } 211 212 signBytes := opts.SignBytes 213 if opts.PrivateKey != nil { 214 key, err := parseKey(opts.PrivateKey) 215 if err != nil { 216 return "", err 217 } 218 signBytes = func(b []byte) ([]byte, error) { 219 sum := sha256.Sum256(b) 220 return rsa.SignPKCS1v15( 221 rand.Reader, 222 key, 223 crypto.SHA256, 224 sum[:], 225 ) 226 } 227 } 228 229 u := &url.URL{ 230 Path: fmt.Sprintf("/%s/%s", bucket, name), 231 } 232 233 buf := &bytes.Buffer{} 234 fmt.Fprintf(buf, "%s\n", opts.Method) 235 fmt.Fprintf(buf, "%s\n", opts.MD5) 236 fmt.Fprintf(buf, "%s\n", opts.ContentType) 237 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) 238 if len(opts.Headers) > 0 { 239 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) 240 } 241 fmt.Fprintf(buf, "%s", u.String()) 242 243 b, err := signBytes(buf.Bytes()) 244 if err != nil { 245 return "", err 246 } 247 encoded := base64.StdEncoding.EncodeToString(b) 248 u.Scheme = "https" 249 u.Host = "storage.googleapis.com" 250 q := u.Query() 251 q.Set("GoogleAccessId", opts.GoogleAccessID) 252 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) 253 q.Set("Signature", string(encoded)) 254 u.RawQuery = q.Encode() 255 return u.String(), nil 256} 257 258// ObjectHandle provides operations on an object in a Google Cloud Storage bucket. 259// Use BucketHandle.Object to get a handle. 260type ObjectHandle struct { 261 c *Client 262 bucket string 263 object string 264 acl ACLHandle 265 gen int64 // a negative value indicates latest 266 conds *Conditions 267 encryptionKey []byte // AES-256 key 268 userProject string // for requester-pays buckets 269 readCompressed bool // Accept-Encoding: gzip 270} 271 272// ACL provides access to the object's access control list. 273// This controls who can read and write this object. 274// This call does not perform any network operations. 275func (o *ObjectHandle) ACL() *ACLHandle { 276 return &o.acl 277} 278 279// Generation returns a new ObjectHandle that operates on a specific generation 280// of the object. 281// By default, the handle operates on the latest generation. Not 282// all operations work when given a specific generation; check the API 283// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. 284func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { 285 o2 := *o 286 o2.gen = gen 287 return &o2 288} 289 290// If returns a new ObjectHandle that applies a set of preconditions. 291// Preconditions already set on the ObjectHandle are ignored. 292// Operations on the new handle will only occur if the preconditions are 293// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions 294// for more details. 295func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { 296 o2 := *o 297 o2.conds = &conds 298 return &o2 299} 300 301// Key returns a new ObjectHandle that uses the supplied encryption 302// key to encrypt and decrypt the object's contents. 303// 304// Encryption key must be a 32-byte AES-256 key. 305// See https://cloud.google.com/storage/docs/encryption for details. 306func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { 307 o2 := *o 308 o2.encryptionKey = encryptionKey 309 return &o2 310} 311 312// Attrs returns meta information about the object. 313// ErrObjectNotExist will be returned if the object is not found. 314func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { 315 if err := o.validate(); err != nil { 316 return nil, err 317 } 318 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) 319 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { 320 return nil, err 321 } 322 if o.userProject != "" { 323 call.UserProject(o.userProject) 324 } 325 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 326 return nil, err 327 } 328 var obj *raw.Object 329 var err error 330 setClientHeader(call.Header()) 331 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 332 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 333 return nil, ErrObjectNotExist 334 } 335 if err != nil { 336 return nil, err 337 } 338 return newObject(obj), nil 339} 340 341// Update updates an object with the provided attributes. 342// All zero-value attributes are ignored. 343// ErrObjectNotExist will be returned if the object is not found. 344func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { 345 if err := o.validate(); err != nil { 346 return nil, err 347 } 348 var attrs ObjectAttrs 349 // Lists of fields to send, and set to null, in the JSON. 350 var forceSendFields, nullFields []string 351 if uattrs.ContentType != nil { 352 attrs.ContentType = optional.ToString(uattrs.ContentType) 353 // For ContentType, sending the empty string is a no-op. 354 // Instead we send a null. 355 if attrs.ContentType == "" { 356 nullFields = append(nullFields, "ContentType") 357 } else { 358 forceSendFields = append(forceSendFields, "ContentType") 359 } 360 } 361 if uattrs.ContentLanguage != nil { 362 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) 363 // For ContentLanguage it's an error to send the empty string. 364 // Instead we send a null. 365 if attrs.ContentLanguage == "" { 366 nullFields = append(nullFields, "ContentLanguage") 367 } else { 368 forceSendFields = append(forceSendFields, "ContentLanguage") 369 } 370 } 371 if uattrs.ContentEncoding != nil { 372 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) 373 forceSendFields = append(forceSendFields, "ContentEncoding") 374 } 375 if uattrs.ContentDisposition != nil { 376 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) 377 forceSendFields = append(forceSendFields, "ContentDisposition") 378 } 379 if uattrs.CacheControl != nil { 380 attrs.CacheControl = optional.ToString(uattrs.CacheControl) 381 forceSendFields = append(forceSendFields, "CacheControl") 382 } 383 if uattrs.Metadata != nil { 384 attrs.Metadata = uattrs.Metadata 385 if len(attrs.Metadata) == 0 { 386 // Sending the empty map is a no-op. We send null instead. 387 nullFields = append(nullFields, "Metadata") 388 } else { 389 forceSendFields = append(forceSendFields, "Metadata") 390 } 391 } 392 if uattrs.ACL != nil { 393 attrs.ACL = uattrs.ACL 394 // It's an error to attempt to delete the ACL, so 395 // we don't append to nullFields here. 396 forceSendFields = append(forceSendFields, "Acl") 397 } 398 rawObj := attrs.toRawObject(o.bucket) 399 rawObj.ForceSendFields = forceSendFields 400 rawObj.NullFields = nullFields 401 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) 402 if err := applyConds("Update", o.gen, o.conds, call); err != nil { 403 return nil, err 404 } 405 if o.userProject != "" { 406 call.UserProject(o.userProject) 407 } 408 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 409 return nil, err 410 } 411 var obj *raw.Object 412 var err error 413 setClientHeader(call.Header()) 414 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 415 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 416 return nil, ErrObjectNotExist 417 } 418 if err != nil { 419 return nil, err 420 } 421 return newObject(obj), nil 422} 423 424// ObjectAttrsToUpdate is used to update the attributes of an object. 425// Only fields set to non-nil values will be updated. 426// Set a field to its zero value to delete it. 427// 428// For example, to change ContentType and delete ContentEncoding and 429// Metadata, use 430// ObjectAttrsToUpdate{ 431// ContentType: "text/html", 432// ContentEncoding: "", 433// Metadata: map[string]string{}, 434// } 435type ObjectAttrsToUpdate struct { 436 ContentType optional.String 437 ContentLanguage optional.String 438 ContentEncoding optional.String 439 ContentDisposition optional.String 440 CacheControl optional.String 441 Metadata map[string]string // set to map[string]string{} to delete 442 ACL []ACLRule 443} 444 445// Delete deletes the single specified object. 446func (o *ObjectHandle) Delete(ctx context.Context) error { 447 if err := o.validate(); err != nil { 448 return err 449 } 450 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) 451 if err := applyConds("Delete", o.gen, o.conds, call); err != nil { 452 return err 453 } 454 if o.userProject != "" { 455 call.UserProject(o.userProject) 456 } 457 // Encryption doesn't apply to Delete. 458 setClientHeader(call.Header()) 459 err := runWithRetry(ctx, func() error { return call.Do() }) 460 switch e := err.(type) { 461 case nil: 462 return nil 463 case *googleapi.Error: 464 if e.Code == http.StatusNotFound { 465 return ErrObjectNotExist 466 } 467 } 468 return err 469} 470 471// ReadCompressed when true causes the read to happen without decompressing. 472func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { 473 o2 := *o 474 o2.readCompressed = compressed 475 return &o2 476} 477 478// NewReader creates a new Reader to read the contents of the 479// object. 480// ErrObjectNotExist will be returned if the object is not found. 481// 482// The caller must call Close on the returned Reader when done reading. 483func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { 484 return o.NewRangeReader(ctx, 0, -1) 485} 486 487// NewRangeReader reads part of an object, reading at most length bytes 488// starting at the given offset. If length is negative, the object is read 489// until the end. 490func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { 491 if err := o.validate(); err != nil { 492 return nil, err 493 } 494 if offset < 0 { 495 return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) 496 } 497 if o.conds != nil { 498 if err := o.conds.validate("NewRangeReader"); err != nil { 499 return nil, err 500 } 501 } 502 u := &url.URL{ 503 Scheme: "https", 504 Host: "storage.googleapis.com", 505 Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), 506 RawQuery: conditionsQuery(o.gen, o.conds), 507 } 508 verb := "GET" 509 if length == 0 { 510 verb = "HEAD" 511 } 512 req, err := http.NewRequest(verb, u.String(), nil) 513 if err != nil { 514 return nil, err 515 } 516 req = withContext(req, ctx) 517 if length < 0 && offset > 0 { 518 req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) 519 } else if length > 0 { 520 req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) 521 } 522 if o.userProject != "" { 523 req.Header.Set("X-Goog-User-Project", o.userProject) 524 } 525 if o.readCompressed { 526 req.Header.Set("Accept-Encoding", "gzip") 527 } 528 if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { 529 return nil, err 530 } 531 var res *http.Response 532 err = runWithRetry(ctx, func() error { 533 res, err = o.c.hc.Do(req) 534 if err != nil { 535 return err 536 } 537 if res.StatusCode == http.StatusNotFound { 538 res.Body.Close() 539 return ErrObjectNotExist 540 } 541 if res.StatusCode < 200 || res.StatusCode > 299 { 542 body, _ := ioutil.ReadAll(res.Body) 543 res.Body.Close() 544 return &googleapi.Error{ 545 Code: res.StatusCode, 546 Header: res.Header, 547 Body: string(body), 548 } 549 } 550 if offset > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { 551 res.Body.Close() 552 return errors.New("storage: partial request not satisfied") 553 } 554 return nil 555 }) 556 if err != nil { 557 return nil, err 558 } 559 560 var size int64 // total size of object, even if a range was requested. 561 if res.StatusCode == http.StatusPartialContent { 562 cr := strings.TrimSpace(res.Header.Get("Content-Range")) 563 if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { 564 return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) 565 } 566 size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) 567 if err != nil { 568 return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) 569 } 570 } else { 571 size = res.ContentLength 572 } 573 574 remain := res.ContentLength 575 body := res.Body 576 if length == 0 { 577 remain = 0 578 body.Close() 579 body = emptyBody 580 } 581 var ( 582 checkCRC bool 583 crc uint32 584 ) 585 // Even if there is a CRC header, we can't compute the hash on partial data. 586 if remain == size { 587 crc, checkCRC = parseCRC32c(res) 588 } 589 return &Reader{ 590 body: body, 591 size: size, 592 remain: remain, 593 contentType: res.Header.Get("Content-Type"), 594 contentEncoding: res.Header.Get("Content-Encoding"), 595 cacheControl: res.Header.Get("Cache-Control"), 596 wantCRC: crc, 597 checkCRC: checkCRC, 598 }, nil 599} 600 601func parseCRC32c(res *http.Response) (uint32, bool) { 602 const prefix = "crc32c=" 603 for _, spec := range res.Header["X-Goog-Hash"] { 604 if strings.HasPrefix(spec, prefix) { 605 c, err := decodeUint32(spec[len(prefix):]) 606 if err == nil { 607 return c, true 608 } 609 } 610 } 611 return 0, false 612} 613 614var emptyBody = ioutil.NopCloser(strings.NewReader("")) 615 616// NewWriter returns a storage Writer that writes to the GCS object 617// associated with this ObjectHandle. 618// 619// A new object will be created unless an object with this name already exists. 620// Otherwise any previous object with the same name will be replaced. 621// The object will not be available (and any previous object will remain) 622// until Close has been called. 623// 624// Attributes can be set on the object by modifying the returned Writer's 625// ObjectAttrs field before the first call to Write. If no ContentType 626// attribute is specified, the content type will be automatically sniffed 627// using net/http.DetectContentType. 628// 629// It is the caller's responsibility to call Close when writing is done. 630func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { 631 return &Writer{ 632 ctx: ctx, 633 o: o, 634 donec: make(chan struct{}), 635 ObjectAttrs: ObjectAttrs{Name: o.object}, 636 ChunkSize: googleapi.DefaultUploadChunkSize, 637 } 638} 639 640func (o *ObjectHandle) validate() error { 641 if o.bucket == "" { 642 return errors.New("storage: bucket name is empty") 643 } 644 if o.object == "" { 645 return errors.New("storage: object name is empty") 646 } 647 if !utf8.ValidString(o.object) { 648 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) 649 } 650 return nil 651} 652 653// parseKey converts the binary contents of a private key file to an 654// *rsa.PrivateKey. It detects whether the private key is in a PEM container or 655// not. If so, it extracts the private key from PEM container before 656// conversion. It only supports PEM containers with no passphrase. 657func parseKey(key []byte) (*rsa.PrivateKey, error) { 658 if block, _ := pem.Decode(key); block != nil { 659 key = block.Bytes 660 } 661 parsedKey, err := x509.ParsePKCS8PrivateKey(key) 662 if err != nil { 663 parsedKey, err = x509.ParsePKCS1PrivateKey(key) 664 if err != nil { 665 return nil, err 666 } 667 } 668 parsed, ok := parsedKey.(*rsa.PrivateKey) 669 if !ok { 670 return nil, errors.New("oauth2: private key is invalid") 671 } 672 return parsed, nil 673} 674 675func toRawObjectACL(oldACL []ACLRule) []*raw.ObjectAccessControl { 676 var acl []*raw.ObjectAccessControl 677 if len(oldACL) > 0 { 678 acl = make([]*raw.ObjectAccessControl, len(oldACL)) 679 for i, rule := range oldACL { 680 acl[i] = &raw.ObjectAccessControl{ 681 Entity: string(rule.Entity), 682 Role: string(rule.Role), 683 } 684 } 685 } 686 return acl 687} 688 689// toRawObject copies the editable attributes from o to the raw library's Object type. 690func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { 691 acl := toRawObjectACL(o.ACL) 692 return &raw.Object{ 693 Bucket: bucket, 694 Name: o.Name, 695 ContentType: o.ContentType, 696 ContentEncoding: o.ContentEncoding, 697 ContentLanguage: o.ContentLanguage, 698 CacheControl: o.CacheControl, 699 ContentDisposition: o.ContentDisposition, 700 StorageClass: o.StorageClass, 701 Acl: acl, 702 Metadata: o.Metadata, 703 } 704} 705 706// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. 707type ObjectAttrs struct { 708 // Bucket is the name of the bucket containing this GCS object. 709 // This field is read-only. 710 Bucket string 711 712 // Name is the name of the object within the bucket. 713 // This field is read-only. 714 Name string 715 716 // ContentType is the MIME type of the object's content. 717 ContentType string 718 719 // ContentLanguage is the content language of the object's content. 720 ContentLanguage string 721 722 // CacheControl is the Cache-Control header to be sent in the response 723 // headers when serving the object data. 724 CacheControl string 725 726 // ACL is the list of access control rules for the object. 727 ACL []ACLRule 728 729 // Owner is the owner of the object. This field is read-only. 730 // 731 // If non-zero, it is in the form of "user-<userId>". 732 Owner string 733 734 // Size is the length of the object's content. This field is read-only. 735 Size int64 736 737 // ContentEncoding is the encoding of the object's content. 738 ContentEncoding string 739 740 // ContentDisposition is the optional Content-Disposition header of the object 741 // sent in the response headers. 742 ContentDisposition string 743 744 // MD5 is the MD5 hash of the object's content. This field is read-only, 745 // except when used from a Writer. If set on a Writer, the uploaded 746 // data is rejected if its MD5 hash does not match this field. 747 MD5 []byte 748 749 // CRC32C is the CRC32 checksum of the object's content using 750 // the Castagnoli93 polynomial. This field is read-only, except when 751 // used from a Writer. If set on a Writer and Writer.SendCRC32C 752 // is true, the uploaded data is rejected if its CRC32c hash does not 753 // match this field. 754 CRC32C uint32 755 756 // MediaLink is an URL to the object's content. This field is read-only. 757 MediaLink string 758 759 // Metadata represents user-provided metadata, in key/value pairs. 760 // It can be nil if no metadata is provided. 761 Metadata map[string]string 762 763 // Generation is the generation number of the object's content. 764 // This field is read-only. 765 Generation int64 766 767 // Metageneration is the version of the metadata for this 768 // object at this generation. This field is used for preconditions 769 // and for detecting changes in metadata. A metageneration number 770 // is only meaningful in the context of a particular generation 771 // of a particular object. This field is read-only. 772 Metageneration int64 773 774 // StorageClass is the storage class of the object. 775 // This value defines how objects in the bucket are stored and 776 // determines the SLA and the cost of storage. Typical values are 777 // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" 778 // and "DURABLE_REDUCED_AVAILABILITY". 779 // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL" 780 // or "REGIONAL" depending on the bucket's location settings. 781 StorageClass string 782 783 // Created is the time the object was created. This field is read-only. 784 Created time.Time 785 786 // Deleted is the time the object was deleted. 787 // If not deleted, it is the zero value. This field is read-only. 788 Deleted time.Time 789 790 // Updated is the creation or modification time of the object. 791 // For buckets with versioning enabled, changing an object's 792 // metadata does not change this property. This field is read-only. 793 Updated time.Time 794 795 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the 796 // customer-supplied encryption key for the object. It is empty if there is 797 // no customer-supplied encryption key. 798 // See // https://cloud.google.com/storage/docs/encryption for more about 799 // encryption in Google Cloud Storage. 800 CustomerKeySHA256 string 801 802 // Prefix is set only for ObjectAttrs which represent synthetic "directory 803 // entries" when iterating over buckets using Query.Delimiter. See 804 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be 805 // populated. 806 Prefix string 807} 808 809// convertTime converts a time in RFC3339 format to time.Time. 810// If any error occurs in parsing, the zero-value time.Time is silently returned. 811func convertTime(t string) time.Time { 812 var r time.Time 813 if t != "" { 814 r, _ = time.Parse(time.RFC3339, t) 815 } 816 return r 817} 818 819func newObject(o *raw.Object) *ObjectAttrs { 820 if o == nil { 821 return nil 822 } 823 acl := make([]ACLRule, len(o.Acl)) 824 for i, rule := range o.Acl { 825 acl[i] = ACLRule{ 826 Entity: ACLEntity(rule.Entity), 827 Role: ACLRole(rule.Role), 828 } 829 } 830 owner := "" 831 if o.Owner != nil { 832 owner = o.Owner.Entity 833 } 834 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) 835 crc32c, _ := decodeUint32(o.Crc32c) 836 var sha256 string 837 if o.CustomerEncryption != nil { 838 sha256 = o.CustomerEncryption.KeySha256 839 } 840 return &ObjectAttrs{ 841 Bucket: o.Bucket, 842 Name: o.Name, 843 ContentType: o.ContentType, 844 ContentLanguage: o.ContentLanguage, 845 CacheControl: o.CacheControl, 846 ACL: acl, 847 Owner: owner, 848 ContentEncoding: o.ContentEncoding, 849 ContentDisposition: o.ContentDisposition, 850 Size: int64(o.Size), 851 MD5: md5, 852 CRC32C: crc32c, 853 MediaLink: o.MediaLink, 854 Metadata: o.Metadata, 855 Generation: o.Generation, 856 Metageneration: o.Metageneration, 857 StorageClass: o.StorageClass, 858 CustomerKeySHA256: sha256, 859 Created: convertTime(o.TimeCreated), 860 Deleted: convertTime(o.TimeDeleted), 861 Updated: convertTime(o.Updated), 862 } 863} 864 865// Decode a uint32 encoded in Base64 in big-endian byte order. 866func decodeUint32(b64 string) (uint32, error) { 867 d, err := base64.StdEncoding.DecodeString(b64) 868 if err != nil { 869 return 0, err 870 } 871 if len(d) != 4 { 872 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) 873 } 874 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil 875} 876 877// Encode a uint32 as Base64 in big-endian byte order. 878func encodeUint32(u uint32) string { 879 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} 880 return base64.StdEncoding.EncodeToString(b) 881} 882 883// Query represents a query to filter objects from a bucket. 884type Query struct { 885 // Delimiter returns results in a directory-like fashion. 886 // Results will contain only objects whose names, aside from the 887 // prefix, do not contain delimiter. Objects whose names, 888 // aside from the prefix, contain delimiter will have their name, 889 // truncated after the delimiter, returned in prefixes. 890 // Duplicate prefixes are omitted. 891 // Optional. 892 Delimiter string 893 894 // Prefix is the prefix filter to query objects 895 // whose names begin with this prefix. 896 // Optional. 897 Prefix string 898 899 // Versions indicates whether multiple versions of the same 900 // object will be included in the results. 901 Versions bool 902} 903 904// contentTyper implements ContentTyper to enable an 905// io.ReadCloser to specify its MIME type. 906type contentTyper struct { 907 io.Reader 908 t string 909} 910 911func (c *contentTyper) ContentType() string { 912 return c.t 913} 914 915// Conditions constrain methods to act on specific generations of 916// objects. 917// 918// The zero value is an empty set of constraints. Not all conditions or 919// combinations of conditions are applicable to all methods. 920// See https://cloud.google.com/storage/docs/generations-preconditions 921// for details on how these operate. 922type Conditions struct { 923 // Generation constraints. 924 // At most one of the following can be set to a non-zero value. 925 926 // GenerationMatch specifies that the object must have the given generation 927 // for the operation to occur. 928 // If GenerationMatch is zero, it has no effect. 929 // Use DoesNotExist to specify that the object does not exist in the bucket. 930 GenerationMatch int64 931 932 // GenerationNotMatch specifies that the object must not have the given 933 // generation for the operation to occur. 934 // If GenerationNotMatch is zero, it has no effect. 935 GenerationNotMatch int64 936 937 // DoesNotExist specifies that the object must not exist in the bucket for 938 // the operation to occur. 939 // If DoesNotExist is false, it has no effect. 940 DoesNotExist bool 941 942 // Metadata generation constraints. 943 // At most one of the following can be set to a non-zero value. 944 945 // MetagenerationMatch specifies that the object must have the given 946 // metageneration for the operation to occur. 947 // If MetagenerationMatch is zero, it has no effect. 948 MetagenerationMatch int64 949 950 // MetagenerationNotMatch specifies that the object must not have the given 951 // metageneration for the operation to occur. 952 // If MetagenerationNotMatch is zero, it has no effect. 953 MetagenerationNotMatch int64 954} 955 956func (c *Conditions) validate(method string) error { 957 if *c == (Conditions{}) { 958 return fmt.Errorf("storage: %s: empty conditions", method) 959 } 960 if !c.isGenerationValid() { 961 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) 962 } 963 if !c.isMetagenerationValid() { 964 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 965 } 966 return nil 967} 968 969func (c *Conditions) isGenerationValid() bool { 970 n := 0 971 if c.GenerationMatch != 0 { 972 n++ 973 } 974 if c.GenerationNotMatch != 0 { 975 n++ 976 } 977 if c.DoesNotExist { 978 n++ 979 } 980 return n <= 1 981} 982 983func (c *Conditions) isMetagenerationValid() bool { 984 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 985} 986 987// applyConds modifies the provided call using the conditions in conds. 988// call is something that quacks like a *raw.WhateverCall. 989func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { 990 cval := reflect.ValueOf(call) 991 if gen >= 0 { 992 if !setConditionField(cval, "Generation", gen) { 993 return fmt.Errorf("storage: %s: generation not supported", method) 994 } 995 } 996 if conds == nil { 997 return nil 998 } 999 if err := conds.validate(method); err != nil { 1000 return err 1001 } 1002 switch { 1003 case conds.GenerationMatch != 0: 1004 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { 1005 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) 1006 } 1007 case conds.GenerationNotMatch != 0: 1008 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { 1009 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) 1010 } 1011 case conds.DoesNotExist: 1012 if !setConditionField(cval, "IfGenerationMatch", int64(0)) { 1013 return fmt.Errorf("storage: %s: DoesNotExist not supported", method) 1014 } 1015 } 1016 switch { 1017 case conds.MetagenerationMatch != 0: 1018 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 1019 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 1020 } 1021 case conds.MetagenerationNotMatch != 0: 1022 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 1023 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 1024 } 1025 } 1026 return nil 1027} 1028 1029func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { 1030 if gen >= 0 { 1031 call.SourceGeneration(gen) 1032 } 1033 if conds == nil { 1034 return nil 1035 } 1036 if err := conds.validate("CopyTo source"); err != nil { 1037 return err 1038 } 1039 switch { 1040 case conds.GenerationMatch != 0: 1041 call.IfSourceGenerationMatch(conds.GenerationMatch) 1042 case conds.GenerationNotMatch != 0: 1043 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) 1044 case conds.DoesNotExist: 1045 call.IfSourceGenerationMatch(0) 1046 } 1047 switch { 1048 case conds.MetagenerationMatch != 0: 1049 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) 1050 case conds.MetagenerationNotMatch != 0: 1051 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) 1052 } 1053 return nil 1054} 1055 1056// setConditionField sets a field on a *raw.WhateverCall. 1057// We can't use anonymous interfaces because the return type is 1058// different, since the field setters are builders. 1059func setConditionField(call reflect.Value, name string, value interface{}) bool { 1060 m := call.MethodByName(name) 1061 if !m.IsValid() { 1062 return false 1063 } 1064 m.Call([]reflect.Value{reflect.ValueOf(value)}) 1065 return true 1066} 1067 1068// conditionsQuery returns the generation and conditions as a URL query 1069// string suitable for URL.RawQuery. It assumes that the conditions 1070// have been validated. 1071func conditionsQuery(gen int64, conds *Conditions) string { 1072 // URL escapes are elided because integer strings are URL-safe. 1073 var buf []byte 1074 1075 appendParam := func(s string, n int64) { 1076 if len(buf) > 0 { 1077 buf = append(buf, '&') 1078 } 1079 buf = append(buf, s...) 1080 buf = strconv.AppendInt(buf, n, 10) 1081 } 1082 1083 if gen >= 0 { 1084 appendParam("generation=", gen) 1085 } 1086 if conds == nil { 1087 return string(buf) 1088 } 1089 switch { 1090 case conds.GenerationMatch != 0: 1091 appendParam("ifGenerationMatch=", conds.GenerationMatch) 1092 case conds.GenerationNotMatch != 0: 1093 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) 1094 case conds.DoesNotExist: 1095 appendParam("ifGenerationMatch=", 0) 1096 } 1097 switch { 1098 case conds.MetagenerationMatch != 0: 1099 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) 1100 case conds.MetagenerationNotMatch != 0: 1101 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) 1102 } 1103 return string(buf) 1104} 1105 1106// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods 1107// that modifyCall searches for by name. 1108type composeSourceObj struct { 1109 src *raw.ComposeRequestSourceObjects 1110} 1111 1112func (c composeSourceObj) Generation(gen int64) { 1113 c.src.Generation = gen 1114} 1115 1116func (c composeSourceObj) IfGenerationMatch(gen int64) { 1117 // It's safe to overwrite ObjectPreconditions, since its only field is 1118 // IfGenerationMatch. 1119 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{ 1120 IfGenerationMatch: gen, 1121 } 1122} 1123 1124func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error { 1125 if key == nil { 1126 return nil 1127 } 1128 // TODO(jbd): Ask the API team to return a more user-friendly error 1129 // and avoid doing this check at the client level. 1130 if len(key) != 32 { 1131 return errors.New("storage: not a 32-byte AES-256 key") 1132 } 1133 var cs string 1134 if copySource { 1135 cs = "copy-source-" 1136 } 1137 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") 1138 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) 1139 keyHash := sha256.Sum256(key) 1140 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) 1141 return nil 1142} 1143 1144// TODO(jbd): Add storage.objects.watch. 1145