1// Copyright 2014 Google Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "crypto" 20 "crypto/rand" 21 "crypto/rsa" 22 "crypto/sha256" 23 "crypto/x509" 24 "encoding/base64" 25 "encoding/pem" 26 "errors" 27 "fmt" 28 "io" 29 "io/ioutil" 30 "net/http" 31 "net/url" 32 "reflect" 33 "regexp" 34 "sort" 35 "strconv" 36 "strings" 37 "time" 38 "unicode/utf8" 39 40 "google.golang.org/api/option" 41 htransport "google.golang.org/api/transport/http" 42 43 "cloud.google.com/go/internal/optional" 44 "cloud.google.com/go/internal/version" 45 "golang.org/x/net/context" 46 "google.golang.org/api/googleapi" 47 raw "google.golang.org/api/storage/v1" 48) 49 50var ( 51 ErrBucketNotExist = errors.New("storage: bucket doesn't exist") 52 ErrObjectNotExist = errors.New("storage: object doesn't exist") 53) 54 55const userAgent = "gcloud-golang-storage/20151204" 56 57const ( 58 // ScopeFullControl grants permissions to manage your 59 // data and permissions in Google Cloud Storage. 60 ScopeFullControl = raw.DevstorageFullControlScope 61 62 // ScopeReadOnly grants permissions to 63 // view your data in Google Cloud Storage. 64 ScopeReadOnly = raw.DevstorageReadOnlyScope 65 66 // ScopeReadWrite grants permissions to manage your 67 // data in Google Cloud Storage. 68 ScopeReadWrite = raw.DevstorageReadWriteScope 69) 70 71var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) 72 73func setClientHeader(headers http.Header) { 74 headers.Set("x-goog-api-client", xGoogHeader) 75} 76 77// Client is a client for interacting with Google Cloud Storage. 78// 79// Clients should be reused instead of created as needed. 80// The methods of Client are safe for concurrent use by multiple goroutines. 81type Client struct { 82 hc *http.Client 83 raw *raw.Service 84} 85 86// NewClient creates a new Google Cloud Storage client. 87// The default scope is ScopeFullControl. To use a different scope, like ScopeReadOnly, use option.WithScopes. 88func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 89 o := []option.ClientOption{ 90 option.WithScopes(ScopeFullControl), 91 option.WithUserAgent(userAgent), 92 } 93 opts = append(o, opts...) 94 hc, ep, err := htransport.NewClient(ctx, opts...) 95 if err != nil { 96 return nil, fmt.Errorf("dialing: %v", err) 97 } 98 rawService, err := raw.New(hc) 99 if err != nil { 100 return nil, fmt.Errorf("storage client: %v", err) 101 } 102 if ep != "" { 103 rawService.BasePath = ep 104 } 105 return &Client{ 106 hc: hc, 107 raw: rawService, 108 }, nil 109} 110 111// Close closes the Client. 112// 113// Close need not be called at program exit. 114func (c *Client) Close() error { 115 // Set fields to nil so that subsequent uses 116 // will panic. 117 c.hc = nil 118 c.raw = nil 119 return nil 120} 121 122// SignedURLOptions allows you to restrict the access to the signed URL. 123type SignedURLOptions struct { 124 // GoogleAccessID represents the authorizer of the signed URL generation. 125 // It is typically the Google service account client email address from 126 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 127 // Required. 128 GoogleAccessID string 129 130 // PrivateKey is the Google service account private key. It is obtainable 131 // from the Google Developers Console. 132 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 133 // create a service account client ID or reuse one of your existing service account 134 // credentials. Click on the "Generate new P12 key" to generate and download 135 // a new private key. Once you download the P12 file, use the following command 136 // to convert it into a PEM file. 137 // 138 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 139 // 140 // Provide the contents of the PEM file as a byte slice. 141 // Exactly one of PrivateKey or SignBytes must be non-nil. 142 PrivateKey []byte 143 144 // SignBytes is a function for implementing custom signing. 145 // If your application is running on Google App Engine, you can use appengine's internal signing function: 146 // ctx := appengine.NewContext(request) 147 // acc, _ := appengine.ServiceAccount(ctx) 148 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 149 // GoogleAccessID: acc, 150 // SignBytes: func(b []byte) ([]byte, error) { 151 // _, signedBytes, err := appengine.SignBytes(ctx, b) 152 // return signedBytes, err 153 // }, 154 // // etc. 155 // }) 156 // 157 // Exactly one of PrivateKey or SignBytes must be non-nil. 158 SignBytes func([]byte) ([]byte, error) 159 160 // Method is the HTTP method to be used with the signed URL. 161 // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. 162 // Required. 163 Method string 164 165 // Expires is the expiration time on the signed URL. It must be 166 // a datetime in the future. 167 // Required. 168 Expires time.Time 169 170 // ContentType is the content type header the client must provide 171 // to use the generated signed URL. 172 // Optional. 173 ContentType string 174 175 // Headers is a list of extension headers the client must provide 176 // in order to use the generated signed URL. 177 // Optional. 178 Headers []string 179 180 // MD5 is the base64 encoded MD5 checksum of the file. 181 // If provided, the client should provide the exact value on the request 182 // header in order to use the signed URL. 183 // Optional. 184 MD5 string 185} 186 187var ( 188 canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) 189 excludedCanonicalHeaders = map[string]bool{ 190 "x-goog-encryption-key": true, 191 "x-goog-encryption-key-sha256": true, 192 } 193) 194 195// sanitizeHeaders applies the specifications for canonical extension headers at 196// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. 197func sanitizeHeaders(hdrs []string) []string { 198 headerMap := map[string][]string{} 199 for _, hdr := range hdrs { 200 // No leading or trailing whitespaces. 201 sanitizedHeader := strings.TrimSpace(hdr) 202 203 // Only keep canonical headers, discard any others. 204 headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) 205 if len(headerMatches) == 0 { 206 continue 207 } 208 209 header := strings.ToLower(strings.TrimSpace(headerMatches[1])) 210 if excludedCanonicalHeaders[headerMatches[1]] { 211 // Do not keep any deliberately excluded canonical headers when signing. 212 continue 213 } 214 value := strings.TrimSpace(headerMatches[2]) 215 if len(value) > 0 { 216 // Remove duplicate headers by appending the values of duplicates 217 // in their order of appearance. 218 headerMap[header] = append(headerMap[header], value) 219 } 220 } 221 222 var sanitizedHeaders []string 223 for header, values := range headerMap { 224 // There should be no spaces around the colon separating the 225 // header name from the header value or around the values 226 // themselves. The values should be separated by commas. 227 // NOTE: The semantics for headers without a value are not clear. 228 // However from specifications these should be edge-cases 229 // anyway and we should assume that there will be no 230 // canonical headers using empty values. Any such headers 231 // are discarded at the regexp stage above. 232 sanitizedHeaders = append( 233 sanitizedHeaders, 234 fmt.Sprintf("%s:%s", header, strings.Join(values, ",")), 235 ) 236 } 237 sort.Strings(sanitizedHeaders) 238 return sanitizedHeaders 239} 240 241// SignedURL returns a URL for the specified object. Signed URLs allow 242// the users access to a restricted resource for a limited time without having a 243// Google account or signing in. For more information about the signed 244// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. 245func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { 246 if opts == nil { 247 return "", errors.New("storage: missing required SignedURLOptions") 248 } 249 if opts.GoogleAccessID == "" { 250 return "", errors.New("storage: missing required GoogleAccessID") 251 } 252 if (opts.PrivateKey == nil) == (opts.SignBytes == nil) { 253 return "", errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 254 } 255 if opts.Method == "" { 256 return "", errors.New("storage: missing required method option") 257 } 258 if opts.Expires.IsZero() { 259 return "", errors.New("storage: missing required expires option") 260 } 261 if opts.MD5 != "" { 262 md5, err := base64.StdEncoding.DecodeString(opts.MD5) 263 if err != nil || len(md5) != 16 { 264 return "", errors.New("storage: invalid MD5 checksum") 265 } 266 } 267 opts.Headers = sanitizeHeaders(opts.Headers) 268 269 signBytes := opts.SignBytes 270 if opts.PrivateKey != nil { 271 key, err := parseKey(opts.PrivateKey) 272 if err != nil { 273 return "", err 274 } 275 signBytes = func(b []byte) ([]byte, error) { 276 sum := sha256.Sum256(b) 277 return rsa.SignPKCS1v15( 278 rand.Reader, 279 key, 280 crypto.SHA256, 281 sum[:], 282 ) 283 } 284 } 285 286 u := &url.URL{ 287 Path: fmt.Sprintf("/%s/%s", bucket, name), 288 } 289 290 buf := &bytes.Buffer{} 291 fmt.Fprintf(buf, "%s\n", opts.Method) 292 fmt.Fprintf(buf, "%s\n", opts.MD5) 293 fmt.Fprintf(buf, "%s\n", opts.ContentType) 294 fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) 295 if len(opts.Headers) > 0 { 296 fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) 297 } 298 fmt.Fprintf(buf, "%s", u.String()) 299 300 b, err := signBytes(buf.Bytes()) 301 if err != nil { 302 return "", err 303 } 304 encoded := base64.StdEncoding.EncodeToString(b) 305 u.Scheme = "https" 306 u.Host = "storage.googleapis.com" 307 q := u.Query() 308 q.Set("GoogleAccessId", opts.GoogleAccessID) 309 q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) 310 q.Set("Signature", string(encoded)) 311 u.RawQuery = q.Encode() 312 return u.String(), nil 313} 314 315// ObjectHandle provides operations on an object in a Google Cloud Storage bucket. 316// Use BucketHandle.Object to get a handle. 317type ObjectHandle struct { 318 c *Client 319 bucket string 320 object string 321 acl ACLHandle 322 gen int64 // a negative value indicates latest 323 conds *Conditions 324 encryptionKey []byte // AES-256 key 325 userProject string // for requester-pays buckets 326 readCompressed bool // Accept-Encoding: gzip 327} 328 329// ACL provides access to the object's access control list. 330// This controls who can read and write this object. 331// This call does not perform any network operations. 332func (o *ObjectHandle) ACL() *ACLHandle { 333 return &o.acl 334} 335 336// Generation returns a new ObjectHandle that operates on a specific generation 337// of the object. 338// By default, the handle operates on the latest generation. Not 339// all operations work when given a specific generation; check the API 340// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. 341func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { 342 o2 := *o 343 o2.gen = gen 344 return &o2 345} 346 347// If returns a new ObjectHandle that applies a set of preconditions. 348// Preconditions already set on the ObjectHandle are ignored. 349// Operations on the new handle will only occur if the preconditions are 350// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions 351// for more details. 352func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { 353 o2 := *o 354 o2.conds = &conds 355 return &o2 356} 357 358// Key returns a new ObjectHandle that uses the supplied encryption 359// key to encrypt and decrypt the object's contents. 360// 361// Encryption key must be a 32-byte AES-256 key. 362// See https://cloud.google.com/storage/docs/encryption for details. 363func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { 364 o2 := *o 365 o2.encryptionKey = encryptionKey 366 return &o2 367} 368 369// Attrs returns meta information about the object. 370// ErrObjectNotExist will be returned if the object is not found. 371func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { 372 if err := o.validate(); err != nil { 373 return nil, err 374 } 375 call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) 376 if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { 377 return nil, err 378 } 379 if o.userProject != "" { 380 call.UserProject(o.userProject) 381 } 382 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 383 return nil, err 384 } 385 var obj *raw.Object 386 var err error 387 setClientHeader(call.Header()) 388 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 389 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 390 return nil, ErrObjectNotExist 391 } 392 if err != nil { 393 return nil, err 394 } 395 return newObject(obj), nil 396} 397 398// Update updates an object with the provided attributes. 399// All zero-value attributes are ignored. 400// ErrObjectNotExist will be returned if the object is not found. 401func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { 402 if err := o.validate(); err != nil { 403 return nil, err 404 } 405 var attrs ObjectAttrs 406 // Lists of fields to send, and set to null, in the JSON. 407 var forceSendFields, nullFields []string 408 if uattrs.ContentType != nil { 409 attrs.ContentType = optional.ToString(uattrs.ContentType) 410 // For ContentType, sending the empty string is a no-op. 411 // Instead we send a null. 412 if attrs.ContentType == "" { 413 nullFields = append(nullFields, "ContentType") 414 } else { 415 forceSendFields = append(forceSendFields, "ContentType") 416 } 417 } 418 if uattrs.ContentLanguage != nil { 419 attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) 420 // For ContentLanguage it's an error to send the empty string. 421 // Instead we send a null. 422 if attrs.ContentLanguage == "" { 423 nullFields = append(nullFields, "ContentLanguage") 424 } else { 425 forceSendFields = append(forceSendFields, "ContentLanguage") 426 } 427 } 428 if uattrs.ContentEncoding != nil { 429 attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) 430 forceSendFields = append(forceSendFields, "ContentEncoding") 431 } 432 if uattrs.ContentDisposition != nil { 433 attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) 434 forceSendFields = append(forceSendFields, "ContentDisposition") 435 } 436 if uattrs.CacheControl != nil { 437 attrs.CacheControl = optional.ToString(uattrs.CacheControl) 438 forceSendFields = append(forceSendFields, "CacheControl") 439 } 440 if uattrs.Metadata != nil { 441 attrs.Metadata = uattrs.Metadata 442 if len(attrs.Metadata) == 0 { 443 // Sending the empty map is a no-op. We send null instead. 444 nullFields = append(nullFields, "Metadata") 445 } else { 446 forceSendFields = append(forceSendFields, "Metadata") 447 } 448 } 449 if uattrs.ACL != nil { 450 attrs.ACL = uattrs.ACL 451 // It's an error to attempt to delete the ACL, so 452 // we don't append to nullFields here. 453 forceSendFields = append(forceSendFields, "Acl") 454 } 455 rawObj := attrs.toRawObject(o.bucket) 456 rawObj.ForceSendFields = forceSendFields 457 rawObj.NullFields = nullFields 458 call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) 459 if err := applyConds("Update", o.gen, o.conds, call); err != nil { 460 return nil, err 461 } 462 if o.userProject != "" { 463 call.UserProject(o.userProject) 464 } 465 if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { 466 return nil, err 467 } 468 var obj *raw.Object 469 var err error 470 setClientHeader(call.Header()) 471 err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) 472 if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { 473 return nil, ErrObjectNotExist 474 } 475 if err != nil { 476 return nil, err 477 } 478 return newObject(obj), nil 479} 480 481// ObjectAttrsToUpdate is used to update the attributes of an object. 482// Only fields set to non-nil values will be updated. 483// Set a field to its zero value to delete it. 484// 485// For example, to change ContentType and delete ContentEncoding and 486// Metadata, use 487// ObjectAttrsToUpdate{ 488// ContentType: "text/html", 489// ContentEncoding: "", 490// Metadata: map[string]string{}, 491// } 492type ObjectAttrsToUpdate struct { 493 ContentType optional.String 494 ContentLanguage optional.String 495 ContentEncoding optional.String 496 ContentDisposition optional.String 497 CacheControl optional.String 498 Metadata map[string]string // set to map[string]string{} to delete 499 ACL []ACLRule 500} 501 502// Delete deletes the single specified object. 503func (o *ObjectHandle) Delete(ctx context.Context) error { 504 if err := o.validate(); err != nil { 505 return err 506 } 507 call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) 508 if err := applyConds("Delete", o.gen, o.conds, call); err != nil { 509 return err 510 } 511 if o.userProject != "" { 512 call.UserProject(o.userProject) 513 } 514 // Encryption doesn't apply to Delete. 515 setClientHeader(call.Header()) 516 err := runWithRetry(ctx, func() error { return call.Do() }) 517 switch e := err.(type) { 518 case nil: 519 return nil 520 case *googleapi.Error: 521 if e.Code == http.StatusNotFound { 522 return ErrObjectNotExist 523 } 524 } 525 return err 526} 527 528// ReadCompressed when true causes the read to happen without decompressing. 529func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { 530 o2 := *o 531 o2.readCompressed = compressed 532 return &o2 533} 534 535// NewReader creates a new Reader to read the contents of the 536// object. 537// ErrObjectNotExist will be returned if the object is not found. 538// 539// The caller must call Close on the returned Reader when done reading. 540func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { 541 return o.NewRangeReader(ctx, 0, -1) 542} 543 544// NewRangeReader reads part of an object, reading at most length bytes 545// starting at the given offset. If length is negative, the object is read 546// until the end. 547func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { 548 if err := o.validate(); err != nil { 549 return nil, err 550 } 551 if offset < 0 { 552 return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) 553 } 554 if o.conds != nil { 555 if err := o.conds.validate("NewRangeReader"); err != nil { 556 return nil, err 557 } 558 } 559 u := &url.URL{ 560 Scheme: "https", 561 Host: "storage.googleapis.com", 562 Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), 563 RawQuery: conditionsQuery(o.gen, o.conds), 564 } 565 verb := "GET" 566 if length == 0 { 567 verb = "HEAD" 568 } 569 req, err := http.NewRequest(verb, u.String(), nil) 570 if err != nil { 571 return nil, err 572 } 573 req = withContext(req, ctx) 574 if length < 0 && offset > 0 { 575 req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) 576 } else if length > 0 { 577 req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) 578 } 579 if o.userProject != "" { 580 req.Header.Set("X-Goog-User-Project", o.userProject) 581 } 582 if o.readCompressed { 583 req.Header.Set("Accept-Encoding", "gzip") 584 } 585 if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { 586 return nil, err 587 } 588 var res *http.Response 589 err = runWithRetry(ctx, func() error { 590 res, err = o.c.hc.Do(req) 591 if err != nil { 592 return err 593 } 594 if res.StatusCode == http.StatusNotFound { 595 res.Body.Close() 596 return ErrObjectNotExist 597 } 598 if res.StatusCode < 200 || res.StatusCode > 299 { 599 body, _ := ioutil.ReadAll(res.Body) 600 res.Body.Close() 601 return &googleapi.Error{ 602 Code: res.StatusCode, 603 Header: res.Header, 604 Body: string(body), 605 } 606 } 607 if offset > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { 608 res.Body.Close() 609 return errors.New("storage: partial request not satisfied") 610 } 611 return nil 612 }) 613 if err != nil { 614 return nil, err 615 } 616 617 var size int64 // total size of object, even if a range was requested. 618 if res.StatusCode == http.StatusPartialContent { 619 cr := strings.TrimSpace(res.Header.Get("Content-Range")) 620 if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { 621 return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) 622 } 623 size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) 624 if err != nil { 625 return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) 626 } 627 } else { 628 size = res.ContentLength 629 } 630 631 remain := res.ContentLength 632 body := res.Body 633 if length == 0 { 634 remain = 0 635 body.Close() 636 body = emptyBody 637 } 638 var ( 639 checkCRC bool 640 crc uint32 641 ) 642 // Even if there is a CRC header, we can't compute the hash on partial data. 643 if remain == size { 644 crc, checkCRC = parseCRC32c(res) 645 } 646 return &Reader{ 647 body: body, 648 size: size, 649 remain: remain, 650 contentType: res.Header.Get("Content-Type"), 651 contentEncoding: res.Header.Get("Content-Encoding"), 652 cacheControl: res.Header.Get("Cache-Control"), 653 wantCRC: crc, 654 checkCRC: checkCRC, 655 }, nil 656} 657 658func parseCRC32c(res *http.Response) (uint32, bool) { 659 const prefix = "crc32c=" 660 for _, spec := range res.Header["X-Goog-Hash"] { 661 if strings.HasPrefix(spec, prefix) { 662 c, err := decodeUint32(spec[len(prefix):]) 663 if err == nil { 664 return c, true 665 } 666 } 667 } 668 return 0, false 669} 670 671var emptyBody = ioutil.NopCloser(strings.NewReader("")) 672 673// NewWriter returns a storage Writer that writes to the GCS object 674// associated with this ObjectHandle. 675// 676// A new object will be created unless an object with this name already exists. 677// Otherwise any previous object with the same name will be replaced. 678// The object will not be available (and any previous object will remain) 679// until Close has been called. 680// 681// Attributes can be set on the object by modifying the returned Writer's 682// ObjectAttrs field before the first call to Write. If no ContentType 683// attribute is specified, the content type will be automatically sniffed 684// using net/http.DetectContentType. 685// 686// It is the caller's responsibility to call Close when writing is done. 687func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { 688 return &Writer{ 689 ctx: ctx, 690 o: o, 691 donec: make(chan struct{}), 692 ObjectAttrs: ObjectAttrs{Name: o.object}, 693 ChunkSize: googleapi.DefaultUploadChunkSize, 694 } 695} 696 697func (o *ObjectHandle) validate() error { 698 if o.bucket == "" { 699 return errors.New("storage: bucket name is empty") 700 } 701 if o.object == "" { 702 return errors.New("storage: object name is empty") 703 } 704 if !utf8.ValidString(o.object) { 705 return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) 706 } 707 return nil 708} 709 710// parseKey converts the binary contents of a private key file to an 711// *rsa.PrivateKey. It detects whether the private key is in a PEM container or 712// not. If so, it extracts the private key from PEM container before 713// conversion. It only supports PEM containers with no passphrase. 714func parseKey(key []byte) (*rsa.PrivateKey, error) { 715 if block, _ := pem.Decode(key); block != nil { 716 key = block.Bytes 717 } 718 parsedKey, err := x509.ParsePKCS8PrivateKey(key) 719 if err != nil { 720 parsedKey, err = x509.ParsePKCS1PrivateKey(key) 721 if err != nil { 722 return nil, err 723 } 724 } 725 parsed, ok := parsedKey.(*rsa.PrivateKey) 726 if !ok { 727 return nil, errors.New("oauth2: private key is invalid") 728 } 729 return parsed, nil 730} 731 732func toRawObjectACL(oldACL []ACLRule) []*raw.ObjectAccessControl { 733 var acl []*raw.ObjectAccessControl 734 if len(oldACL) > 0 { 735 acl = make([]*raw.ObjectAccessControl, len(oldACL)) 736 for i, rule := range oldACL { 737 acl[i] = &raw.ObjectAccessControl{ 738 Entity: string(rule.Entity), 739 Role: string(rule.Role), 740 } 741 } 742 } 743 return acl 744} 745 746// toRawObject copies the editable attributes from o to the raw library's Object type. 747func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { 748 acl := toRawObjectACL(o.ACL) 749 return &raw.Object{ 750 Bucket: bucket, 751 Name: o.Name, 752 ContentType: o.ContentType, 753 ContentEncoding: o.ContentEncoding, 754 ContentLanguage: o.ContentLanguage, 755 CacheControl: o.CacheControl, 756 ContentDisposition: o.ContentDisposition, 757 StorageClass: o.StorageClass, 758 Acl: acl, 759 Metadata: o.Metadata, 760 } 761} 762 763// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. 764type ObjectAttrs struct { 765 // Bucket is the name of the bucket containing this GCS object. 766 // This field is read-only. 767 Bucket string 768 769 // Name is the name of the object within the bucket. 770 // This field is read-only. 771 Name string 772 773 // ContentType is the MIME type of the object's content. 774 ContentType string 775 776 // ContentLanguage is the content language of the object's content. 777 ContentLanguage string 778 779 // CacheControl is the Cache-Control header to be sent in the response 780 // headers when serving the object data. 781 CacheControl string 782 783 // ACL is the list of access control rules for the object. 784 ACL []ACLRule 785 786 // Owner is the owner of the object. This field is read-only. 787 // 788 // If non-zero, it is in the form of "user-<userId>". 789 Owner string 790 791 // Size is the length of the object's content. This field is read-only. 792 Size int64 793 794 // ContentEncoding is the encoding of the object's content. 795 ContentEncoding string 796 797 // ContentDisposition is the optional Content-Disposition header of the object 798 // sent in the response headers. 799 ContentDisposition string 800 801 // MD5 is the MD5 hash of the object's content. This field is read-only, 802 // except when used from a Writer. If set on a Writer, the uploaded 803 // data is rejected if its MD5 hash does not match this field. 804 MD5 []byte 805 806 // CRC32C is the CRC32 checksum of the object's content using 807 // the Castagnoli93 polynomial. This field is read-only, except when 808 // used from a Writer. If set on a Writer and Writer.SendCRC32C 809 // is true, the uploaded data is rejected if its CRC32c hash does not 810 // match this field. 811 CRC32C uint32 812 813 // MediaLink is an URL to the object's content. This field is read-only. 814 MediaLink string 815 816 // Metadata represents user-provided metadata, in key/value pairs. 817 // It can be nil if no metadata is provided. 818 Metadata map[string]string 819 820 // Generation is the generation number of the object's content. 821 // This field is read-only. 822 Generation int64 823 824 // Metageneration is the version of the metadata for this 825 // object at this generation. This field is used for preconditions 826 // and for detecting changes in metadata. A metageneration number 827 // is only meaningful in the context of a particular generation 828 // of a particular object. This field is read-only. 829 Metageneration int64 830 831 // StorageClass is the storage class of the object. 832 // This value defines how objects in the bucket are stored and 833 // determines the SLA and the cost of storage. Typical values are 834 // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" 835 // and "DURABLE_REDUCED_AVAILABILITY". 836 // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL" 837 // or "REGIONAL" depending on the bucket's location settings. 838 StorageClass string 839 840 // Created is the time the object was created. This field is read-only. 841 Created time.Time 842 843 // Deleted is the time the object was deleted. 844 // If not deleted, it is the zero value. This field is read-only. 845 Deleted time.Time 846 847 // Updated is the creation or modification time of the object. 848 // For buckets with versioning enabled, changing an object's 849 // metadata does not change this property. This field is read-only. 850 Updated time.Time 851 852 // CustomerKeySHA256 is the base64-encoded SHA-256 hash of the 853 // customer-supplied encryption key for the object. It is empty if there is 854 // no customer-supplied encryption key. 855 // See // https://cloud.google.com/storage/docs/encryption for more about 856 // encryption in Google Cloud Storage. 857 CustomerKeySHA256 string 858 859 // Prefix is set only for ObjectAttrs which represent synthetic "directory 860 // entries" when iterating over buckets using Query.Delimiter. See 861 // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be 862 // populated. 863 Prefix string 864} 865 866// convertTime converts a time in RFC3339 format to time.Time. 867// If any error occurs in parsing, the zero-value time.Time is silently returned. 868func convertTime(t string) time.Time { 869 var r time.Time 870 if t != "" { 871 r, _ = time.Parse(time.RFC3339, t) 872 } 873 return r 874} 875 876func newObject(o *raw.Object) *ObjectAttrs { 877 if o == nil { 878 return nil 879 } 880 acl := make([]ACLRule, len(o.Acl)) 881 for i, rule := range o.Acl { 882 acl[i] = ACLRule{ 883 Entity: ACLEntity(rule.Entity), 884 Role: ACLRole(rule.Role), 885 } 886 } 887 owner := "" 888 if o.Owner != nil { 889 owner = o.Owner.Entity 890 } 891 md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) 892 crc32c, _ := decodeUint32(o.Crc32c) 893 var sha256 string 894 if o.CustomerEncryption != nil { 895 sha256 = o.CustomerEncryption.KeySha256 896 } 897 return &ObjectAttrs{ 898 Bucket: o.Bucket, 899 Name: o.Name, 900 ContentType: o.ContentType, 901 ContentLanguage: o.ContentLanguage, 902 CacheControl: o.CacheControl, 903 ACL: acl, 904 Owner: owner, 905 ContentEncoding: o.ContentEncoding, 906 ContentDisposition: o.ContentDisposition, 907 Size: int64(o.Size), 908 MD5: md5, 909 CRC32C: crc32c, 910 MediaLink: o.MediaLink, 911 Metadata: o.Metadata, 912 Generation: o.Generation, 913 Metageneration: o.Metageneration, 914 StorageClass: o.StorageClass, 915 CustomerKeySHA256: sha256, 916 Created: convertTime(o.TimeCreated), 917 Deleted: convertTime(o.TimeDeleted), 918 Updated: convertTime(o.Updated), 919 } 920} 921 922// Decode a uint32 encoded in Base64 in big-endian byte order. 923func decodeUint32(b64 string) (uint32, error) { 924 d, err := base64.StdEncoding.DecodeString(b64) 925 if err != nil { 926 return 0, err 927 } 928 if len(d) != 4 { 929 return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) 930 } 931 return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil 932} 933 934// Encode a uint32 as Base64 in big-endian byte order. 935func encodeUint32(u uint32) string { 936 b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} 937 return base64.StdEncoding.EncodeToString(b) 938} 939 940// Query represents a query to filter objects from a bucket. 941type Query struct { 942 // Delimiter returns results in a directory-like fashion. 943 // Results will contain only objects whose names, aside from the 944 // prefix, do not contain delimiter. Objects whose names, 945 // aside from the prefix, contain delimiter will have their name, 946 // truncated after the delimiter, returned in prefixes. 947 // Duplicate prefixes are omitted. 948 // Optional. 949 Delimiter string 950 951 // Prefix is the prefix filter to query objects 952 // whose names begin with this prefix. 953 // Optional. 954 Prefix string 955 956 // Versions indicates whether multiple versions of the same 957 // object will be included in the results. 958 Versions bool 959} 960 961// contentTyper implements ContentTyper to enable an 962// io.ReadCloser to specify its MIME type. 963type contentTyper struct { 964 io.Reader 965 t string 966} 967 968func (c *contentTyper) ContentType() string { 969 return c.t 970} 971 972// Conditions constrain methods to act on specific generations of 973// objects. 974// 975// The zero value is an empty set of constraints. Not all conditions or 976// combinations of conditions are applicable to all methods. 977// See https://cloud.google.com/storage/docs/generations-preconditions 978// for details on how these operate. 979type Conditions struct { 980 // Generation constraints. 981 // At most one of the following can be set to a non-zero value. 982 983 // GenerationMatch specifies that the object must have the given generation 984 // for the operation to occur. 985 // If GenerationMatch is zero, it has no effect. 986 // Use DoesNotExist to specify that the object does not exist in the bucket. 987 GenerationMatch int64 988 989 // GenerationNotMatch specifies that the object must not have the given 990 // generation for the operation to occur. 991 // If GenerationNotMatch is zero, it has no effect. 992 GenerationNotMatch int64 993 994 // DoesNotExist specifies that the object must not exist in the bucket for 995 // the operation to occur. 996 // If DoesNotExist is false, it has no effect. 997 DoesNotExist bool 998 999 // Metadata generation constraints. 1000 // At most one of the following can be set to a non-zero value. 1001 1002 // MetagenerationMatch specifies that the object must have the given 1003 // metageneration for the operation to occur. 1004 // If MetagenerationMatch is zero, it has no effect. 1005 MetagenerationMatch int64 1006 1007 // MetagenerationNotMatch specifies that the object must not have the given 1008 // metageneration for the operation to occur. 1009 // If MetagenerationNotMatch is zero, it has no effect. 1010 MetagenerationNotMatch int64 1011} 1012 1013func (c *Conditions) validate(method string) error { 1014 if *c == (Conditions{}) { 1015 return fmt.Errorf("storage: %s: empty conditions", method) 1016 } 1017 if !c.isGenerationValid() { 1018 return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) 1019 } 1020 if !c.isMetagenerationValid() { 1021 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 1022 } 1023 return nil 1024} 1025 1026func (c *Conditions) isGenerationValid() bool { 1027 n := 0 1028 if c.GenerationMatch != 0 { 1029 n++ 1030 } 1031 if c.GenerationNotMatch != 0 { 1032 n++ 1033 } 1034 if c.DoesNotExist { 1035 n++ 1036 } 1037 return n <= 1 1038} 1039 1040func (c *Conditions) isMetagenerationValid() bool { 1041 return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 1042} 1043 1044// applyConds modifies the provided call using the conditions in conds. 1045// call is something that quacks like a *raw.WhateverCall. 1046func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { 1047 cval := reflect.ValueOf(call) 1048 if gen >= 0 { 1049 if !setConditionField(cval, "Generation", gen) { 1050 return fmt.Errorf("storage: %s: generation not supported", method) 1051 } 1052 } 1053 if conds == nil { 1054 return nil 1055 } 1056 if err := conds.validate(method); err != nil { 1057 return err 1058 } 1059 switch { 1060 case conds.GenerationMatch != 0: 1061 if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { 1062 return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) 1063 } 1064 case conds.GenerationNotMatch != 0: 1065 if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { 1066 return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) 1067 } 1068 case conds.DoesNotExist: 1069 if !setConditionField(cval, "IfGenerationMatch", int64(0)) { 1070 return fmt.Errorf("storage: %s: DoesNotExist not supported", method) 1071 } 1072 } 1073 switch { 1074 case conds.MetagenerationMatch != 0: 1075 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 1076 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 1077 } 1078 case conds.MetagenerationNotMatch != 0: 1079 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 1080 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 1081 } 1082 } 1083 return nil 1084} 1085 1086func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { 1087 if gen >= 0 { 1088 call.SourceGeneration(gen) 1089 } 1090 if conds == nil { 1091 return nil 1092 } 1093 if err := conds.validate("CopyTo source"); err != nil { 1094 return err 1095 } 1096 switch { 1097 case conds.GenerationMatch != 0: 1098 call.IfSourceGenerationMatch(conds.GenerationMatch) 1099 case conds.GenerationNotMatch != 0: 1100 call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) 1101 case conds.DoesNotExist: 1102 call.IfSourceGenerationMatch(0) 1103 } 1104 switch { 1105 case conds.MetagenerationMatch != 0: 1106 call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) 1107 case conds.MetagenerationNotMatch != 0: 1108 call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) 1109 } 1110 return nil 1111} 1112 1113// setConditionField sets a field on a *raw.WhateverCall. 1114// We can't use anonymous interfaces because the return type is 1115// different, since the field setters are builders. 1116func setConditionField(call reflect.Value, name string, value interface{}) bool { 1117 m := call.MethodByName(name) 1118 if !m.IsValid() { 1119 return false 1120 } 1121 m.Call([]reflect.Value{reflect.ValueOf(value)}) 1122 return true 1123} 1124 1125// conditionsQuery returns the generation and conditions as a URL query 1126// string suitable for URL.RawQuery. It assumes that the conditions 1127// have been validated. 1128func conditionsQuery(gen int64, conds *Conditions) string { 1129 // URL escapes are elided because integer strings are URL-safe. 1130 var buf []byte 1131 1132 appendParam := func(s string, n int64) { 1133 if len(buf) > 0 { 1134 buf = append(buf, '&') 1135 } 1136 buf = append(buf, s...) 1137 buf = strconv.AppendInt(buf, n, 10) 1138 } 1139 1140 if gen >= 0 { 1141 appendParam("generation=", gen) 1142 } 1143 if conds == nil { 1144 return string(buf) 1145 } 1146 switch { 1147 case conds.GenerationMatch != 0: 1148 appendParam("ifGenerationMatch=", conds.GenerationMatch) 1149 case conds.GenerationNotMatch != 0: 1150 appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) 1151 case conds.DoesNotExist: 1152 appendParam("ifGenerationMatch=", 0) 1153 } 1154 switch { 1155 case conds.MetagenerationMatch != 0: 1156 appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) 1157 case conds.MetagenerationNotMatch != 0: 1158 appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) 1159 } 1160 return string(buf) 1161} 1162 1163// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods 1164// that modifyCall searches for by name. 1165type composeSourceObj struct { 1166 src *raw.ComposeRequestSourceObjects 1167} 1168 1169func (c composeSourceObj) Generation(gen int64) { 1170 c.src.Generation = gen 1171} 1172 1173func (c composeSourceObj) IfGenerationMatch(gen int64) { 1174 // It's safe to overwrite ObjectPreconditions, since its only field is 1175 // IfGenerationMatch. 1176 c.src.ObjectPreconditions = &raw.ComposeRequestSourceObjectsObjectPreconditions{ 1177 IfGenerationMatch: gen, 1178 } 1179} 1180 1181func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) error { 1182 if key == nil { 1183 return nil 1184 } 1185 // TODO(jbd): Ask the API team to return a more user-friendly error 1186 // and avoid doing this check at the client level. 1187 if len(key) != 32 { 1188 return errors.New("storage: not a 32-byte AES-256 key") 1189 } 1190 var cs string 1191 if copySource { 1192 cs = "copy-source-" 1193 } 1194 headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256") 1195 headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key)) 1196 keyHash := sha256.Sum256(key) 1197 headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:])) 1198 return nil 1199} 1200 1201// TODO(jbd): Add storage.objects.watch. 1202