1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "context" 19 "encoding/base64" 20 "encoding/json" 21 "errors" 22 "fmt" 23 "net/http" 24 "reflect" 25 "time" 26 27 "cloud.google.com/go/compute/metadata" 28 "cloud.google.com/go/internal/optional" 29 "cloud.google.com/go/internal/trace" 30 "golang.org/x/xerrors" 31 "google.golang.org/api/googleapi" 32 "google.golang.org/api/iamcredentials/v1" 33 "google.golang.org/api/iterator" 34 "google.golang.org/api/option" 35 raw "google.golang.org/api/storage/v1" 36) 37 38// BucketHandle provides operations on a Google Cloud Storage bucket. 39// Use Client.Bucket to get a handle. 40type BucketHandle struct { 41 c *Client 42 name string 43 acl ACLHandle 44 defaultObjectACL ACLHandle 45 conds *BucketConditions 46 userProject string // project for Requester Pays buckets 47} 48 49// Bucket returns a BucketHandle, which provides operations on the named bucket. 50// This call does not perform any network operations. 51// 52// The supplied name must contain only lowercase letters, numbers, dashes, 53// underscores, and dots. The full specification for valid bucket names can be 54// found at: 55// https://cloud.google.com/storage/docs/bucket-naming 56func (c *Client) Bucket(name string) *BucketHandle { 57 return &BucketHandle{ 58 c: c, 59 name: name, 60 acl: ACLHandle{ 61 c: c, 62 bucket: name, 63 }, 64 defaultObjectACL: ACLHandle{ 65 c: c, 66 bucket: name, 67 isDefault: true, 68 }, 69 } 70} 71 72// Create creates the Bucket in the project. 73// If attrs is nil the API defaults will be used. 74func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) (err error) { 75 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") 76 defer func() { trace.EndSpan(ctx, err) }() 77 78 var bkt *raw.Bucket 79 if attrs != nil { 80 bkt = attrs.toRawBucket() 81 } else { 82 bkt = &raw.Bucket{} 83 } 84 bkt.Name = b.name 85 // If there is lifecycle information but no location, explicitly set 86 // the location. This is a GCS quirk/bug. 87 if bkt.Location == "" && bkt.Lifecycle != nil { 88 bkt.Location = "US" 89 } 90 req := b.c.raw.Buckets.Insert(projectID, bkt) 91 setClientHeader(req.Header()) 92 if attrs != nil && attrs.PredefinedACL != "" { 93 req.PredefinedAcl(attrs.PredefinedACL) 94 } 95 if attrs != nil && attrs.PredefinedDefaultObjectACL != "" { 96 req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL) 97 } 98 return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) 99} 100 101// Delete deletes the Bucket. 102func (b *BucketHandle) Delete(ctx context.Context) (err error) { 103 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Delete") 104 defer func() { trace.EndSpan(ctx, err) }() 105 106 req, err := b.newDeleteCall() 107 if err != nil { 108 return err 109 } 110 return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) 111} 112 113func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) { 114 req := b.c.raw.Buckets.Delete(b.name) 115 setClientHeader(req.Header()) 116 if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil { 117 return nil, err 118 } 119 if b.userProject != "" { 120 req.UserProject(b.userProject) 121 } 122 return req, nil 123} 124 125// ACL returns an ACLHandle, which provides access to the bucket's access control list. 126// This controls who can list, create or overwrite the objects in a bucket. 127// This call does not perform any network operations. 128func (b *BucketHandle) ACL() *ACLHandle { 129 return &b.acl 130} 131 132// DefaultObjectACL returns an ACLHandle, which provides access to the bucket's default object ACLs. 133// These ACLs are applied to newly created objects in this bucket that do not have a defined ACL. 134// This call does not perform any network operations. 135func (b *BucketHandle) DefaultObjectACL() *ACLHandle { 136 return &b.defaultObjectACL 137} 138 139// Object returns an ObjectHandle, which provides operations on the named object. 140// This call does not perform any network operations such as fetching the object or verifying its existence. 141// Use methods on ObjectHandle to perform network operations. 142// 143// name must consist entirely of valid UTF-8-encoded runes. The full specification 144// for valid object names can be found at: 145// https://cloud.google.com/storage/docs/naming-objects 146func (b *BucketHandle) Object(name string) *ObjectHandle { 147 return &ObjectHandle{ 148 c: b.c, 149 bucket: b.name, 150 object: name, 151 acl: ACLHandle{ 152 c: b.c, 153 bucket: b.name, 154 object: name, 155 userProject: b.userProject, 156 }, 157 gen: -1, 158 userProject: b.userProject, 159 } 160} 161 162// Attrs returns the metadata for the bucket. 163func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error) { 164 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Attrs") 165 defer func() { trace.EndSpan(ctx, err) }() 166 167 req, err := b.newGetCall() 168 if err != nil { 169 return nil, err 170 } 171 var resp *raw.Bucket 172 err = runWithRetry(ctx, func() error { 173 resp, err = req.Context(ctx).Do() 174 return err 175 }) 176 var e *googleapi.Error 177 if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { 178 return nil, ErrBucketNotExist 179 } 180 if err != nil { 181 return nil, err 182 } 183 return newBucket(resp) 184} 185 186func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { 187 req := b.c.raw.Buckets.Get(b.name).Projection("full") 188 setClientHeader(req.Header()) 189 if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil { 190 return nil, err 191 } 192 if b.userProject != "" { 193 req.UserProject(b.userProject) 194 } 195 return req, nil 196} 197 198// Update updates a bucket's attributes. 199func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (attrs *BucketAttrs, err error) { 200 ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") 201 defer func() { trace.EndSpan(ctx, err) }() 202 203 req, err := b.newPatchCall(&uattrs) 204 if err != nil { 205 return nil, err 206 } 207 if uattrs.PredefinedACL != "" { 208 req.PredefinedAcl(uattrs.PredefinedACL) 209 } 210 if uattrs.PredefinedDefaultObjectACL != "" { 211 req.PredefinedDefaultObjectAcl(uattrs.PredefinedDefaultObjectACL) 212 } 213 // TODO(jba): retry iff metagen is set? 214 rb, err := req.Context(ctx).Do() 215 if err != nil { 216 return nil, err 217 } 218 return newBucket(rb) 219} 220 221func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) { 222 rb := uattrs.toRawBucket() 223 req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full") 224 setClientHeader(req.Header()) 225 if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil { 226 return nil, err 227 } 228 if b.userProject != "" { 229 req.UserProject(b.userProject) 230 } 231 return req, nil 232} 233 234// SignedURL returns a URL for the specified object. Signed URLs allow anyone 235// access to a restricted resource for a limited time without needing a 236// Google account or signing in. For more information about signed URLs, see 237// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication 238// 239// This method only requires the Method and Expires fields in the specified 240// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the 241// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable. 242// If you are authenticating with a custom HTTP client, Service Account based 243// auto-detection will be hindered. 244// 245// If no private key is found, it attempts to use the GoogleAccessID to sign the URL. 246// This requires the IAM Service Account Credentials API to be enabled 247// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview) 248// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. 249// If you do not want these fields set for you, you may pass them in through opts or use 250// SignedURL(bucket, name string, opts *SignedURLOptions) instead. 251func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) { 252 if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) { 253 return SignedURL(b.name, object, opts) 254 } 255 // Make a copy of opts so we don't modify the pointer parameter. 256 newopts := opts.clone() 257 258 if newopts.GoogleAccessID == "" { 259 id, err := b.detectDefaultGoogleAccessID() 260 if err != nil { 261 return "", err 262 } 263 newopts.GoogleAccessID = id 264 } 265 if newopts.SignBytes == nil && len(newopts.PrivateKey) == 0 { 266 if b.c.creds != nil && len(b.c.creds.JSON) > 0 { 267 var sa struct { 268 PrivateKey string `json:"private_key"` 269 } 270 err := json.Unmarshal(b.c.creds.JSON, &sa) 271 if err == nil && sa.PrivateKey != "" { 272 newopts.PrivateKey = []byte(sa.PrivateKey) 273 } 274 } 275 276 // Don't error out if we can't unmarshal the private key from the client, 277 // fallback to the default sign function for the service account. 278 if len(newopts.PrivateKey) == 0 { 279 newopts.SignBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID) 280 } 281 } 282 return SignedURL(b.name, object, newopts) 283} 284 285// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to 286// omit PrivateKey/SignBytes 287 288func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { 289 returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)") 290 291 if b.c.creds != nil && len(b.c.creds.JSON) > 0 { 292 var sa struct { 293 ClientEmail string `json:"client_email"` 294 } 295 err := json.Unmarshal(b.c.creds.JSON, &sa) 296 if err == nil && sa.ClientEmail != "" { 297 return sa.ClientEmail, nil 298 } else if err != nil { 299 returnErr = err 300 } else { 301 returnErr = errors.New("storage: empty client email in credentials") 302 } 303 304 } 305 306 // Don't error out if we can't unmarshal, fallback to GCE check. 307 if metadata.OnGCE() { 308 email, err := metadata.Email("default") 309 if err == nil && email != "" { 310 return email, nil 311 } else if err != nil { 312 returnErr = err 313 } else { 314 returnErr = errors.New("got empty email from GCE metadata service") 315 } 316 317 } 318 return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr) 319} 320 321func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) { 322 return func(in []byte) ([]byte, error) { 323 ctx := context.Background() 324 325 // It's ok to recreate this service per call since we pass in the http client, 326 // circumventing the cost of recreating the auth/transport layer 327 svc, err := iamcredentials.NewService(ctx, option.WithHTTPClient(b.c.hc)) 328 if err != nil { 329 return nil, fmt.Errorf("unable to create iamcredentials client: %v", err) 330 } 331 332 resp, err := svc.Projects.ServiceAccounts.SignBlob(fmt.Sprintf("projects/-/serviceAccounts/%s", email), &iamcredentials.SignBlobRequest{ 333 Payload: base64.StdEncoding.EncodeToString(in), 334 }).Do() 335 if err != nil { 336 return nil, fmt.Errorf("unable to sign bytes: %v", err) 337 } 338 out, err := base64.StdEncoding.DecodeString(resp.SignedBlob) 339 if err != nil { 340 return nil, fmt.Errorf("unable to base64 decode response: %v", err) 341 } 342 return out, nil 343 } 344} 345 346// BucketAttrs represents the metadata for a Google Cloud Storage bucket. 347// Read-only fields are ignored by BucketHandle.Create. 348type BucketAttrs struct { 349 // Name is the name of the bucket. 350 // This field is read-only. 351 Name string 352 353 // ACL is the list of access control rules on the bucket. 354 ACL []ACLRule 355 356 // BucketPolicyOnly is an alias for UniformBucketLevelAccess. Use of 357 // UniformBucketLevelAccess is recommended above the use of this field. 358 // Setting BucketPolicyOnly.Enabled OR UniformBucketLevelAccess.Enabled to 359 // true, will enable UniformBucketLevelAccess. 360 BucketPolicyOnly BucketPolicyOnly 361 362 // UniformBucketLevelAccess configures access checks to use only bucket-level IAM 363 // policies and ignore any ACL rules for the bucket. 364 // See https://cloud.google.com/storage/docs/uniform-bucket-level-access 365 // for more information. 366 UniformBucketLevelAccess UniformBucketLevelAccess 367 368 // PublicAccessPrevention is the setting for the bucket's 369 // PublicAccessPrevention policy, which can be used to prevent public access 370 // of data in the bucket. See 371 // https://cloud.google.com/storage/docs/public-access-prevention for more 372 // information. 373 PublicAccessPrevention PublicAccessPrevention 374 375 // DefaultObjectACL is the list of access controls to 376 // apply to new objects when no object ACL is provided. 377 DefaultObjectACL []ACLRule 378 379 // DefaultEventBasedHold is the default value for event-based hold on 380 // newly created objects in this bucket. It defaults to false. 381 DefaultEventBasedHold bool 382 383 // If not empty, applies a predefined set of access controls. It should be set 384 // only when creating a bucket. 385 // It is always empty for BucketAttrs returned from the service. 386 // See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert 387 // for valid values. 388 PredefinedACL string 389 390 // If not empty, applies a predefined set of default object access controls. 391 // It should be set only when creating a bucket. 392 // It is always empty for BucketAttrs returned from the service. 393 // See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert 394 // for valid values. 395 PredefinedDefaultObjectACL string 396 397 // Location is the location of the bucket. It defaults to "US". 398 Location string 399 400 // MetaGeneration is the metadata generation of the bucket. 401 // This field is read-only. 402 MetaGeneration int64 403 404 // StorageClass is the default storage class of the bucket. This defines 405 // how objects in the bucket are stored and determines the SLA 406 // and the cost of storage. Typical values are "STANDARD", "NEARLINE", 407 // "COLDLINE" and "ARCHIVE". Defaults to "STANDARD". 408 // See https://cloud.google.com/storage/docs/storage-classes for all 409 // valid values. 410 StorageClass string 411 412 // Created is the creation time of the bucket. 413 // This field is read-only. 414 Created time.Time 415 416 // VersioningEnabled reports whether this bucket has versioning enabled. 417 VersioningEnabled bool 418 419 // Labels are the bucket's labels. 420 Labels map[string]string 421 422 // RequesterPays reports whether the bucket is a Requester Pays bucket. 423 // Clients performing operations on Requester Pays buckets must provide 424 // a user project (see BucketHandle.UserProject), which will be billed 425 // for the operations. 426 RequesterPays bool 427 428 // Lifecycle is the lifecycle configuration for objects in the bucket. 429 Lifecycle Lifecycle 430 431 // Retention policy enforces a minimum retention time for all objects 432 // contained in the bucket. A RetentionPolicy of nil implies the bucket 433 // has no minimum data retention. 434 // 435 // This feature is in private alpha release. It is not currently available to 436 // most customers. It might be changed in backwards-incompatible ways and is not 437 // subject to any SLA or deprecation policy. 438 RetentionPolicy *RetentionPolicy 439 440 // The bucket's Cross-Origin Resource Sharing (CORS) configuration. 441 CORS []CORS 442 443 // The encryption configuration used by default for newly inserted objects. 444 Encryption *BucketEncryption 445 446 // The logging configuration. 447 Logging *BucketLogging 448 449 // The website configuration. 450 Website *BucketWebsite 451 452 // Etag is the HTTP/1.1 Entity tag for the bucket. 453 // This field is read-only. 454 Etag string 455 456 // LocationType describes how data is stored and replicated. 457 // Typical values are "multi-region", "region" and "dual-region". 458 // This field is read-only. 459 LocationType string 460 461 // The project number of the project the bucket belongs to. 462 // This field is read-only. 463 ProjectNumber uint64 464} 465 466// BucketPolicyOnly is an alias for UniformBucketLevelAccess. 467// Use of UniformBucketLevelAccess is preferred above BucketPolicyOnly. 468type BucketPolicyOnly struct { 469 // Enabled specifies whether access checks use only bucket-level IAM 470 // policies. Enabled may be disabled until the locked time. 471 Enabled bool 472 // LockedTime specifies the deadline for changing Enabled from true to 473 // false. 474 LockedTime time.Time 475} 476 477// UniformBucketLevelAccess configures access checks to use only bucket-level IAM 478// policies. 479type UniformBucketLevelAccess struct { 480 // Enabled specifies whether access checks use only bucket-level IAM 481 // policies. Enabled may be disabled until the locked time. 482 Enabled bool 483 // LockedTime specifies the deadline for changing Enabled from true to 484 // false. 485 LockedTime time.Time 486} 487 488// PublicAccessPrevention configures the Public Access Prevention feature, which 489// can be used to disallow public access to any data in a bucket. See 490// https://cloud.google.com/storage/docs/public-access-prevention for more 491// information. 492type PublicAccessPrevention int 493 494const ( 495 // PublicAccessPreventionUnknown is a zero value, used only if this field is 496 // not set in a call to GCS. 497 PublicAccessPreventionUnknown PublicAccessPrevention = iota 498 499 // PublicAccessPreventionUnspecified corresponds to a value of "unspecified". 500 // Deprecated: use PublicAccessPreventionInherited 501 PublicAccessPreventionUnspecified 502 503 // PublicAccessPreventionEnforced corresponds to a value of "enforced". This 504 // enforces Public Access Prevention on the bucket. 505 PublicAccessPreventionEnforced 506 507 // PublicAccessPreventionInherited corresponds to a value of "inherited" 508 // and is the default for buckets. 509 PublicAccessPreventionInherited 510 511 publicAccessPreventionUnknown string = "" 512 // TODO: remove unspecified when change is fully completed 513 publicAccessPreventionUnspecified = "unspecified" 514 publicAccessPreventionEnforced = "enforced" 515 publicAccessPreventionInherited = "inherited" 516) 517 518func (p PublicAccessPrevention) String() string { 519 switch p { 520 case PublicAccessPreventionInherited, PublicAccessPreventionUnspecified: 521 return publicAccessPreventionInherited 522 case PublicAccessPreventionEnforced: 523 return publicAccessPreventionEnforced 524 default: 525 return publicAccessPreventionUnknown 526 } 527} 528 529// Lifecycle is the lifecycle configuration for objects in the bucket. 530type Lifecycle struct { 531 Rules []LifecycleRule 532} 533 534// RetentionPolicy enforces a minimum retention time for all objects 535// contained in the bucket. 536// 537// Any attempt to overwrite or delete objects younger than the retention 538// period will result in an error. An unlocked retention policy can be 539// modified or removed from the bucket via the Update method. A 540// locked retention policy cannot be removed or shortened in duration 541// for the lifetime of the bucket. 542// 543// This feature is in private alpha release. It is not currently available to 544// most customers. It might be changed in backwards-incompatible ways and is not 545// subject to any SLA or deprecation policy. 546type RetentionPolicy struct { 547 // RetentionPeriod specifies the duration that objects need to be 548 // retained. Retention duration must be greater than zero and less than 549 // 100 years. Note that enforcement of retention periods less than a day 550 // is not guaranteed. Such periods should only be used for testing 551 // purposes. 552 RetentionPeriod time.Duration 553 554 // EffectiveTime is the time from which the policy was enforced and 555 // effective. This field is read-only. 556 EffectiveTime time.Time 557 558 // IsLocked describes whether the bucket is locked. Once locked, an 559 // object retention policy cannot be modified. 560 // This field is read-only. 561 IsLocked bool 562} 563 564const ( 565 // RFC3339 timestamp with only the date segment, used for CreatedBefore, 566 // CustomTimeBefore, and NoncurrentTimeBefore in LifecycleRule. 567 rfc3339Date = "2006-01-02" 568 569 // DeleteAction is a lifecycle action that deletes a live and/or archived 570 // objects. Takes precedence over SetStorageClass actions. 571 DeleteAction = "Delete" 572 573 // SetStorageClassAction changes the storage class of live and/or archived 574 // objects. 575 SetStorageClassAction = "SetStorageClass" 576) 577 578// LifecycleRule is a lifecycle configuration rule. 579// 580// When all the configured conditions are met by an object in the bucket, the 581// configured action will automatically be taken on that object. 582type LifecycleRule struct { 583 // Action is the action to take when all of the associated conditions are 584 // met. 585 Action LifecycleAction 586 587 // Condition is the set of conditions that must be met for the associated 588 // action to be taken. 589 Condition LifecycleCondition 590} 591 592// LifecycleAction is a lifecycle configuration action. 593type LifecycleAction struct { 594 // Type is the type of action to take on matching objects. 595 // 596 // Acceptable values are "Delete" to delete matching objects and 597 // "SetStorageClass" to set the storage class defined in StorageClass on 598 // matching objects. 599 Type string 600 601 // StorageClass is the storage class to set on matching objects if the Action 602 // is "SetStorageClass". 603 StorageClass string 604} 605 606// Liveness specifies whether the object is live or not. 607type Liveness int 608 609const ( 610 // LiveAndArchived includes both live and archived objects. 611 LiveAndArchived Liveness = iota 612 // Live specifies that the object is still live. 613 Live 614 // Archived specifies that the object is archived. 615 Archived 616) 617 618// LifecycleCondition is a set of conditions used to match objects and take an 619// action automatically. 620// 621// All configured conditions must be met for the associated action to be taken. 622type LifecycleCondition struct { 623 // AgeInDays is the age of the object in days. 624 AgeInDays int64 625 626 // CreatedBefore is the time the object was created. 627 // 628 // This condition is satisfied when an object is created before midnight of 629 // the specified date in UTC. 630 CreatedBefore time.Time 631 632 // CustomTimeBefore is the CustomTime metadata field of the object. This 633 // condition is satisfied when an object's CustomTime timestamp is before 634 // midnight of the specified date in UTC. 635 // 636 // This condition can only be satisfied if CustomTime has been set. 637 CustomTimeBefore time.Time 638 639 // DaysSinceCustomTime is the days elapsed since the CustomTime date of the 640 // object. This condition can only be satisfied if CustomTime has been set. 641 DaysSinceCustomTime int64 642 643 // DaysSinceNoncurrentTime is the days elapsed since the noncurrent timestamp 644 // of the object. This condition is relevant only for versioned objects. 645 DaysSinceNoncurrentTime int64 646 647 // Liveness specifies the object's liveness. Relevant only for versioned objects 648 Liveness Liveness 649 650 // MatchesStorageClasses is the condition matching the object's storage 651 // class. 652 // 653 // Values include "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE". 654 MatchesStorageClasses []string 655 656 // NoncurrentTimeBefore is the noncurrent timestamp of the object. This 657 // condition is satisfied when an object's noncurrent timestamp is before 658 // midnight of the specified date in UTC. 659 // 660 // This condition is relevant only for versioned objects. 661 NoncurrentTimeBefore time.Time 662 663 // NumNewerVersions is the condition matching objects with a number of newer versions. 664 // 665 // If the value is N, this condition is satisfied when there are at least N 666 // versions (including the live version) newer than this version of the 667 // object. 668 NumNewerVersions int64 669} 670 671// BucketLogging holds the bucket's logging configuration, which defines the 672// destination bucket and optional name prefix for the current bucket's 673// logs. 674type BucketLogging struct { 675 // The destination bucket where the current bucket's logs 676 // should be placed. 677 LogBucket string 678 679 // A prefix for log object names. 680 LogObjectPrefix string 681} 682 683// BucketWebsite holds the bucket's website configuration, controlling how the 684// service behaves when accessing bucket contents as a web site. See 685// https://cloud.google.com/storage/docs/static-website for more information. 686type BucketWebsite struct { 687 // If the requested object path is missing, the service will ensure the path has 688 // a trailing '/', append this suffix, and attempt to retrieve the resulting 689 // object. This allows the creation of index.html objects to represent directory 690 // pages. 691 MainPageSuffix string 692 693 // If the requested object path is missing, and any mainPageSuffix object is 694 // missing, if applicable, the service will return the named object from this 695 // bucket as the content for a 404 Not Found result. 696 NotFoundPage string 697} 698 699func newBucket(b *raw.Bucket) (*BucketAttrs, error) { 700 if b == nil { 701 return nil, nil 702 } 703 rp, err := toRetentionPolicy(b.RetentionPolicy) 704 if err != nil { 705 return nil, err 706 } 707 return &BucketAttrs{ 708 Name: b.Name, 709 Location: b.Location, 710 MetaGeneration: b.Metageneration, 711 DefaultEventBasedHold: b.DefaultEventBasedHold, 712 StorageClass: b.StorageClass, 713 Created: convertTime(b.TimeCreated), 714 VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled, 715 ACL: toBucketACLRules(b.Acl), 716 DefaultObjectACL: toObjectACLRules(b.DefaultObjectAcl), 717 Labels: b.Labels, 718 RequesterPays: b.Billing != nil && b.Billing.RequesterPays, 719 Lifecycle: toLifecycle(b.Lifecycle), 720 RetentionPolicy: rp, 721 CORS: toCORS(b.Cors), 722 Encryption: toBucketEncryption(b.Encryption), 723 Logging: toBucketLogging(b.Logging), 724 Website: toBucketWebsite(b.Website), 725 BucketPolicyOnly: toBucketPolicyOnly(b.IamConfiguration), 726 UniformBucketLevelAccess: toUniformBucketLevelAccess(b.IamConfiguration), 727 PublicAccessPrevention: toPublicAccessPrevention(b.IamConfiguration), 728 Etag: b.Etag, 729 LocationType: b.LocationType, 730 ProjectNumber: b.ProjectNumber, 731 }, nil 732} 733 734// toRawBucket copies the editable attribute from b to the raw library's Bucket type. 735func (b *BucketAttrs) toRawBucket() *raw.Bucket { 736 // Copy label map. 737 var labels map[string]string 738 if len(b.Labels) > 0 { 739 labels = make(map[string]string, len(b.Labels)) 740 for k, v := range b.Labels { 741 labels[k] = v 742 } 743 } 744 // Ignore VersioningEnabled if it is false. This is OK because 745 // we only call this method when creating a bucket, and by default 746 // new buckets have versioning off. 747 var v *raw.BucketVersioning 748 if b.VersioningEnabled { 749 v = &raw.BucketVersioning{Enabled: true} 750 } 751 var bb *raw.BucketBilling 752 if b.RequesterPays { 753 bb = &raw.BucketBilling{RequesterPays: true} 754 } 755 var bktIAM *raw.BucketIamConfiguration 756 if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled || b.PublicAccessPrevention != PublicAccessPreventionUnknown { 757 bktIAM = &raw.BucketIamConfiguration{} 758 if b.UniformBucketLevelAccess.Enabled || b.BucketPolicyOnly.Enabled { 759 bktIAM.UniformBucketLevelAccess = &raw.BucketIamConfigurationUniformBucketLevelAccess{ 760 Enabled: true, 761 } 762 } 763 if b.PublicAccessPrevention != PublicAccessPreventionUnknown { 764 bktIAM.PublicAccessPrevention = b.PublicAccessPrevention.String() 765 } 766 } 767 return &raw.Bucket{ 768 Name: b.Name, 769 Location: b.Location, 770 StorageClass: b.StorageClass, 771 Acl: toRawBucketACL(b.ACL), 772 DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL), 773 Versioning: v, 774 Labels: labels, 775 Billing: bb, 776 Lifecycle: toRawLifecycle(b.Lifecycle), 777 RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(), 778 Cors: toRawCORS(b.CORS), 779 Encryption: b.Encryption.toRawBucketEncryption(), 780 Logging: b.Logging.toRawBucketLogging(), 781 Website: b.Website.toRawBucketWebsite(), 782 IamConfiguration: bktIAM, 783 } 784} 785 786// CORS is the bucket's Cross-Origin Resource Sharing (CORS) configuration. 787type CORS struct { 788 // MaxAge is the value to return in the Access-Control-Max-Age 789 // header used in preflight responses. 790 MaxAge time.Duration 791 792 // Methods is the list of HTTP methods on which to include CORS response 793 // headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list 794 // of methods, and means "any method". 795 Methods []string 796 797 // Origins is the list of Origins eligible to receive CORS response 798 // headers. Note: "*" is permitted in the list of origins, and means 799 // "any Origin". 800 Origins []string 801 802 // ResponseHeaders is the list of HTTP headers other than the simple 803 // response headers to give permission for the user-agent to share 804 // across domains. 805 ResponseHeaders []string 806} 807 808// BucketEncryption is a bucket's encryption configuration. 809type BucketEncryption struct { 810 // A Cloud KMS key name, in the form 811 // projects/P/locations/L/keyRings/R/cryptoKeys/K, that will be used to encrypt 812 // objects inserted into this bucket, if no encryption method is specified. 813 // The key's location must be the same as the bucket's. 814 DefaultKMSKeyName string 815} 816 817// BucketAttrsToUpdate define the attributes to update during an Update call. 818type BucketAttrsToUpdate struct { 819 // If set, updates whether the bucket uses versioning. 820 VersioningEnabled optional.Bool 821 822 // If set, updates whether the bucket is a Requester Pays bucket. 823 RequesterPays optional.Bool 824 825 // DefaultEventBasedHold is the default value for event-based hold on 826 // newly created objects in this bucket. 827 DefaultEventBasedHold optional.Bool 828 829 // BucketPolicyOnly is an alias for UniformBucketLevelAccess. Use of 830 // UniformBucketLevelAccess is recommended above the use of this field. 831 // Setting BucketPolicyOnly.Enabled OR UniformBucketLevelAccess.Enabled to 832 // true, will enable UniformBucketLevelAccess. If both BucketPolicyOnly and 833 // UniformBucketLevelAccess are set, the value of UniformBucketLevelAccess 834 // will take precedence. 835 BucketPolicyOnly *BucketPolicyOnly 836 837 // UniformBucketLevelAccess configures access checks to use only bucket-level IAM 838 // policies and ignore any ACL rules for the bucket. 839 // See https://cloud.google.com/storage/docs/uniform-bucket-level-access 840 // for more information. 841 UniformBucketLevelAccess *UniformBucketLevelAccess 842 843 // PublicAccessPrevention is the setting for the bucket's 844 // PublicAccessPrevention policy, which can be used to prevent public access 845 // of data in the bucket. See 846 // https://cloud.google.com/storage/docs/public-access-prevention for more 847 // information. 848 PublicAccessPrevention PublicAccessPrevention 849 850 // StorageClass is the default storage class of the bucket. This defines 851 // how objects in the bucket are stored and determines the SLA 852 // and the cost of storage. Typical values are "STANDARD", "NEARLINE", 853 // "COLDLINE" and "ARCHIVE". Defaults to "STANDARD". 854 // See https://cloud.google.com/storage/docs/storage-classes for all 855 // valid values. 856 StorageClass string 857 858 // If set, updates the retention policy of the bucket. Using 859 // RetentionPolicy.RetentionPeriod = 0 will delete the existing policy. 860 // 861 // This feature is in private alpha release. It is not currently available to 862 // most customers. It might be changed in backwards-incompatible ways and is not 863 // subject to any SLA or deprecation policy. 864 RetentionPolicy *RetentionPolicy 865 866 // If set, replaces the CORS configuration with a new configuration. 867 // An empty (rather than nil) slice causes all CORS policies to be removed. 868 CORS []CORS 869 870 // If set, replaces the encryption configuration of the bucket. Using 871 // BucketEncryption.DefaultKMSKeyName = "" will delete the existing 872 // configuration. 873 Encryption *BucketEncryption 874 875 // If set, replaces the lifecycle configuration of the bucket. 876 Lifecycle *Lifecycle 877 878 // If set, replaces the logging configuration of the bucket. 879 Logging *BucketLogging 880 881 // If set, replaces the website configuration of the bucket. 882 Website *BucketWebsite 883 884 // If not empty, applies a predefined set of access controls. 885 // See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch. 886 PredefinedACL string 887 888 // If not empty, applies a predefined set of default object access controls. 889 // See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch. 890 PredefinedDefaultObjectACL string 891 892 setLabels map[string]string 893 deleteLabels map[string]bool 894} 895 896// SetLabel causes a label to be added or modified when ua is used 897// in a call to Bucket.Update. 898func (ua *BucketAttrsToUpdate) SetLabel(name, value string) { 899 if ua.setLabels == nil { 900 ua.setLabels = map[string]string{} 901 } 902 ua.setLabels[name] = value 903} 904 905// DeleteLabel causes a label to be deleted when ua is used in a 906// call to Bucket.Update. 907func (ua *BucketAttrsToUpdate) DeleteLabel(name string) { 908 if ua.deleteLabels == nil { 909 ua.deleteLabels = map[string]bool{} 910 } 911 ua.deleteLabels[name] = true 912} 913 914func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { 915 rb := &raw.Bucket{} 916 if ua.CORS != nil { 917 rb.Cors = toRawCORS(ua.CORS) 918 rb.ForceSendFields = append(rb.ForceSendFields, "Cors") 919 } 920 if ua.DefaultEventBasedHold != nil { 921 rb.DefaultEventBasedHold = optional.ToBool(ua.DefaultEventBasedHold) 922 rb.ForceSendFields = append(rb.ForceSendFields, "DefaultEventBasedHold") 923 } 924 if ua.RetentionPolicy != nil { 925 if ua.RetentionPolicy.RetentionPeriod == 0 { 926 rb.NullFields = append(rb.NullFields, "RetentionPolicy") 927 rb.RetentionPolicy = nil 928 } else { 929 rb.RetentionPolicy = ua.RetentionPolicy.toRawRetentionPolicy() 930 } 931 } 932 if ua.VersioningEnabled != nil { 933 rb.Versioning = &raw.BucketVersioning{ 934 Enabled: optional.ToBool(ua.VersioningEnabled), 935 ForceSendFields: []string{"Enabled"}, 936 } 937 } 938 if ua.RequesterPays != nil { 939 rb.Billing = &raw.BucketBilling{ 940 RequesterPays: optional.ToBool(ua.RequesterPays), 941 ForceSendFields: []string{"RequesterPays"}, 942 } 943 } 944 if ua.BucketPolicyOnly != nil { 945 rb.IamConfiguration = &raw.BucketIamConfiguration{ 946 UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{ 947 Enabled: ua.BucketPolicyOnly.Enabled, 948 ForceSendFields: []string{"Enabled"}, 949 }, 950 } 951 } 952 if ua.UniformBucketLevelAccess != nil { 953 rb.IamConfiguration = &raw.BucketIamConfiguration{ 954 UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{ 955 Enabled: ua.UniformBucketLevelAccess.Enabled, 956 ForceSendFields: []string{"Enabled"}, 957 }, 958 } 959 } 960 if ua.PublicAccessPrevention != PublicAccessPreventionUnknown { 961 if rb.IamConfiguration == nil { 962 rb.IamConfiguration = &raw.BucketIamConfiguration{} 963 } 964 rb.IamConfiguration.PublicAccessPrevention = ua.PublicAccessPrevention.String() 965 } 966 if ua.Encryption != nil { 967 if ua.Encryption.DefaultKMSKeyName == "" { 968 rb.NullFields = append(rb.NullFields, "Encryption") 969 rb.Encryption = nil 970 } else { 971 rb.Encryption = ua.Encryption.toRawBucketEncryption() 972 } 973 } 974 if ua.Lifecycle != nil { 975 rb.Lifecycle = toRawLifecycle(*ua.Lifecycle) 976 rb.ForceSendFields = append(rb.ForceSendFields, "Lifecycle") 977 } 978 if ua.Logging != nil { 979 if *ua.Logging == (BucketLogging{}) { 980 rb.NullFields = append(rb.NullFields, "Logging") 981 rb.Logging = nil 982 } else { 983 rb.Logging = ua.Logging.toRawBucketLogging() 984 } 985 } 986 if ua.Website != nil { 987 if *ua.Website == (BucketWebsite{}) { 988 rb.NullFields = append(rb.NullFields, "Website") 989 rb.Website = nil 990 } else { 991 rb.Website = ua.Website.toRawBucketWebsite() 992 } 993 } 994 if ua.PredefinedACL != "" { 995 // Clear ACL or the call will fail. 996 rb.Acl = nil 997 rb.ForceSendFields = append(rb.ForceSendFields, "Acl") 998 } 999 if ua.PredefinedDefaultObjectACL != "" { 1000 // Clear ACLs or the call will fail. 1001 rb.DefaultObjectAcl = nil 1002 rb.ForceSendFields = append(rb.ForceSendFields, "DefaultObjectAcl") 1003 } 1004 rb.StorageClass = ua.StorageClass 1005 if ua.setLabels != nil || ua.deleteLabels != nil { 1006 rb.Labels = map[string]string{} 1007 for k, v := range ua.setLabels { 1008 rb.Labels[k] = v 1009 } 1010 if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 { 1011 rb.ForceSendFields = append(rb.ForceSendFields, "Labels") 1012 } 1013 for l := range ua.deleteLabels { 1014 rb.NullFields = append(rb.NullFields, "Labels."+l) 1015 } 1016 } 1017 return rb 1018} 1019 1020// If returns a new BucketHandle that applies a set of preconditions. 1021// Preconditions already set on the BucketHandle are ignored. 1022// Operations on the new handle will return an error if the preconditions are not 1023// satisfied. The only valid preconditions for buckets are MetagenerationMatch 1024// and MetagenerationNotMatch. 1025func (b *BucketHandle) If(conds BucketConditions) *BucketHandle { 1026 b2 := *b 1027 b2.conds = &conds 1028 return &b2 1029} 1030 1031// BucketConditions constrain bucket methods to act on specific metagenerations. 1032// 1033// The zero value is an empty set of constraints. 1034type BucketConditions struct { 1035 // MetagenerationMatch specifies that the bucket must have the given 1036 // metageneration for the operation to occur. 1037 // If MetagenerationMatch is zero, it has no effect. 1038 MetagenerationMatch int64 1039 1040 // MetagenerationNotMatch specifies that the bucket must not have the given 1041 // metageneration for the operation to occur. 1042 // If MetagenerationNotMatch is zero, it has no effect. 1043 MetagenerationNotMatch int64 1044} 1045 1046func (c *BucketConditions) validate(method string) error { 1047 if *c == (BucketConditions{}) { 1048 return fmt.Errorf("storage: %s: empty conditions", method) 1049 } 1050 if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 { 1051 return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) 1052 } 1053 return nil 1054} 1055 1056// UserProject returns a new BucketHandle that passes the project ID as the user 1057// project for all subsequent calls. Calls with a user project will be billed to that 1058// project rather than to the bucket's owning project. 1059// 1060// A user project is required for all operations on Requester Pays buckets. 1061func (b *BucketHandle) UserProject(projectID string) *BucketHandle { 1062 b2 := *b 1063 b2.userProject = projectID 1064 b2.acl.userProject = projectID 1065 b2.defaultObjectACL.userProject = projectID 1066 return &b2 1067} 1068 1069// LockRetentionPolicy locks a bucket's retention policy until a previously-configured 1070// RetentionPeriod past the EffectiveTime. Note that if RetentionPeriod is set to less 1071// than a day, the retention policy is treated as a development configuration and locking 1072// will have no effect. The BucketHandle must have a metageneration condition that 1073// matches the bucket's metageneration. See BucketHandle.If. 1074// 1075// This feature is in private alpha release. It is not currently available to 1076// most customers. It might be changed in backwards-incompatible ways and is not 1077// subject to any SLA or deprecation policy. 1078func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error { 1079 var metageneration int64 1080 if b.conds != nil { 1081 metageneration = b.conds.MetagenerationMatch 1082 } 1083 req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration) 1084 return runWithRetry(ctx, func() error { 1085 _, err := req.Context(ctx).Do() 1086 return err 1087 }) 1088} 1089 1090// applyBucketConds modifies the provided call using the conditions in conds. 1091// call is something that quacks like a *raw.WhateverCall. 1092func applyBucketConds(method string, conds *BucketConditions, call interface{}) error { 1093 if conds == nil { 1094 return nil 1095 } 1096 if err := conds.validate(method); err != nil { 1097 return err 1098 } 1099 cval := reflect.ValueOf(call) 1100 switch { 1101 case conds.MetagenerationMatch != 0: 1102 if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { 1103 return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) 1104 } 1105 case conds.MetagenerationNotMatch != 0: 1106 if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { 1107 return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) 1108 } 1109 } 1110 return nil 1111} 1112 1113func (rp *RetentionPolicy) toRawRetentionPolicy() *raw.BucketRetentionPolicy { 1114 if rp == nil { 1115 return nil 1116 } 1117 return &raw.BucketRetentionPolicy{ 1118 RetentionPeriod: int64(rp.RetentionPeriod / time.Second), 1119 } 1120} 1121 1122func toRetentionPolicy(rp *raw.BucketRetentionPolicy) (*RetentionPolicy, error) { 1123 if rp == nil { 1124 return nil, nil 1125 } 1126 t, err := time.Parse(time.RFC3339, rp.EffectiveTime) 1127 if err != nil { 1128 return nil, err 1129 } 1130 return &RetentionPolicy{ 1131 RetentionPeriod: time.Duration(rp.RetentionPeriod) * time.Second, 1132 EffectiveTime: t, 1133 IsLocked: rp.IsLocked, 1134 }, nil 1135} 1136 1137func toRawCORS(c []CORS) []*raw.BucketCors { 1138 var out []*raw.BucketCors 1139 for _, v := range c { 1140 out = append(out, &raw.BucketCors{ 1141 MaxAgeSeconds: int64(v.MaxAge / time.Second), 1142 Method: v.Methods, 1143 Origin: v.Origins, 1144 ResponseHeader: v.ResponseHeaders, 1145 }) 1146 } 1147 return out 1148} 1149 1150func toCORS(rc []*raw.BucketCors) []CORS { 1151 var out []CORS 1152 for _, v := range rc { 1153 out = append(out, CORS{ 1154 MaxAge: time.Duration(v.MaxAgeSeconds) * time.Second, 1155 Methods: v.Method, 1156 Origins: v.Origin, 1157 ResponseHeaders: v.ResponseHeader, 1158 }) 1159 } 1160 return out 1161} 1162 1163func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { 1164 var rl raw.BucketLifecycle 1165 if len(l.Rules) == 0 { 1166 rl.ForceSendFields = []string{"Rule"} 1167 } 1168 for _, r := range l.Rules { 1169 rr := &raw.BucketLifecycleRule{ 1170 Action: &raw.BucketLifecycleRuleAction{ 1171 Type: r.Action.Type, 1172 StorageClass: r.Action.StorageClass, 1173 }, 1174 Condition: &raw.BucketLifecycleRuleCondition{ 1175 Age: r.Condition.AgeInDays, 1176 DaysSinceCustomTime: r.Condition.DaysSinceCustomTime, 1177 DaysSinceNoncurrentTime: r.Condition.DaysSinceNoncurrentTime, 1178 MatchesStorageClass: r.Condition.MatchesStorageClasses, 1179 NumNewerVersions: r.Condition.NumNewerVersions, 1180 }, 1181 } 1182 1183 switch r.Condition.Liveness { 1184 case LiveAndArchived: 1185 rr.Condition.IsLive = nil 1186 case Live: 1187 rr.Condition.IsLive = googleapi.Bool(true) 1188 case Archived: 1189 rr.Condition.IsLive = googleapi.Bool(false) 1190 } 1191 1192 if !r.Condition.CreatedBefore.IsZero() { 1193 rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date) 1194 } 1195 if !r.Condition.CustomTimeBefore.IsZero() { 1196 rr.Condition.CustomTimeBefore = r.Condition.CustomTimeBefore.Format(rfc3339Date) 1197 } 1198 if !r.Condition.NoncurrentTimeBefore.IsZero() { 1199 rr.Condition.NoncurrentTimeBefore = r.Condition.NoncurrentTimeBefore.Format(rfc3339Date) 1200 } 1201 rl.Rule = append(rl.Rule, rr) 1202 } 1203 return &rl 1204} 1205 1206func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { 1207 var l Lifecycle 1208 if rl == nil { 1209 return l 1210 } 1211 for _, rr := range rl.Rule { 1212 r := LifecycleRule{ 1213 Action: LifecycleAction{ 1214 Type: rr.Action.Type, 1215 StorageClass: rr.Action.StorageClass, 1216 }, 1217 Condition: LifecycleCondition{ 1218 AgeInDays: rr.Condition.Age, 1219 DaysSinceCustomTime: rr.Condition.DaysSinceCustomTime, 1220 DaysSinceNoncurrentTime: rr.Condition.DaysSinceNoncurrentTime, 1221 MatchesStorageClasses: rr.Condition.MatchesStorageClass, 1222 NumNewerVersions: rr.Condition.NumNewerVersions, 1223 }, 1224 } 1225 1226 if rr.Condition.IsLive == nil { 1227 r.Condition.Liveness = LiveAndArchived 1228 } else if *rr.Condition.IsLive { 1229 r.Condition.Liveness = Live 1230 } else { 1231 r.Condition.Liveness = Archived 1232 } 1233 1234 if rr.Condition.CreatedBefore != "" { 1235 r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore) 1236 } 1237 if rr.Condition.CustomTimeBefore != "" { 1238 r.Condition.CustomTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.CustomTimeBefore) 1239 } 1240 if rr.Condition.NoncurrentTimeBefore != "" { 1241 r.Condition.NoncurrentTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.NoncurrentTimeBefore) 1242 } 1243 l.Rules = append(l.Rules, r) 1244 } 1245 return l 1246} 1247 1248func (e *BucketEncryption) toRawBucketEncryption() *raw.BucketEncryption { 1249 if e == nil { 1250 return nil 1251 } 1252 return &raw.BucketEncryption{ 1253 DefaultKmsKeyName: e.DefaultKMSKeyName, 1254 } 1255} 1256 1257func toBucketEncryption(e *raw.BucketEncryption) *BucketEncryption { 1258 if e == nil { 1259 return nil 1260 } 1261 return &BucketEncryption{DefaultKMSKeyName: e.DefaultKmsKeyName} 1262} 1263 1264func (b *BucketLogging) toRawBucketLogging() *raw.BucketLogging { 1265 if b == nil { 1266 return nil 1267 } 1268 return &raw.BucketLogging{ 1269 LogBucket: b.LogBucket, 1270 LogObjectPrefix: b.LogObjectPrefix, 1271 } 1272} 1273 1274func toBucketLogging(b *raw.BucketLogging) *BucketLogging { 1275 if b == nil { 1276 return nil 1277 } 1278 return &BucketLogging{ 1279 LogBucket: b.LogBucket, 1280 LogObjectPrefix: b.LogObjectPrefix, 1281 } 1282} 1283 1284func (w *BucketWebsite) toRawBucketWebsite() *raw.BucketWebsite { 1285 if w == nil { 1286 return nil 1287 } 1288 return &raw.BucketWebsite{ 1289 MainPageSuffix: w.MainPageSuffix, 1290 NotFoundPage: w.NotFoundPage, 1291 } 1292} 1293 1294func toBucketWebsite(w *raw.BucketWebsite) *BucketWebsite { 1295 if w == nil { 1296 return nil 1297 } 1298 return &BucketWebsite{ 1299 MainPageSuffix: w.MainPageSuffix, 1300 NotFoundPage: w.NotFoundPage, 1301 } 1302} 1303 1304func toBucketPolicyOnly(b *raw.BucketIamConfiguration) BucketPolicyOnly { 1305 if b == nil || b.BucketPolicyOnly == nil || !b.BucketPolicyOnly.Enabled { 1306 return BucketPolicyOnly{} 1307 } 1308 lt, err := time.Parse(time.RFC3339, b.BucketPolicyOnly.LockedTime) 1309 if err != nil { 1310 return BucketPolicyOnly{ 1311 Enabled: true, 1312 } 1313 } 1314 return BucketPolicyOnly{ 1315 Enabled: true, 1316 LockedTime: lt, 1317 } 1318} 1319 1320func toUniformBucketLevelAccess(b *raw.BucketIamConfiguration) UniformBucketLevelAccess { 1321 if b == nil || b.UniformBucketLevelAccess == nil || !b.UniformBucketLevelAccess.Enabled { 1322 return UniformBucketLevelAccess{} 1323 } 1324 lt, err := time.Parse(time.RFC3339, b.UniformBucketLevelAccess.LockedTime) 1325 if err != nil { 1326 return UniformBucketLevelAccess{ 1327 Enabled: true, 1328 } 1329 } 1330 return UniformBucketLevelAccess{ 1331 Enabled: true, 1332 LockedTime: lt, 1333 } 1334} 1335 1336func toPublicAccessPrevention(b *raw.BucketIamConfiguration) PublicAccessPrevention { 1337 if b == nil { 1338 return PublicAccessPreventionUnknown 1339 } 1340 switch b.PublicAccessPrevention { 1341 case publicAccessPreventionInherited, publicAccessPreventionUnspecified: 1342 return PublicAccessPreventionInherited 1343 case publicAccessPreventionEnforced: 1344 return PublicAccessPreventionEnforced 1345 default: 1346 return PublicAccessPreventionUnknown 1347 } 1348} 1349 1350// Objects returns an iterator over the objects in the bucket that match the 1351// Query q. If q is nil, no filtering is done. Objects will be iterated over 1352// lexicographically by name. 1353// 1354// Note: The returned iterator is not safe for concurrent operations without explicit synchronization. 1355func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { 1356 it := &ObjectIterator{ 1357 ctx: ctx, 1358 bucket: b, 1359 } 1360 it.pageInfo, it.nextFunc = iterator.NewPageInfo( 1361 it.fetch, 1362 func() int { return len(it.items) }, 1363 func() interface{} { b := it.items; it.items = nil; return b }) 1364 if q != nil { 1365 it.query = *q 1366 } 1367 return it 1368} 1369 1370// An ObjectIterator is an iterator over ObjectAttrs. 1371// 1372// Note: This iterator is not safe for concurrent operations without explicit synchronization. 1373type ObjectIterator struct { 1374 ctx context.Context 1375 bucket *BucketHandle 1376 query Query 1377 pageInfo *iterator.PageInfo 1378 nextFunc func() error 1379 items []*ObjectAttrs 1380} 1381 1382// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. 1383// 1384// Note: This method is not safe for concurrent operations without explicit synchronization. 1385func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } 1386 1387// Next returns the next result. Its second return value is iterator.Done if 1388// there are no more results. Once Next returns iterator.Done, all subsequent 1389// calls will return iterator.Done. 1390// 1391// In addition, if Next returns an error other than iterator.Done, all 1392// subsequent calls will return the same error. To continue iteration, a new 1393// `ObjectIterator` must be created. Since objects are ordered lexicographically 1394// by name, `Query.StartOffset` can be used to create a new iterator which will 1395// start at the desired place. See 1396// https://pkg.go.dev/cloud.google.com/go/storage?tab=doc#hdr-Listing_objects. 1397// 1398// If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will 1399// have a non-empty Prefix field, and a zero value for all other fields. These 1400// represent prefixes. 1401// 1402// Note: This method is not safe for concurrent operations without explicit synchronization. 1403func (it *ObjectIterator) Next() (*ObjectAttrs, error) { 1404 if err := it.nextFunc(); err != nil { 1405 return nil, err 1406 } 1407 item := it.items[0] 1408 it.items = it.items[1:] 1409 return item, nil 1410} 1411 1412func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) { 1413 req := it.bucket.c.raw.Objects.List(it.bucket.name) 1414 setClientHeader(req.Header()) 1415 projection := it.query.Projection 1416 if projection == ProjectionDefault { 1417 projection = ProjectionFull 1418 } 1419 req.Projection(projection.String()) 1420 req.Delimiter(it.query.Delimiter) 1421 req.Prefix(it.query.Prefix) 1422 req.StartOffset(it.query.StartOffset) 1423 req.EndOffset(it.query.EndOffset) 1424 req.Versions(it.query.Versions) 1425 if len(it.query.fieldSelection) > 0 { 1426 req.Fields("nextPageToken", googleapi.Field(it.query.fieldSelection)) 1427 } 1428 req.PageToken(pageToken) 1429 if it.bucket.userProject != "" { 1430 req.UserProject(it.bucket.userProject) 1431 } 1432 if pageSize > 0 { 1433 req.MaxResults(int64(pageSize)) 1434 } 1435 var resp *raw.Objects 1436 var err error 1437 err = runWithRetry(it.ctx, func() error { 1438 resp, err = req.Context(it.ctx).Do() 1439 return err 1440 }) 1441 if err != nil { 1442 var e *googleapi.Error 1443 if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { 1444 err = ErrBucketNotExist 1445 } 1446 return "", err 1447 } 1448 for _, item := range resp.Items { 1449 it.items = append(it.items, newObject(item)) 1450 } 1451 for _, prefix := range resp.Prefixes { 1452 it.items = append(it.items, &ObjectAttrs{Prefix: prefix}) 1453 } 1454 return resp.NextPageToken, nil 1455} 1456 1457// Buckets returns an iterator over the buckets in the project. You may 1458// optionally set the iterator's Prefix field to restrict the list to buckets 1459// whose names begin with the prefix. By default, all buckets in the project 1460// are returned. 1461// 1462// Note: The returned iterator is not safe for concurrent operations without explicit synchronization. 1463func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator { 1464 it := &BucketIterator{ 1465 ctx: ctx, 1466 client: c, 1467 projectID: projectID, 1468 } 1469 it.pageInfo, it.nextFunc = iterator.NewPageInfo( 1470 it.fetch, 1471 func() int { return len(it.buckets) }, 1472 func() interface{} { b := it.buckets; it.buckets = nil; return b }) 1473 1474 return it 1475} 1476 1477// A BucketIterator is an iterator over BucketAttrs. 1478// 1479// Note: This iterator is not safe for concurrent operations without explicit synchronization. 1480type BucketIterator struct { 1481 // Prefix restricts the iterator to buckets whose names begin with it. 1482 Prefix string 1483 1484 ctx context.Context 1485 client *Client 1486 projectID string 1487 buckets []*BucketAttrs 1488 pageInfo *iterator.PageInfo 1489 nextFunc func() error 1490} 1491 1492// Next returns the next result. Its second return value is iterator.Done if 1493// there are no more results. Once Next returns iterator.Done, all subsequent 1494// calls will return iterator.Done. 1495// 1496// Note: This method is not safe for concurrent operations without explicit synchronization. 1497func (it *BucketIterator) Next() (*BucketAttrs, error) { 1498 if err := it.nextFunc(); err != nil { 1499 return nil, err 1500 } 1501 b := it.buckets[0] 1502 it.buckets = it.buckets[1:] 1503 return b, nil 1504} 1505 1506// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. 1507// 1508// Note: This method is not safe for concurrent operations without explicit synchronization. 1509func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } 1510 1511func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, err error) { 1512 req := it.client.raw.Buckets.List(it.projectID) 1513 setClientHeader(req.Header()) 1514 req.Projection("full") 1515 req.Prefix(it.Prefix) 1516 req.PageToken(pageToken) 1517 if pageSize > 0 { 1518 req.MaxResults(int64(pageSize)) 1519 } 1520 var resp *raw.Buckets 1521 err = runWithRetry(it.ctx, func() error { 1522 resp, err = req.Context(it.ctx).Do() 1523 return err 1524 }) 1525 if err != nil { 1526 return "", err 1527 } 1528 for _, item := range resp.Items { 1529 b, err := newBucket(item) 1530 if err != nil { 1531 return "", err 1532 } 1533 it.buckets = append(it.buckets, b) 1534 } 1535 return resp.NextPageToken, nil 1536} 1537