1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "compress/gzip" 20 "context" 21 "crypto/md5" 22 "crypto/sha256" 23 "encoding/base64" 24 "encoding/json" 25 "flag" 26 "fmt" 27 "hash/crc32" 28 "io" 29 "io/ioutil" 30 "log" 31 "math/rand" 32 "mime/multipart" 33 "net/http" 34 "net/http/httputil" 35 "os" 36 "path/filepath" 37 "runtime" 38 "sort" 39 "strconv" 40 "strings" 41 "testing" 42 "time" 43 44 "cloud.google.com/go/httpreplay" 45 "cloud.google.com/go/iam" 46 "cloud.google.com/go/internal/testutil" 47 "cloud.google.com/go/internal/uid" 48 "github.com/google/go-cmp/cmp" 49 "github.com/google/go-cmp/cmp/cmpopts" 50 "golang.org/x/oauth2/google" 51 "google.golang.org/api/googleapi" 52 "google.golang.org/api/iterator" 53 itesting "google.golang.org/api/iterator/testing" 54 "google.golang.org/api/option" 55 iampb "google.golang.org/genproto/googleapis/iam/v1" 56) 57 58const ( 59 testPrefix = "go-integration-test" 60 replayFilename = "storage.replay" 61 // TODO(jba): move to testutil, factor out from firestore/integration_test.go. 62 envFirestoreProjID = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID" 63 envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY" 64) 65 66var ( 67 record = flag.Bool("record", false, "record RPCs") 68 69 uidSpace *uid.Space 70 bucketName string 71 // Use our own random number generator to isolate the sequence of random numbers from 72 // other packages. This makes it possible to use HTTP replay and draw the same sequence 73 // of numbers as during recording. 74 rng *rand.Rand 75 newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error) 76 replaying bool 77 testTime time.Time 78) 79 80func TestMain(m *testing.M) { 81 cleanup := initIntegrationTest() 82 exit := m.Run() 83 if err := cleanup(); err != nil { 84 // Don't fail the test if cleanup fails. 85 log.Printf("Post-test cleanup failed: %v", err) 86 } 87 os.Exit(exit) 88} 89 90// If integration tests will be run, create a unique bucket for them. 91// Also, set newTestClient to handle record/replay. 92// Return a cleanup function. 93func initIntegrationTest() func() error { 94 flag.Parse() // needed for testing.Short() 95 switch { 96 case testing.Short() && *record: 97 log.Fatal("cannot combine -short and -record") 98 return nil 99 100 case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "": 101 // go test -short with a replay file will replay the integration tests, if 102 // the appropriate environment variables have been set. 103 replaying = true 104 httpreplay.DebugHeaders() 105 replayer, err := httpreplay.NewReplayer(replayFilename) 106 if err != nil { 107 log.Fatal(err) 108 } 109 var t time.Time 110 if err := json.Unmarshal(replayer.Initial(), &t); err != nil { 111 log.Fatal(err) 112 } 113 initUIDsAndRand(t) 114 newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) { 115 hc, err := replayer.Client(ctx) // no creds needed 116 if err != nil { 117 return nil, err 118 } 119 return NewClient(ctx, option.WithHTTPClient(hc)) 120 } 121 log.Printf("replaying from %s", replayFilename) 122 return func() error { return replayer.Close() } 123 124 case testing.Short(): 125 // go test -short without a replay file skips the integration tests. 126 if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" { 127 log.Print("replay not supported for Go versions before 1.8") 128 } 129 newTestClient = nil 130 return func() error { return nil } 131 132 default: // Run integration tests against a real backend. 133 now := time.Now().UTC() 134 initUIDsAndRand(now) 135 var cleanup func() error 136 if *record && httpreplay.Supported() { 137 // Remember the time for replay. 138 nowBytes, err := json.Marshal(now) 139 if err != nil { 140 log.Fatal(err) 141 } 142 recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes) 143 if err != nil { 144 log.Fatalf("could not record: %v", err) 145 } 146 newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 147 hc, err := recorder.Client(ctx, opts...) 148 if err != nil { 149 return nil, err 150 } 151 return NewClient(ctx, option.WithHTTPClient(hc)) 152 } 153 cleanup = func() error { 154 err1 := cleanupBuckets() 155 err2 := recorder.Close() 156 if err1 != nil { 157 return err1 158 } 159 return err2 160 } 161 log.Printf("recording to %s", replayFilename) 162 } else { 163 if *record { 164 log.Print("record not supported for Go versions before 1.8") 165 } 166 newTestClient = NewClient 167 cleanup = cleanupBuckets 168 } 169 ctx := context.Background() 170 client := config(ctx) 171 if client == nil { 172 return func() error { return nil } 173 } 174 defer client.Close() 175 if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil { 176 log.Fatalf("creating bucket %q: %v", bucketName, err) 177 } 178 return cleanup 179 } 180} 181 182func initUIDsAndRand(t time.Time) { 183 uidSpace = uid.NewSpace(testPrefix, &uid.Options{Time: t}) 184 bucketName = uidSpace.New() 185 // Use our own random source, to avoid other parts of the program taking 186 // random numbers from the global source and putting record and replay 187 // out of sync. 188 rng = testutil.NewRand(t) 189 testTime = t 190} 191 192// testConfig returns the Client used to access GCS. testConfig skips 193// the current test if credentials are not available or when being run 194// in Short mode. 195func testConfig(ctx context.Context, t *testing.T) *Client { 196 if testing.Short() && !replaying { 197 t.Skip("Integration tests skipped in short mode") 198 } 199 client := config(ctx) 200 if client == nil { 201 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 202 } 203 return client 204} 205 206// config is like testConfig, but it doesn't need a *testing.T. 207func config(ctx context.Context) *Client { 208 ts := testutil.TokenSource(ctx, ScopeFullControl) 209 if ts == nil { 210 return nil 211 } 212 client, err := newTestClient(ctx, option.WithTokenSource(ts)) 213 if err != nil { 214 log.Fatalf("NewClient: %v", err) 215 } 216 return client 217} 218 219func TestIntegration_BucketMethods(t *testing.T) { 220 ctx := context.Background() 221 client := testConfig(ctx, t) 222 defer client.Close() 223 h := testHelper{t} 224 225 projectID := testutil.ProjID() 226 newBucketName := uidSpace.New() 227 b := client.Bucket(newBucketName) 228 // Test Create and Delete. 229 h.mustCreate(b, projectID, nil) 230 attrs := h.mustBucketAttrs(b) 231 if got, want := attrs.MetaGeneration, int64(1); got != want { 232 t.Errorf("got metagen %d, want %d", got, want) 233 } 234 if got, want := attrs.StorageClass, "STANDARD"; got != want { 235 t.Errorf("got storage class %q, want %q", got, want) 236 } 237 if attrs.VersioningEnabled { 238 t.Error("got versioning enabled, wanted it disabled") 239 } 240 if attrs.LocationType == "" { 241 t.Error("got an empty LocationType") 242 } 243 h.mustDeleteBucket(b) 244 245 // Test Create and Delete with attributes. 246 labels := map[string]string{ 247 "l1": "v1", 248 "empty": "", 249 } 250 attrs = &BucketAttrs{ 251 StorageClass: "NEARLINE", 252 VersioningEnabled: true, 253 Labels: labels, 254 Lifecycle: Lifecycle{ 255 Rules: []LifecycleRule{{ 256 Action: LifecycleAction{ 257 Type: SetStorageClassAction, 258 StorageClass: "NEARLINE", 259 }, 260 Condition: LifecycleCondition{ 261 AgeInDays: 10, 262 Liveness: Archived, 263 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 264 MatchesStorageClasses: []string{"STANDARD"}, 265 NumNewerVersions: 3, 266 }, 267 }, { 268 Action: LifecycleAction{ 269 Type: SetStorageClassAction, 270 StorageClass: "ARCHIVE", 271 }, 272 Condition: LifecycleCondition{ 273 CustomTimeBefore: time.Date(2020, 1, 2, 3, 0, 0, 0, time.UTC), 274 DaysSinceCustomTime: 20, 275 Liveness: Live, 276 MatchesStorageClasses: []string{"STANDARD"}, 277 }, 278 }, { 279 Action: LifecycleAction{ 280 Type: DeleteAction, 281 }, 282 Condition: LifecycleCondition{ 283 DaysSinceNoncurrentTime: 30, 284 Liveness: Live, 285 NoncurrentTimeBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 286 MatchesStorageClasses: []string{"NEARLINE"}, 287 NumNewerVersions: 10, 288 }, 289 }}, 290 }, 291 } 292 h.mustCreate(b, projectID, attrs) 293 attrs = h.mustBucketAttrs(b) 294 if got, want := attrs.MetaGeneration, int64(1); got != want { 295 t.Errorf("got metagen %d, want %d", got, want) 296 } 297 if got, want := attrs.StorageClass, "NEARLINE"; got != want { 298 t.Errorf("got storage class %q, want %q", got, want) 299 } 300 if !attrs.VersioningEnabled { 301 t.Error("got versioning disabled, wanted it enabled") 302 } 303 if got, want := attrs.Labels, labels; !testutil.Equal(got, want) { 304 t.Errorf("labels: got %v, want %v", got, want) 305 } 306 if attrs.LocationType == "" { 307 t.Error("got an empty LocationType") 308 } 309 h.mustDeleteBucket(b) 310} 311 312func TestIntegration_BucketUpdate(t *testing.T) { 313 ctx := context.Background() 314 client := testConfig(ctx, t) 315 defer client.Close() 316 h := testHelper{t} 317 318 b := client.Bucket(uidSpace.New()) 319 h.mustCreate(b, testutil.ProjID(), nil) 320 defer h.mustDeleteBucket(b) 321 322 attrs := h.mustBucketAttrs(b) 323 if attrs.VersioningEnabled { 324 t.Fatal("bucket should not have versioning by default") 325 } 326 if len(attrs.Labels) > 0 { 327 t.Fatal("bucket should not have labels initially") 328 } 329 330 // Using empty BucketAttrsToUpdate should be a no-nop. 331 attrs = h.mustUpdateBucket(b, BucketAttrsToUpdate{}) 332 if attrs.VersioningEnabled { 333 t.Fatal("should not have versioning") 334 } 335 if len(attrs.Labels) > 0 { 336 t.Fatal("should not have labels") 337 } 338 339 // Turn on versioning, add some labels. 340 ua := BucketAttrsToUpdate{VersioningEnabled: true} 341 ua.SetLabel("l1", "v1") 342 ua.SetLabel("empty", "") 343 attrs = h.mustUpdateBucket(b, ua) 344 if !attrs.VersioningEnabled { 345 t.Fatal("should have versioning now") 346 } 347 wantLabels := map[string]string{ 348 "l1": "v1", 349 "empty": "", 350 } 351 if !testutil.Equal(attrs.Labels, wantLabels) { 352 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 353 } 354 355 // Turn off versioning again; add and remove some more labels. 356 ua = BucketAttrsToUpdate{VersioningEnabled: false} 357 ua.SetLabel("l1", "v2") // update 358 ua.SetLabel("new", "new") // create 359 ua.DeleteLabel("empty") // delete 360 ua.DeleteLabel("absent") // delete non-existent 361 attrs = h.mustUpdateBucket(b, ua) 362 if attrs.VersioningEnabled { 363 t.Fatal("should have versioning off") 364 } 365 wantLabels = map[string]string{ 366 "l1": "v2", 367 "new": "new", 368 } 369 if !testutil.Equal(attrs.Labels, wantLabels) { 370 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 371 } 372 373 // Configure a lifecycle 374 wantLifecycle := Lifecycle{ 375 Rules: []LifecycleRule{ 376 { 377 Action: LifecycleAction{Type: "Delete"}, 378 Condition: LifecycleCondition{AgeInDays: 30}, 379 }, 380 }, 381 } 382 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle} 383 attrs = h.mustUpdateBucket(b, ua) 384 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 385 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 386 } 387 // Check that StorageClass has "STANDARD" value for unset field by default 388 // before passing new value. 389 wantStorageClass := "STANDARD" 390 if !testutil.Equal(attrs.StorageClass, wantStorageClass) { 391 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass) 392 } 393 wantStorageClass = "NEARLINE" 394 ua = BucketAttrsToUpdate{StorageClass: wantStorageClass} 395 attrs = h.mustUpdateBucket(b, ua) 396 if !testutil.Equal(attrs.StorageClass, wantStorageClass) { 397 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass) 398 } 399} 400 401func TestIntegration_BucketPolicyOnly(t *testing.T) { 402 ctx := context.Background() 403 client := testConfig(ctx, t) 404 defer client.Close() 405 h := testHelper{t} 406 bkt := client.Bucket(bucketName) 407 408 // Insert an object with custom ACL. 409 o := bkt.Object("bucketPolicyOnly") 410 defer func() { 411 if err := o.Delete(ctx); err != nil { 412 log.Printf("failed to delete test object: %v", err) 413 } 414 }() 415 wc := o.NewWriter(ctx) 416 wc.ContentType = "text/plain" 417 h.mustWrite(wc, []byte("test")) 418 a := o.ACL() 419 aclEntity := ACLEntity("user-test@example.com") 420 err := a.Set(ctx, aclEntity, RoleReader) 421 if err != nil { 422 t.Fatalf("set ACL failed: %v", err) 423 } 424 425 // Enable BucketPolicyOnly. 426 ua := BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: true}} 427 attrs := h.mustUpdateBucket(bkt, ua) 428 if got, want := attrs.BucketPolicyOnly.Enabled, true; got != want { 429 t.Fatalf("got %v, want %v", got, want) 430 } 431 if got := attrs.BucketPolicyOnly.LockedTime; got.IsZero() { 432 t.Fatal("got a zero time value, want a populated value") 433 } 434 435 // Confirm BucketAccessControl returns error. 436 err = retry(ctx, func() error { 437 _, err = bkt.ACL().List(ctx) 438 return nil 439 }, func() error { 440 if err == nil { 441 return fmt.Errorf("ACL.List: expected bucket ACL list to fail") 442 } 443 return nil 444 }) 445 if err != nil { 446 t.Fatal(err) 447 } 448 449 // Confirm ObjectAccessControl returns error. 450 err = retry(ctx, func() error { 451 _, err = o.ACL().List(ctx) 452 return nil 453 }, func() error { 454 if err == nil { 455 return fmt.Errorf("ACL.List: expected object ACL list to fail") 456 } 457 return nil 458 }) 459 if err != nil { 460 t.Fatal(err) 461 } 462 463 // Disable BucketPolicyOnly. 464 ua = BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: false}} 465 attrs = h.mustUpdateBucket(bkt, ua) 466 if got, want := attrs.BucketPolicyOnly.Enabled, false; got != want { 467 t.Fatalf("got %v, want %v", got, want) 468 } 469 470 // Check that the object ACLs are the same. 471 var acls []ACLRule 472 err = retry(ctx, func() error { 473 acls, err = o.ACL().List(ctx) 474 if err != nil { 475 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 476 } 477 return nil 478 }, func() error { 479 if !containsACL(acls, aclEntity, RoleReader) { 480 return fmt.Errorf("containsACL: expected ACLs %v to include custom ACL entity %v", acls, aclEntity) 481 } 482 return nil 483 }) 484 if err != nil { 485 t.Fatal(err) 486 } 487} 488 489func TestIntegration_UniformBucketLevelAccess(t *testing.T) { 490 ctx := context.Background() 491 client := testConfig(ctx, t) 492 defer client.Close() 493 h := testHelper{t} 494 bkt := client.Bucket(uidSpace.New()) 495 h.mustCreate(bkt, testutil.ProjID(), nil) 496 defer h.mustDeleteBucket(bkt) 497 498 // Insert an object with custom ACL. 499 o := bkt.Object("uniformBucketLevelAccess") 500 defer func() { 501 if err := o.Delete(ctx); err != nil { 502 log.Printf("failed to delete test object: %v", err) 503 } 504 }() 505 wc := o.NewWriter(ctx) 506 wc.ContentType = "text/plain" 507 h.mustWrite(wc, []byte("test")) 508 a := o.ACL() 509 aclEntity := ACLEntity("user-test@example.com") 510 err := a.Set(ctx, aclEntity, RoleReader) 511 if err != nil { 512 t.Fatalf("set ACL failed: %v", err) 513 } 514 515 // Enable UniformBucketLevelAccess. 516 ua := BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}} 517 attrs := h.mustUpdateBucket(bkt, ua) 518 if got, want := attrs.UniformBucketLevelAccess.Enabled, true; got != want { 519 t.Fatalf("got %v, want %v", got, want) 520 } 521 if got := attrs.UniformBucketLevelAccess.LockedTime; got.IsZero() { 522 t.Fatal("got a zero time value, want a populated value") 523 } 524 525 // Confirm BucketAccessControl returns error. 526 err = retry(ctx, func() error { 527 _, err = bkt.ACL().List(ctx) 528 return nil 529 }, func() error { 530 if err == nil { 531 return fmt.Errorf("ACL.List: expected bucket ACL list to fail") 532 } 533 return nil 534 }) 535 if err != nil { 536 t.Fatal(err) 537 } 538 539 // Confirm ObjectAccessControl returns error. 540 err = retry(ctx, func() error { 541 _, err = o.ACL().List(ctx) 542 return nil 543 }, func() error { 544 if err == nil { 545 return fmt.Errorf("ACL.List: expected object ACL list to fail") 546 } 547 return nil 548 }) 549 if err != nil { 550 t.Fatal(err) 551 } 552 553 // Disable UniformBucketLevelAccess. 554 ua = BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: false}} 555 attrs = h.mustUpdateBucket(bkt, ua) 556 if got, want := attrs.UniformBucketLevelAccess.Enabled, false; got != want { 557 t.Fatalf("got %v, want %v", got, want) 558 } 559 560 // Check that the object ACLs are the same. 561 var acls []ACLRule 562 err = retry(ctx, func() error { 563 acls, err = o.ACL().List(ctx) 564 if err != nil { 565 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 566 } 567 return nil 568 }, func() error { 569 if !containsACL(acls, aclEntity, RoleReader) { 570 return fmt.Errorf("containsACL: expected ACLs %v to include custom ACL entity %v", acls, aclEntity) 571 } 572 return nil 573 }) 574 if err != nil { 575 t.Fatal(err) 576 } 577} 578 579func TestIntegration_PublicAccessPrevention(t *testing.T) { 580 ctx := context.Background() 581 client := testConfig(ctx, t) 582 defer client.Close() 583 h := testHelper{t} 584 585 // Create a bucket with PublicAccessPrevention enforced. 586 bkt := client.Bucket(uidSpace.New()) 587 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{PublicAccessPrevention: PublicAccessPreventionEnforced}) 588 defer h.mustDeleteBucket(bkt) 589 590 // Making bucket public should fail. 591 policy, err := bkt.IAM().V3().Policy(ctx) 592 if err != nil { 593 t.Fatalf("fetching bucket IAM policy: %v", err) 594 } 595 policy.Bindings = append(policy.Bindings, &iampb.Binding{ 596 Role: "roles/storage.objectViewer", 597 Members: []string{iam.AllUsers}, 598 }) 599 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err == nil { 600 t.Error("SetPolicy: expected adding AllUsers policy to bucket should fail") 601 } 602 603 // Making object public via ACL should fail. 604 o := bkt.Object("publicAccessPrevention") 605 defer func() { 606 if err := o.Delete(ctx); err != nil { 607 log.Printf("failed to delete test object: %v", err) 608 } 609 }() 610 wc := o.NewWriter(ctx) 611 wc.ContentType = "text/plain" 612 h.mustWrite(wc, []byte("test")) 613 a := o.ACL() 614 if err := a.Set(ctx, AllUsers, RoleReader); err == nil { 615 t.Error("ACL.Set: expected adding AllUsers ACL to object should fail") 616 } 617 618 // Update PAP setting to unspecified should work and not affect UBLA setting. 619 attrs, err := bkt.Update(ctx, BucketAttrsToUpdate{PublicAccessPrevention: PublicAccessPreventionUnspecified}) 620 if err != nil { 621 t.Fatalf("updating PublicAccessPrevention failed: %v", err) 622 } 623 if attrs.PublicAccessPrevention != PublicAccessPreventionUnspecified { 624 t.Errorf("updating PublicAccessPrevention: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionUnspecified) 625 } 626 if attrs.UniformBucketLevelAccess.Enabled || attrs.BucketPolicyOnly.Enabled { 627 t.Error("updating PublicAccessPrevention changed UBLA setting") 628 } 629 630 // Now, making object public or making bucket public should succeed. Run with 631 // retry because ACL settings may take time to propagate. 632 if err := retry(ctx, 633 func() error { 634 a = o.ACL() 635 return a.Set(ctx, AllUsers, RoleReader) 636 }, 637 nil); err != nil { 638 t.Errorf("ACL.Set: making object public failed: %v", err) 639 } 640 policy, err = bkt.IAM().V3().Policy(ctx) 641 if err != nil { 642 t.Fatalf("fetching bucket IAM policy: %v", err) 643 } 644 policy.Bindings = append(policy.Bindings, &iampb.Binding{ 645 Role: "roles/storage.objectViewer", 646 Members: []string{iam.AllUsers}, 647 }) 648 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err != nil { 649 t.Errorf("SetPolicy: making bucket public failed: %v", err) 650 } 651 652 // Updating UBLA should not affect PAP setting. 653 attrs, err = bkt.Update(ctx, BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}}) 654 if err != nil { 655 t.Fatalf("updating UBLA failed: %v", err) 656 } 657 if !attrs.UniformBucketLevelAccess.Enabled { 658 t.Error("updating UBLA: got UBLA not enabled, want enabled") 659 } 660 if attrs.PublicAccessPrevention != PublicAccessPreventionUnspecified { 661 t.Errorf("updating UBLA: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionUnspecified) 662 } 663} 664 665func TestIntegration_ConditionalDelete(t *testing.T) { 666 ctx := context.Background() 667 client := testConfig(ctx, t) 668 defer client.Close() 669 h := testHelper{t} 670 671 o := client.Bucket(bucketName).Object("conddel") 672 673 wc := o.NewWriter(ctx) 674 wc.ContentType = "text/plain" 675 h.mustWrite(wc, []byte("foo")) 676 677 gen := wc.Attrs().Generation 678 metaGen := wc.Attrs().Metageneration 679 680 if err := o.Generation(gen - 1).Delete(ctx); err == nil { 681 t.Fatalf("Unexpected successful delete with Generation") 682 } 683 if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil { 684 t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch") 685 } 686 if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil { 687 t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch") 688 } 689 if err := o.Generation(gen).Delete(ctx); err != nil { 690 t.Fatalf("final delete failed: %v", err) 691 } 692} 693 694func TestIntegration_ObjectsRangeReader(t *testing.T) { 695 ctx := context.Background() 696 client := testConfig(ctx, t) 697 defer client.Close() 698 bkt := client.Bucket(bucketName) 699 700 objName := uidSpace.New() 701 obj := bkt.Object(objName) 702 contents := []byte("Hello, world this is a range request") 703 704 if err := retry(ctx, func() error { 705 w := obj.NewWriter(ctx) 706 if _, err := w.Write(contents); err != nil { 707 return fmt.Errorf("Failed to write contents: %v", err) 708 } 709 if err := w.Close(); err != nil { 710 return fmt.Errorf("Failed to close writer: %v", err) 711 } 712 return nil 713 }, nil); err != nil { 714 t.Fatal(err) 715 } 716 717 last5s := []struct { 718 name string 719 start int64 720 length int64 721 }{ 722 {name: "negative offset", start: -5, length: -1}, 723 {name: "offset with specified length", start: int64(len(contents)) - 5, length: 5}, 724 {name: "offset and read till end", start: int64(len(contents)) - 5, length: -1}, 725 } 726 727 for _, last5 := range last5s { 728 t.Run(last5.name, func(t *testing.T) { 729 r, err := obj.NewRangeReader(ctx, last5.start, last5.length) 730 if err != nil { 731 t.Fatalf("Failed to make range read: %v", err) 732 } 733 defer r.Close() 734 735 if got, want := r.Attrs.StartOffset, int64(len(contents))-5; got != want { 736 t.Fatalf("StartOffset mismatch, got %d want %d", got, want) 737 } 738 739 nr, _ := io.Copy(ioutil.Discard, r) 740 if got, want := nr, int64(5); got != want { 741 t.Fatalf("Body length mismatch, got %d want %d", got, want) 742 } 743 }) 744 } 745} 746 747func TestIntegration_ConditionalDownload(t *testing.T) { 748 ctx := context.Background() 749 client := testConfig(ctx, t) 750 defer client.Close() 751 h := testHelper{t} 752 753 o := client.Bucket(bucketName).Object("condread") 754 defer o.Delete(ctx) 755 756 wc := o.NewWriter(ctx) 757 wc.ContentType = "text/plain" 758 h.mustWrite(wc, []byte("foo")) 759 760 gen := wc.Attrs().Generation 761 metaGen := wc.Attrs().Metageneration 762 763 if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil { 764 t.Fatalf("Unexpected successful download with nonexistent Generation") 765 } 766 if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil { 767 t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch") 768 } 769 if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil { 770 t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch") 771 } 772 if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil { 773 t.Fatalf("Download failed: %v", err) 774 } 775} 776 777func TestIntegration_Objects(t *testing.T) { 778 // TODO(jba): Use subtests (Go 1.7). 779 ctx := context.Background() 780 client := testConfig(ctx, t) 781 defer client.Close() 782 // Reset testTime, 'cause object last modification time should be within 5 min 783 // from test (test iteration if -count passed) start time. 784 testTime = time.Now().UTC() 785 newBucketName := uidSpace.New() 786 h := testHelper{t} 787 bkt := client.Bucket(newBucketName) 788 789 h.mustCreate(bkt, testutil.ProjID(), nil) 790 defer func() { 791 if err := killBucket(ctx, client, newBucketName); err != nil { 792 log.Printf("deleting %q: %v", newBucketName, err) 793 } 794 }() 795 const defaultType = "text/plain" 796 797 // Populate object names and make a map for their contents. 798 objects := []string{ 799 "obj1", 800 "obj2", 801 "obj/with/slashes", 802 } 803 contents := make(map[string][]byte) 804 805 // Test Writer. 806 for _, obj := range objects { 807 c := randomContents() 808 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 809 t.Errorf("Write for %v failed with %v", obj, err) 810 } 811 contents[obj] = c 812 } 813 814 testObjectIterator(t, bkt, objects) 815 testObjectsIterateSelectedAttrs(t, bkt, objects) 816 testObjectsIterateAllSelectedAttrs(t, bkt, objects) 817 testObjectIteratorWithOffset(t, bkt, objects) 818 testObjectsIterateWithProjection(t, bkt) 819 t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) { 820 query := &Query{Prefix: "", Delimiter: "/"} 821 if err := query.SetAttrSelection([]string{"Name"}); err != nil { 822 t.Fatalf("selecting query attrs: %v", err) 823 } 824 825 var gotNames []string 826 var gotPrefixes []string 827 it := bkt.Objects(context.Background(), query) 828 for { 829 attrs, err := it.Next() 830 if err == iterator.Done { 831 break 832 } 833 if err != nil { 834 t.Fatalf("iterator.Next: %v", err) 835 } 836 if attrs.Name != "" { 837 gotNames = append(gotNames, attrs.Name) 838 } else if attrs.Prefix != "" { 839 gotPrefixes = append(gotPrefixes, attrs.Prefix) 840 } 841 842 if attrs.Bucket != "" { 843 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 844 } 845 } 846 847 sortedNames := []string{"obj1", "obj2"} 848 if !cmp.Equal(sortedNames, gotNames) { 849 t.Errorf("names = %v, want %v", gotNames, sortedNames) 850 } 851 sortedPrefixes := []string{"obj/"} 852 if !cmp.Equal(sortedPrefixes, gotPrefixes) { 853 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes) 854 } 855 }) 856 857 // Test Reader. 858 for _, obj := range objects { 859 rc, err := bkt.Object(obj).NewReader(ctx) 860 if err != nil { 861 t.Errorf("Can't create a reader for %v, errored with %v", obj, err) 862 continue 863 } 864 if !rc.checkCRC { 865 t.Errorf("%v: not checking CRC", obj) 866 } 867 slurp, err := ioutil.ReadAll(rc) 868 if err != nil { 869 t.Errorf("Can't ReadAll object %v, errored with %v", obj, err) 870 } 871 if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { 872 t.Errorf("Contents (%q) = %q; want %q", obj, got, want) 873 } 874 if got, want := rc.Size(), len(contents[obj]); got != int64(want) { 875 t.Errorf("Size (%q) = %d; want %d", obj, got, want) 876 } 877 if got, want := rc.ContentType(), "text/plain"; got != want { 878 t.Errorf("ContentType (%q) = %q; want %q", obj, got, want) 879 } 880 if got, want := rc.CacheControl(), "public, max-age=60"; got != want { 881 t.Errorf("CacheControl (%q) = %q; want %q", obj, got, want) 882 } 883 // We just wrote these objects, so they should have a recent last-modified time. 884 lm, err := rc.LastModified() 885 // Accept a time within +/- of the test time, to account for natural 886 // variation and the fact that testTime is set at the start of the test run. 887 expectedVariance := 5 * time.Minute 888 if err != nil { 889 t.Errorf("LastModified (%q): got error %v", obj, err) 890 } else if lm.Before(testTime.Add(-expectedVariance)) || lm.After(testTime.Add(expectedVariance)) { 891 t.Errorf("LastModified (%q): got %s, which not the %v from now (%v)", obj, lm, expectedVariance, testTime) 892 } 893 rc.Close() 894 895 // Check early close. 896 buf := make([]byte, 1) 897 rc, err = bkt.Object(obj).NewReader(ctx) 898 if err != nil { 899 t.Fatalf("%v: %v", obj, err) 900 } 901 _, err = rc.Read(buf) 902 if err != nil { 903 t.Fatalf("%v: %v", obj, err) 904 } 905 if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) { 906 t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want) 907 } 908 if err := rc.Close(); err != nil { 909 t.Errorf("%v Close: %v", obj, err) 910 } 911 } 912 913 obj := objects[0] 914 objlen := int64(len(contents[obj])) 915 // Test Range Reader. 916 for i, r := range []struct { 917 offset, length, want int64 918 }{ 919 {0, objlen, objlen}, 920 {0, objlen / 2, objlen / 2}, 921 {objlen / 2, objlen, objlen / 2}, 922 {0, 0, 0}, 923 {objlen / 2, 0, 0}, 924 {objlen / 2, -1, objlen / 2}, 925 {0, objlen * 2, objlen}, 926 {-2, -1, 2}, 927 {-objlen, -1, objlen}, 928 {-(objlen / 2), -1, objlen / 2}, 929 } { 930 rc, err := bkt.Object(obj).NewRangeReader(ctx, r.offset, r.length) 931 if err != nil { 932 t.Errorf("%+v: Can't create a range reader for %v, errored with %v", i, obj, err) 933 continue 934 } 935 if rc.Size() != objlen { 936 t.Errorf("%+v: Reader has a content-size of %d, want %d", i, rc.Size(), objlen) 937 } 938 if rc.Remain() != r.want { 939 t.Errorf("%+v: Reader's available bytes reported as %d, want %d", i, rc.Remain(), r.want) 940 } 941 slurp, err := ioutil.ReadAll(rc) 942 if err != nil { 943 t.Errorf("%+v: can't ReadAll object %v, errored with %v", r, obj, err) 944 continue 945 } 946 if len(slurp) != int(r.want) { 947 t.Errorf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", i, r.offset, r.length, len(slurp), r.want) 948 continue 949 } 950 951 switch { 952 case r.offset < 0: // The case of reading the last N bytes. 953 start := objlen + r.offset 954 if got, want := slurp, contents[obj][start:]; !bytes.Equal(got, want) { 955 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 956 } 957 958 default: 959 if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) { 960 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 961 } 962 } 963 rc.Close() 964 } 965 966 objName := objects[0] 967 968 // Test NewReader googleapi.Error. 969 // Since a 429 or 5xx is hard to cause, we trigger a 416. 970 realLen := len(contents[objName]) 971 _, err := bkt.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10) 972 if err, ok := err.(*googleapi.Error); !ok { 973 t.Error("NewRangeReader did not return a googleapi.Error") 974 } else { 975 if err.Code != 416 { 976 t.Errorf("Code = %d; want %d", err.Code, 416) 977 } 978 if len(err.Header) == 0 { 979 t.Error("Missing googleapi.Error.Header") 980 } 981 if len(err.Body) == 0 { 982 t.Error("Missing googleapi.Error.Body") 983 } 984 } 985 986 // Test StatObject. 987 o := h.mustObjectAttrs(bkt.Object(objName)) 988 if got, want := o.Name, objName; got != want { 989 t.Errorf("Name (%v) = %q; want %q", objName, got, want) 990 } 991 if got, want := o.ContentType, defaultType; got != want { 992 t.Errorf("ContentType (%v) = %q; want %q", objName, got, want) 993 } 994 created := o.Created 995 // Check that the object is newer than its containing bucket. 996 bAttrs := h.mustBucketAttrs(bkt) 997 if o.Created.Before(bAttrs.Created) { 998 t.Errorf("Object %v is older than its containing bucket, %v", o, bAttrs) 999 } 1000 1001 // Test object copy. 1002 copyName := "copy-" + objName 1003 copyObj, err := bkt.Object(copyName).CopierFrom(bkt.Object(objName)).Run(ctx) 1004 if err != nil { 1005 t.Errorf("Copier.Run failed with %v", err) 1006 } else if !namesEqual(copyObj, newBucketName, copyName) { 1007 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 1008 copyObj.Bucket, copyObj.Name, newBucketName, copyName) 1009 } 1010 1011 // Copying with attributes. 1012 const contentEncoding = "identity" 1013 copier := bkt.Object(copyName).CopierFrom(bkt.Object(objName)) 1014 copier.ContentEncoding = contentEncoding 1015 copyObj, err = copier.Run(ctx) 1016 if err != nil { 1017 t.Errorf("Copier.Run failed with %v", err) 1018 } else { 1019 if !namesEqual(copyObj, newBucketName, copyName) { 1020 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 1021 copyObj.Bucket, copyObj.Name, newBucketName, copyName) 1022 } 1023 if copyObj.ContentEncoding != contentEncoding { 1024 t.Errorf("Copy ContentEncoding: got %q, want %q", copyObj.ContentEncoding, contentEncoding) 1025 } 1026 } 1027 1028 // Test UpdateAttrs. 1029 metadata := map[string]string{"key": "value"} 1030 updated := h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 1031 ContentType: "text/html", 1032 ContentLanguage: "en", 1033 Metadata: metadata, 1034 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, 1035 }) 1036 if got, want := updated.ContentType, "text/html"; got != want { 1037 t.Errorf("updated.ContentType == %q; want %q", got, want) 1038 } 1039 if got, want := updated.ContentLanguage, "en"; got != want { 1040 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 1041 } 1042 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) { 1043 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want) 1044 } 1045 if got, want := updated.Created, created; got != want { 1046 t.Errorf("updated.Created == %q; want %q", got, want) 1047 } 1048 if !updated.Created.Before(updated.Updated) { 1049 t.Errorf("updated.Updated should be newer than update.Created") 1050 } 1051 1052 // Delete ContentType and ContentLanguage. 1053 updated = h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 1054 ContentType: "", 1055 ContentLanguage: "", 1056 Metadata: map[string]string{}, 1057 }) 1058 if got, want := updated.ContentType, ""; got != want { 1059 t.Errorf("updated.ContentType == %q; want %q", got, want) 1060 } 1061 if got, want := updated.ContentLanguage, ""; got != want { 1062 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 1063 } 1064 if updated.Metadata != nil { 1065 t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata) 1066 } 1067 if got, want := updated.Created, created; got != want { 1068 t.Errorf("updated.Created == %q; want %q", got, want) 1069 } 1070 if !updated.Created.Before(updated.Updated) { 1071 t.Errorf("updated.Updated should be newer than update.Created") 1072 } 1073 1074 // Test checksums. 1075 checksumCases := []struct { 1076 name string 1077 contents [][]byte 1078 size int64 1079 md5 string 1080 crc32c uint32 1081 }{ 1082 { 1083 name: "checksum-object", 1084 contents: [][]byte{[]byte("hello"), []byte("world")}, 1085 size: 10, 1086 md5: "fc5e038d38a57032085441e7fe7010b0", 1087 crc32c: 1456190592, 1088 }, 1089 { 1090 name: "zero-object", 1091 contents: [][]byte{}, 1092 size: 0, 1093 md5: "d41d8cd98f00b204e9800998ecf8427e", 1094 crc32c: 0, 1095 }, 1096 } 1097 for _, c := range checksumCases { 1098 wc := bkt.Object(c.name).NewWriter(ctx) 1099 for _, data := range c.contents { 1100 if _, err := wc.Write(data); err != nil { 1101 t.Errorf("Write(%q) failed with %q", data, err) 1102 } 1103 } 1104 if err = wc.Close(); err != nil { 1105 t.Errorf("%q: close failed with %q", c.name, err) 1106 } 1107 obj := wc.Attrs() 1108 if got, want := obj.Size, c.size; got != want { 1109 t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want) 1110 } 1111 if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want { 1112 t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want) 1113 } 1114 if got, want := obj.CRC32C, c.crc32c; got != want { 1115 t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want) 1116 } 1117 } 1118 1119 // Test public ACL. 1120 publicObj := objects[0] 1121 if err = bkt.Object(publicObj).ACL().Set(ctx, AllUsers, RoleReader); err != nil { 1122 t.Errorf("PutACLEntry failed with %v", err) 1123 } 1124 publicClient, err := newTestClient(ctx, option.WithoutAuthentication()) 1125 if err != nil { 1126 t.Fatal(err) 1127 } 1128 1129 slurp := h.mustRead(publicClient.Bucket(newBucketName).Object(publicObj)) 1130 if !bytes.Equal(slurp, contents[publicObj]) { 1131 t.Errorf("Public object's content: got %q, want %q", slurp, contents[publicObj]) 1132 } 1133 1134 // Test writer error handling. 1135 wc := publicClient.Bucket(newBucketName).Object(publicObj).NewWriter(ctx) 1136 if _, err := wc.Write([]byte("hello")); err != nil { 1137 t.Errorf("Write unexpectedly failed with %v", err) 1138 } 1139 if err = wc.Close(); err == nil { 1140 t.Error("Close expected an error, found none") 1141 } 1142 1143 // Test deleting the copy object. 1144 h.mustDeleteObject(bkt.Object(copyName)) 1145 // Deleting it a second time should return ErrObjectNotExist. 1146 if err := bkt.Object(copyName).Delete(ctx); err != ErrObjectNotExist { 1147 t.Errorf("second deletion of %v = %v; want ErrObjectNotExist", copyName, err) 1148 } 1149 _, err = bkt.Object(copyName).Attrs(ctx) 1150 if err != ErrObjectNotExist { 1151 t.Errorf("Copy is expected to be deleted, stat errored with %v", err) 1152 } 1153 1154 // Test object composition. 1155 var compSrcs []*ObjectHandle 1156 var wantContents []byte 1157 for _, obj := range objects { 1158 compSrcs = append(compSrcs, bkt.Object(obj)) 1159 wantContents = append(wantContents, contents[obj]...) 1160 } 1161 checkCompose := func(obj *ObjectHandle, wantContentType string) { 1162 rc := h.mustNewReader(obj) 1163 slurp, err = ioutil.ReadAll(rc) 1164 if err != nil { 1165 t.Fatalf("ioutil.ReadAll: %v", err) 1166 } 1167 defer rc.Close() 1168 if !bytes.Equal(slurp, wantContents) { 1169 t.Errorf("Composed object contents\ngot: %q\nwant: %q", slurp, wantContents) 1170 } 1171 if got := rc.ContentType(); got != wantContentType { 1172 t.Errorf("Composed object content-type = %q, want %q", got, wantContentType) 1173 } 1174 } 1175 1176 // Compose should work even if the user sets no destination attributes. 1177 compDst := bkt.Object("composed1") 1178 c := compDst.ComposerFrom(compSrcs...) 1179 if _, err := c.Run(ctx); err != nil { 1180 t.Fatalf("ComposeFrom error: %v", err) 1181 } 1182 checkCompose(compDst, "application/octet-stream") 1183 1184 // It should also work if we do. 1185 compDst = bkt.Object("composed2") 1186 c = compDst.ComposerFrom(compSrcs...) 1187 c.ContentType = "text/json" 1188 if _, err := c.Run(ctx); err != nil { 1189 t.Fatalf("ComposeFrom error: %v", err) 1190 } 1191 checkCompose(compDst, "text/json") 1192} 1193 1194func TestIntegration_Encoding(t *testing.T) { 1195 ctx := context.Background() 1196 client := testConfig(ctx, t) 1197 defer client.Close() 1198 bkt := client.Bucket(bucketName) 1199 1200 // Test content encoding 1201 const zeroCount = 20 << 1 // TODO: should be 20 << 20 1202 obj := bkt.Object("gzip-test") 1203 w := obj.NewWriter(ctx) 1204 w.ContentEncoding = "gzip" 1205 gw := gzip.NewWriter(w) 1206 if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil { 1207 t.Fatalf("io.Copy, upload: %v", err) 1208 } 1209 if err := gw.Close(); err != nil { 1210 t.Errorf("gzip.Close(): %v", err) 1211 } 1212 if err := w.Close(); err != nil { 1213 t.Errorf("w.Close(): %v", err) 1214 } 1215 r, err := obj.NewReader(ctx) 1216 if err != nil { 1217 t.Fatalf("NewReader(gzip-test): %v", err) 1218 } 1219 n, err := io.Copy(ioutil.Discard, r) 1220 if err != nil { 1221 t.Errorf("io.Copy, download: %v", err) 1222 } 1223 if n != zeroCount { 1224 t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount) 1225 } 1226 1227 // Test NotFound. 1228 _, err = bkt.Object("obj-not-exists").NewReader(ctx) 1229 if err != ErrObjectNotExist { 1230 t.Errorf("Object should not exist, err found to be %v", err) 1231 } 1232} 1233 1234func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) { 1235 ctx := context.Background() 1236 h := testHelper{t} 1237 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1238 names := make([]string, len(objects)) 1239 copy(names, objects) 1240 sort.Strings(names) 1241 var attrs []*ObjectAttrs 1242 for _, name := range names { 1243 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1244 } 1245 msg, ok := itesting.TestIterator(attrs, 1246 func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) }, 1247 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1248 if !ok { 1249 t.Errorf("ObjectIterator.Next: %s", msg) 1250 } 1251 // TODO(jba): test query.Delimiter != "" 1252} 1253 1254func testObjectIteratorWithOffset(t *testing.T, bkt *BucketHandle, objects []string) { 1255 ctx := context.Background() 1256 h := testHelper{t} 1257 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1258 names := make([]string, len(objects)) 1259 copy(names, objects) 1260 sort.Strings(names) 1261 var attrs []*ObjectAttrs 1262 for _, name := range names { 1263 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1264 } 1265 m := make(map[string][]*ObjectAttrs) 1266 for i, name := range names { 1267 // StartOffset takes the value of object names, the result must be for: 1268 // ― obj/with/slashes: obj/with/slashes, obj1, obj2 1269 // ― obj1: obj1, obj2 1270 // ― obj2: obj2. 1271 m[name] = attrs[i:] 1272 msg, ok := itesting.TestIterator(m[name], 1273 func() interface{} { return bkt.Objects(ctx, &Query{StartOffset: name}) }, 1274 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1275 if !ok { 1276 t.Errorf("ObjectIterator.Next: %s", msg) 1277 } 1278 // EndOffset takes the value of object names, the result must be for: 1279 // ― obj/with/slashes: "" 1280 // ― obj1: obj/with/slashes 1281 // ― obj2: obj/with/slashes, obj1. 1282 m[name] = attrs[:i] 1283 msg, ok = itesting.TestIterator(m[name], 1284 func() interface{} { return bkt.Objects(ctx, &Query{EndOffset: name}) }, 1285 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1286 if !ok { 1287 t.Errorf("ObjectIterator.Next: %s", msg) 1288 } 1289 } 1290} 1291 1292func testObjectsIterateSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1293 // Create a query that will only select the "Name" attr of objects, and 1294 // invoke object listing. 1295 query := &Query{Prefix: ""} 1296 query.SetAttrSelection([]string{"Name"}) 1297 1298 var gotNames []string 1299 it := bkt.Objects(context.Background(), query) 1300 for { 1301 attrs, err := it.Next() 1302 if err == iterator.Done { 1303 break 1304 } 1305 if err != nil { 1306 t.Fatalf("iterator.Next: %v", err) 1307 } 1308 gotNames = append(gotNames, attrs.Name) 1309 1310 if len(attrs.Bucket) > 0 { 1311 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 1312 } 1313 } 1314 1315 sortedNames := make([]string, len(objects)) 1316 copy(sortedNames, objects) 1317 sort.Strings(sortedNames) 1318 sort.Strings(gotNames) 1319 1320 if !cmp.Equal(sortedNames, gotNames) { 1321 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1322 } 1323} 1324 1325func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1326 // Tests that all selected attributes work - query succeeds (without actually 1327 // verifying the returned results). 1328 query := &Query{ 1329 Prefix: "", 1330 StartOffset: "obj/with/slashes", 1331 EndOffset: "obj2", 1332 } 1333 var selectedAttrs []string 1334 for k := range attrToFieldMap { 1335 selectedAttrs = append(selectedAttrs, k) 1336 } 1337 query.SetAttrSelection(selectedAttrs) 1338 1339 count := 0 1340 it := bkt.Objects(context.Background(), query) 1341 for { 1342 _, err := it.Next() 1343 if err == iterator.Done { 1344 break 1345 } 1346 if err != nil { 1347 t.Fatalf("iterator.Next: %v", err) 1348 } 1349 count++ 1350 } 1351 1352 if count != len(objects)-1 { 1353 t.Errorf("count = %v, want %v", count, len(objects)-1) 1354 } 1355} 1356 1357func testObjectsIterateWithProjection(t *testing.T, bkt *BucketHandle) { 1358 projections := map[Projection]bool{ 1359 ProjectionDefault: true, 1360 ProjectionFull: true, 1361 ProjectionNoACL: false, 1362 } 1363 1364 for projection, expectACL := range projections { 1365 query := &Query{Projection: projection} 1366 it := bkt.Objects(context.Background(), query) 1367 attrs, err := it.Next() 1368 if err == iterator.Done { 1369 t.Fatalf("iterator: no objects") 1370 } 1371 if err != nil { 1372 t.Fatalf("iterator.Next: %v", err) 1373 } 1374 1375 if expectACL { 1376 if attrs.Owner == "" { 1377 t.Errorf("projection %q: Owner is empty, want nonempty Owner", projection) 1378 } 1379 if len(attrs.ACL) == 0 { 1380 t.Errorf("projection %q: ACL is empty, want at least one ACL rule", projection) 1381 } 1382 } else { 1383 if attrs.Owner != "" { 1384 t.Errorf("projection %q: got Owner = %q, want empty Owner", projection, attrs.Owner) 1385 } 1386 if len(attrs.ACL) != 0 { 1387 t.Errorf("projection %q: got %d ACL rules, want empty ACL", projection, len(attrs.ACL)) 1388 } 1389 } 1390 } 1391} 1392 1393func TestIntegration_SignedURL(t *testing.T) { 1394 if testing.Short() { // do not test during replay 1395 t.Skip("Integration tests skipped in short mode") 1396 } 1397 // To test SignedURL, we need a real user email and private key. Extract them 1398 // from the JSON key file. 1399 jwtConf, err := testutil.JWTConfig() 1400 if err != nil { 1401 t.Fatal(err) 1402 } 1403 if jwtConf == nil { 1404 t.Skip("JSON key file is not present") 1405 } 1406 1407 ctx := context.Background() 1408 client := testConfig(ctx, t) 1409 defer client.Close() 1410 1411 bkt := client.Bucket(bucketName) 1412 obj := "signedURL" 1413 contents := []byte("This is a test of SignedURL.\n") 1414 md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents 1415 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 1416 t.Fatalf("writing: %v", err) 1417 } 1418 for _, test := range []struct { 1419 desc string 1420 opts SignedURLOptions 1421 headers map[string][]string 1422 fail bool 1423 }{ 1424 { 1425 desc: "basic v2", 1426 }, 1427 { 1428 desc: "basic v4", 1429 opts: SignedURLOptions{Scheme: SigningSchemeV4}, 1430 }, 1431 { 1432 desc: "MD5 sent and matches", 1433 opts: SignedURLOptions{MD5: md5}, 1434 headers: map[string][]string{"Content-MD5": {md5}}, 1435 }, 1436 { 1437 desc: "MD5 not sent", 1438 opts: SignedURLOptions{MD5: md5}, 1439 fail: true, 1440 }, 1441 { 1442 desc: "Content-Type sent and matches", 1443 opts: SignedURLOptions{ContentType: "text/plain"}, 1444 headers: map[string][]string{"Content-Type": {"text/plain"}}, 1445 }, 1446 { 1447 desc: "Content-Type sent but does not match", 1448 opts: SignedURLOptions{ContentType: "text/plain"}, 1449 headers: map[string][]string{"Content-Type": {"application/json"}}, 1450 fail: true, 1451 }, 1452 { 1453 desc: "Canonical headers sent and match", 1454 opts: SignedURLOptions{Headers: []string{ 1455 " X-Goog-Foo: Bar baz ", 1456 "X-Goog-Novalue", // ignored: no value 1457 "X-Google-Foo", // ignored: wrong prefix 1458 }}, 1459 headers: map[string][]string{"X-Goog-foo": {"Bar baz "}}, 1460 }, 1461 { 1462 desc: "Canonical headers sent but don't match", 1463 opts: SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}}, 1464 headers: map[string][]string{"X-Goog-Foo": {"bar baz"}}, 1465 fail: true, 1466 }, 1467 } { 1468 opts := test.opts 1469 opts.GoogleAccessID = jwtConf.Email 1470 opts.PrivateKey = jwtConf.PrivateKey 1471 opts.Method = "GET" 1472 opts.Expires = time.Now().Add(time.Hour) 1473 u, err := SignedURL(bucketName, obj, &opts) 1474 if err != nil { 1475 t.Errorf("%s: SignedURL: %v", test.desc, err) 1476 continue 1477 } 1478 got, err := getURL(u, test.headers) 1479 if err != nil && !test.fail { 1480 t.Errorf("%s: getURL %q: %v", test.desc, u, err) 1481 } else if err == nil && !bytes.Equal(got, contents) { 1482 t.Errorf("%s: got %q, want %q", test.desc, got, contents) 1483 } 1484 } 1485} 1486 1487func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { 1488 if testing.Short() { // do not test during replay 1489 t.Skip("Integration tests skipped in short mode") 1490 } 1491 // To test SignedURL, we need a real user email and private key. Extract 1492 // them from the JSON key file. 1493 jwtConf, err := testutil.JWTConfig() 1494 if err != nil { 1495 t.Fatal(err) 1496 } 1497 if jwtConf == nil { 1498 t.Skip("JSON key file is not present") 1499 } 1500 1501 ctx := context.Background() 1502 client := testConfig(ctx, t) 1503 defer client.Close() 1504 1505 // TODO(deklerk): document how these were generated and their significance 1506 encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=" 1507 encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=" 1508 headers := map[string][]string{ 1509 "x-goog-encryption-algorithm": {"AES256"}, 1510 "x-goog-encryption-key": {encryptionKey}, 1511 "x-goog-encryption-key-sha256": {encryptionKeySha256}, 1512 } 1513 contents := []byte(`{"message":"encryption with csek works"}`) 1514 tests := []struct { 1515 desc string 1516 opts *SignedURLOptions 1517 }{ 1518 { 1519 desc: "v4 URL with customer supplied encryption keys for PUT", 1520 opts: &SignedURLOptions{ 1521 Method: "PUT", 1522 Headers: []string{ 1523 "x-goog-encryption-algorithm:AES256", 1524 "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=", 1525 "x-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=", 1526 }, 1527 Scheme: SigningSchemeV4, 1528 }, 1529 }, 1530 { 1531 desc: "v4 URL with customer supplied encryption keys for GET", 1532 opts: &SignedURLOptions{ 1533 Method: "GET", 1534 Headers: []string{ 1535 "x-goog-encryption-algorithm:AES256", 1536 fmt.Sprintf("x-goog-encryption-key:%s", encryptionKey), 1537 fmt.Sprintf("x-goog-encryption-key-sha256:%s", encryptionKeySha256), 1538 }, 1539 Scheme: SigningSchemeV4, 1540 }, 1541 }, 1542 } 1543 defer func() { 1544 // Delete encrypted object. 1545 bkt := client.Bucket(bucketName) 1546 err := bkt.Object("csek.json").Delete(ctx) 1547 if err != nil { 1548 log.Printf("failed to deleted encrypted file: %v", err) 1549 } 1550 }() 1551 1552 for _, test := range tests { 1553 opts := test.opts 1554 opts.GoogleAccessID = jwtConf.Email 1555 opts.PrivateKey = jwtConf.PrivateKey 1556 opts.Expires = time.Now().Add(time.Hour) 1557 1558 u, err := SignedURL(bucketName, "csek.json", test.opts) 1559 if err != nil { 1560 t.Fatalf("%s: %v", test.desc, err) 1561 } 1562 1563 if test.opts.Method == "PUT" { 1564 if _, err := putURL(u, headers, bytes.NewReader(contents)); err != nil { 1565 t.Fatalf("%s: %v", test.desc, err) 1566 } 1567 } 1568 1569 if test.opts.Method == "GET" { 1570 got, err := getURL(u, headers) 1571 if err != nil { 1572 t.Fatalf("%s: %v", test.desc, err) 1573 } 1574 if !bytes.Equal(got, contents) { 1575 t.Fatalf("%s: got %q, want %q", test.desc, got, contents) 1576 } 1577 } 1578 } 1579} 1580 1581func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) { 1582 if testing.Short() { // do not test during replay 1583 t.Skip("Integration tests skipped in short mode") 1584 } 1585 1586 // To test SignedURL, we need a real user email and private key. Extract them 1587 // from the JSON key file. 1588 jwtConf, err := testutil.JWTConfig() 1589 if err != nil { 1590 t.Fatal(err) 1591 } 1592 if jwtConf == nil { 1593 t.Skip("JSON key file is not present") 1594 } 1595 1596 ctx := context.Background() 1597 client := testConfig(ctx, t) 1598 defer client.Close() 1599 1600 opts := &SignedURLOptions{ 1601 Scheme: SigningSchemeV4, 1602 Method: "GET", 1603 GoogleAccessID: jwtConf.Email, 1604 PrivateKey: jwtConf.PrivateKey, 1605 Expires: time.Now().Add(time.Hour), 1606 } 1607 1608 u, err := SignedURL(bucketName, "", opts) 1609 if err != nil { 1610 t.Fatal(err) 1611 } 1612 1613 // Should be some ListBucketResult response. 1614 _, err = getURL(u, nil) 1615 if err != nil { 1616 t.Fatal(err) 1617 } 1618} 1619 1620func TestIntegration_ACL(t *testing.T) { 1621 ctx := context.Background() 1622 client := testConfig(ctx, t) 1623 defer client.Close() 1624 1625 bkt := client.Bucket(bucketName) 1626 1627 entity := ACLEntity("domain-google.com") 1628 rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"} 1629 if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil { 1630 t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err) 1631 } 1632 1633 acl, err := bkt.DefaultObjectACL().List(ctx) 1634 if err != nil { 1635 t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucketName, err) 1636 } else if !hasRule(acl, rule) { 1637 t.Errorf("default ACL missing %#v", rule) 1638 } 1639 aclObjects := []string{"acl1", "acl2"} 1640 name := aclObjects[0] 1641 o := bkt.Object(name) 1642 err = retry(ctx, func() error { 1643 for _, obj := range aclObjects { 1644 c := randomContents() 1645 if err := writeObject(ctx, bkt.Object(obj), "", c); err != nil { 1646 t.Errorf("Write for %v failed with %v", obj, err) 1647 } 1648 } 1649 acl, err = o.ACL().List(ctx) 1650 if err != nil { 1651 return fmt.Errorf("ACL.List: can't retrieve ACL of %v", name) 1652 } 1653 return nil 1654 }, func() error { 1655 if !hasRule(acl, rule) { 1656 return fmt.Errorf("hasRule: object ACL missing %+v", rule) 1657 } 1658 return nil 1659 }) 1660 if err != nil { 1661 t.Error(err) 1662 } 1663 if err := o.ACL().Delete(ctx, entity); err != nil { 1664 t.Errorf("object ACL: could not delete entity %s", entity) 1665 } 1666 // Delete the default ACL rule. We can't move this code earlier in the 1667 // test, because the test depends on the fact that the object ACL inherits 1668 // it. 1669 if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil { 1670 t.Errorf("default ACL: could not delete entity %s", entity) 1671 } 1672 1673 entity2 := ACLEntity("user-jbd@google.com") 1674 rule2 := ACLRule{Entity: entity2, Role: RoleReader, Email: "jbd@google.com"} 1675 if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil { 1676 t.Errorf("Error while putting bucket ACL rule: %v", err) 1677 } 1678 var bACL []ACLRule 1679 err = retry(ctx, func() error { 1680 bACL, err = bkt.ACL().List(ctx) 1681 if err != nil { 1682 return fmt.Errorf("ACL.List: error while getting the ACL of the bucket: %v", err) 1683 } 1684 return nil 1685 }, func() error { 1686 if !hasRule(bACL, rule2) { 1687 return fmt.Errorf("hasRule: bucket ACL missing %+v", rule2) 1688 } 1689 return nil 1690 }) 1691 if err != nil { 1692 t.Error(err) 1693 } 1694 if err := bkt.ACL().Delete(ctx, entity2); err != nil { 1695 t.Errorf("Error while deleting bucket ACL rule: %v", err) 1696 } 1697 1698} 1699 1700func TestIntegration_ValidObjectNames(t *testing.T) { 1701 ctx := context.Background() 1702 client := testConfig(ctx, t) 1703 defer client.Close() 1704 1705 bkt := client.Bucket(bucketName) 1706 1707 validNames := []string{ 1708 "gopher", 1709 "Гоферови", 1710 "a", 1711 strings.Repeat("a", 1024), 1712 } 1713 for _, name := range validNames { 1714 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 1715 t.Errorf("Object %q write failed: %v. Want success", name, err) 1716 continue 1717 } 1718 defer bkt.Object(name).Delete(ctx) 1719 } 1720 1721 invalidNames := []string{ 1722 "", // Too short. 1723 strings.Repeat("a", 1025), // Too long. 1724 "new\nlines", 1725 "bad\xffunicode", 1726 } 1727 for _, name := range invalidNames { 1728 // Invalid object names will either cause failure during Write or Close. 1729 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 1730 continue 1731 } 1732 defer bkt.Object(name).Delete(ctx) 1733 t.Errorf("%q should have failed. Didn't", name) 1734 } 1735} 1736 1737func TestIntegration_WriterContentType(t *testing.T) { 1738 ctx := context.Background() 1739 client := testConfig(ctx, t) 1740 defer client.Close() 1741 1742 obj := client.Bucket(bucketName).Object("content") 1743 testCases := []struct { 1744 content string 1745 setType, wantType string 1746 }{ 1747 { 1748 content: "It was the best of times, it was the worst of times.", 1749 wantType: "text/plain; charset=utf-8", 1750 }, 1751 { 1752 content: "<html><head><title>My first page</title></head></html>", 1753 wantType: "text/html; charset=utf-8", 1754 }, 1755 { 1756 content: "<html><head><title>My first page</title></head></html>", 1757 setType: "text/html", 1758 wantType: "text/html", 1759 }, 1760 { 1761 content: "<html><head><title>My first page</title></head></html>", 1762 setType: "image/jpeg", 1763 wantType: "image/jpeg", 1764 }, 1765 } 1766 for i, tt := range testCases { 1767 if err := writeObject(ctx, obj, tt.setType, []byte(tt.content)); err != nil { 1768 t.Errorf("writing #%d: %v", i, err) 1769 } 1770 attrs, err := obj.Attrs(ctx) 1771 if err != nil { 1772 t.Errorf("obj.Attrs: %v", err) 1773 continue 1774 } 1775 if got := attrs.ContentType; got != tt.wantType { 1776 t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType) 1777 } 1778 } 1779} 1780 1781func TestIntegration_ZeroSizedObject(t *testing.T) { 1782 t.Parallel() 1783 ctx := context.Background() 1784 client := testConfig(ctx, t) 1785 defer client.Close() 1786 h := testHelper{t} 1787 1788 obj := client.Bucket(bucketName).Object("zero") 1789 1790 // Check writing it works as expected. 1791 w := obj.NewWriter(ctx) 1792 if err := w.Close(); err != nil { 1793 t.Fatalf("Writer.Close: %v", err) 1794 } 1795 defer obj.Delete(ctx) 1796 1797 // Check we can read it too. 1798 body := h.mustRead(obj) 1799 if len(body) != 0 { 1800 t.Errorf("Body is %v, want empty []byte{}", body) 1801 } 1802} 1803 1804func TestIntegration_Encryption(t *testing.T) { 1805 // This function tests customer-supplied encryption keys for all operations 1806 // involving objects. Bucket and ACL operations aren't tested because they 1807 // aren't affected by customer encryption. Neither is deletion. 1808 ctx := context.Background() 1809 client := testConfig(ctx, t) 1810 defer client.Close() 1811 h := testHelper{t} 1812 1813 obj := client.Bucket(bucketName).Object("customer-encryption") 1814 key := []byte("my-secret-AES-256-encryption-key") 1815 keyHash := sha256.Sum256(key) 1816 keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:]) 1817 key2 := []byte("My-Secret-AES-256-Encryption-Key") 1818 contents := "top secret." 1819 1820 checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) { 1821 // Performing a metadata operation without the key should succeed. 1822 attrs, err := f(obj) 1823 if err != nil { 1824 t.Fatalf("%s: %v", msg, err) 1825 } 1826 // The key hash should match... 1827 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1828 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1829 } 1830 // ...but CRC and MD5 should not be present. 1831 if attrs.CRC32C != 0 { 1832 t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C) 1833 } 1834 if len(attrs.MD5) > 0 { 1835 t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5) 1836 } 1837 1838 // Performing a metadata operation with the key should succeed. 1839 attrs, err = f(obj.Key(key)) 1840 if err != nil { 1841 t.Fatalf("%s: %v", msg, err) 1842 } 1843 // Check the key and content hashes. 1844 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1845 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1846 } 1847 if attrs.CRC32C == 0 { 1848 t.Errorf("%s: CRC: got 0, want non-zero", msg) 1849 } 1850 if len(attrs.MD5) == 0 { 1851 t.Errorf("%s: MD5: got len == 0, want len > 0", msg) 1852 } 1853 } 1854 1855 checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) { 1856 // Reading the object without the key should fail. 1857 if _, err := readObject(ctx, o); err == nil { 1858 t.Errorf("%s: reading without key: want error, got nil", msg) 1859 } 1860 // Reading the object with the key should succeed. 1861 got := h.mustRead(o.Key(k)) 1862 gotContents := string(got) 1863 // And the contents should match what we wrote. 1864 if gotContents != wantContents { 1865 t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents) 1866 } 1867 } 1868 1869 checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) { 1870 got := h.mustRead(obj) 1871 gotContents := string(got) 1872 if gotContents != wantContents { 1873 t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents) 1874 } 1875 } 1876 1877 // Write to obj using our own encryption key, which is a valid 32-byte 1878 // AES-256 key. 1879 h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents)) 1880 1881 checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) { 1882 return o.Attrs(ctx) 1883 }) 1884 1885 checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) { 1886 return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 1887 }) 1888 1889 checkRead("first object", obj, key, contents) 1890 1891 obj2 := client.Bucket(bucketName).Object("customer-encryption-2") 1892 // Copying an object without the key should fail. 1893 if _, err := obj2.CopierFrom(obj).Run(ctx); err == nil { 1894 t.Fatal("want error, got nil") 1895 } 1896 // Copying an object with the key should succeed. 1897 if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1898 t.Fatal(err) 1899 } 1900 // The destination object is not encrypted; we can read it without a key. 1901 checkReadUnencrypted("copy dest", obj2, contents) 1902 1903 // Providing a key on the destination but not the source should fail, 1904 // since the source is encrypted. 1905 if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil { 1906 t.Fatal("want error, got nil") 1907 } 1908 1909 // But copying with keys for both source and destination should succeed. 1910 if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1911 t.Fatal(err) 1912 } 1913 // And the destination should be encrypted, meaning we can only read it 1914 // with a key. 1915 checkRead("copy destination", obj2, key2, contents) 1916 1917 // Change obj2's key to prepare for compose, where all objects must have 1918 // the same key. Also illustrates key rotation: copy an object to itself 1919 // with a different key. 1920 if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil { 1921 t.Fatal(err) 1922 } 1923 obj3 := client.Bucket(bucketName).Object("customer-encryption-3") 1924 // Composing without keys should fail. 1925 if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil { 1926 t.Fatal("want error, got nil") 1927 } 1928 // Keys on the source objects result in an error. 1929 if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil { 1930 t.Fatal("want error, got nil") 1931 } 1932 // A key on the destination object both decrypts the source objects 1933 // and encrypts the destination. 1934 if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil { 1935 t.Fatalf("got %v, want nil", err) 1936 } 1937 // Check that the destination in encrypted. 1938 checkRead("compose destination", obj3, key, contents+contents) 1939 1940 // You can't compose one or more unencrypted source objects into an 1941 // encrypted destination object. 1942 _, err := obj2.CopierFrom(obj2.Key(key)).Run(ctx) // unencrypt obj2 1943 if err != nil { 1944 t.Fatal(err) 1945 } 1946 if _, err := obj3.Key(key).ComposerFrom(obj2).Run(ctx); err == nil { 1947 t.Fatal("got nil, want error") 1948 } 1949} 1950 1951func TestIntegration_NonexistentBucket(t *testing.T) { 1952 t.Parallel() 1953 ctx := context.Background() 1954 client := testConfig(ctx, t) 1955 defer client.Close() 1956 1957 bkt := client.Bucket(uidSpace.New()) 1958 if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist { 1959 t.Errorf("Attrs: got %v, want ErrBucketNotExist", err) 1960 } 1961 it := bkt.Objects(ctx, nil) 1962 if _, err := it.Next(); err != ErrBucketNotExist { 1963 t.Errorf("Objects: got %v, want ErrBucketNotExist", err) 1964 } 1965} 1966 1967func TestIntegration_PerObjectStorageClass(t *testing.T) { 1968 const ( 1969 defaultStorageClass = "STANDARD" 1970 newStorageClass = "NEARLINE" 1971 ) 1972 ctx := context.Background() 1973 client := testConfig(ctx, t) 1974 defer client.Close() 1975 h := testHelper{t} 1976 1977 bkt := client.Bucket(bucketName) 1978 1979 // The bucket should have the default storage class. 1980 battrs := h.mustBucketAttrs(bkt) 1981 if battrs.StorageClass != defaultStorageClass { 1982 t.Fatalf("bucket storage class: got %q, want %q", 1983 battrs.StorageClass, defaultStorageClass) 1984 } 1985 // Write an object; it should start with the bucket's storage class. 1986 obj := bkt.Object("posc") 1987 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 1988 oattrs, err := obj.Attrs(ctx) 1989 if err != nil { 1990 t.Fatal(err) 1991 } 1992 if oattrs.StorageClass != defaultStorageClass { 1993 t.Fatalf("object storage class: got %q, want %q", 1994 oattrs.StorageClass, defaultStorageClass) 1995 } 1996 // Now use Copy to change the storage class. 1997 copier := obj.CopierFrom(obj) 1998 copier.StorageClass = newStorageClass 1999 oattrs2, err := copier.Run(ctx) 2000 if err != nil { 2001 log.Fatal(err) 2002 } 2003 if oattrs2.StorageClass != newStorageClass { 2004 t.Fatalf("new object storage class: got %q, want %q", 2005 oattrs2.StorageClass, newStorageClass) 2006 } 2007 2008 // We can also write a new object using a non-default storage class. 2009 obj2 := bkt.Object("posc2") 2010 w := obj2.NewWriter(ctx) 2011 w.StorageClass = newStorageClass 2012 h.mustWrite(w, []byte("xxx")) 2013 if w.Attrs().StorageClass != newStorageClass { 2014 t.Fatalf("new object storage class: got %q, want %q", 2015 w.Attrs().StorageClass, newStorageClass) 2016 } 2017} 2018 2019func TestIntegration_BucketInCopyAttrs(t *testing.T) { 2020 // Confirm that if bucket is included in the object attributes of a rewrite 2021 // call, but object name and content-type aren't, then we get an error. See 2022 // the comment in Copier.Run. 2023 ctx := context.Background() 2024 client := testConfig(ctx, t) 2025 defer client.Close() 2026 h := testHelper{t} 2027 2028 bkt := client.Bucket(bucketName) 2029 obj := bkt.Object("bucketInCopyAttrs") 2030 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 2031 copier := obj.CopierFrom(obj) 2032 rawObject := copier.ObjectAttrs.toRawObject(bucketName) 2033 _, err := copier.callRewrite(ctx, rawObject) 2034 if err == nil { 2035 t.Errorf("got nil, want error") 2036 } 2037} 2038 2039func TestIntegration_NoUnicodeNormalization(t *testing.T) { 2040 t.Parallel() 2041 ctx := context.Background() 2042 client := testConfig(ctx, t) 2043 defer client.Close() 2044 bkt := client.Bucket(bucketName) 2045 h := testHelper{t} 2046 2047 for _, tst := range []struct { 2048 nameQuoted, content string 2049 }{ 2050 {`"Caf\u00e9"`, "Normalization Form C"}, 2051 {`"Cafe\u0301"`, "Normalization Form D"}, 2052 } { 2053 name, err := strconv.Unquote(tst.nameQuoted) 2054 w := bkt.Object(name).NewWriter(ctx) 2055 h.mustWrite(w, []byte(tst.content)) 2056 if err != nil { 2057 t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err) 2058 } 2059 if got := string(h.mustRead(bkt.Object(name))); got != tst.content { 2060 t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content) 2061 } 2062 } 2063} 2064 2065func TestIntegration_HashesOnUpload(t *testing.T) { 2066 // Check that the user can provide hashes on upload, and that these are checked. 2067 ctx := context.Background() 2068 client := testConfig(ctx, t) 2069 defer client.Close() 2070 obj := client.Bucket(bucketName).Object("hashesOnUpload-1") 2071 data := []byte("I can't wait to be verified") 2072 2073 write := func(w *Writer) error { 2074 if _, err := w.Write(data); err != nil { 2075 _ = w.Close() 2076 return err 2077 } 2078 return w.Close() 2079 } 2080 2081 crc32c := crc32.Checksum(data, crc32cTable) 2082 // The correct CRC should succeed. 2083 w := obj.NewWriter(ctx) 2084 w.CRC32C = crc32c 2085 w.SendCRC32C = true 2086 if err := write(w); err != nil { 2087 t.Fatal(err) 2088 } 2089 2090 // If we change the CRC, validation should fail. 2091 w = obj.NewWriter(ctx) 2092 w.CRC32C = crc32c + 1 2093 w.SendCRC32C = true 2094 if err := write(w); err == nil { 2095 t.Fatal("write with bad CRC32c: want error, got nil") 2096 } 2097 2098 // If we have the wrong CRC but forget to send it, we succeed. 2099 w = obj.NewWriter(ctx) 2100 w.CRC32C = crc32c + 1 2101 if err := write(w); err != nil { 2102 t.Fatal(err) 2103 } 2104 2105 // MD5 2106 md5 := md5.Sum(data) 2107 // The correct MD5 should succeed. 2108 w = obj.NewWriter(ctx) 2109 w.MD5 = md5[:] 2110 if err := write(w); err != nil { 2111 t.Fatal(err) 2112 } 2113 2114 // If we change the MD5, validation should fail. 2115 w = obj.NewWriter(ctx) 2116 w.MD5 = append([]byte(nil), md5[:]...) 2117 w.MD5[0]++ 2118 if err := write(w); err == nil { 2119 t.Fatal("write with bad MD5: want error, got nil") 2120 } 2121} 2122 2123func TestIntegration_BucketIAM(t *testing.T) { 2124 ctx := context.Background() 2125 client := testConfig(ctx, t) 2126 defer client.Close() 2127 h := testHelper{t} 2128 bkt := client.Bucket(uidSpace.New()) 2129 h.mustCreate(bkt, testutil.ProjID(), nil) 2130 defer h.mustDeleteBucket(bkt) 2131 // This bucket is unique to this test run. So we don't have 2132 // to worry about other runs interfering with our IAM policy 2133 // changes. 2134 2135 member := "projectViewer:" + testutil.ProjID() 2136 role := iam.RoleName("roles/storage.objectViewer") 2137 // Get the bucket's IAM policy. 2138 policy, err := bkt.IAM().Policy(ctx) 2139 if err != nil { 2140 t.Fatalf("Getting policy: %v", err) 2141 } 2142 // The member should not have the role. 2143 if policy.HasRole(member, role) { 2144 t.Errorf("member %q has role %q", member, role) 2145 } 2146 // Change the policy. 2147 policy.Add(member, role) 2148 if err := bkt.IAM().SetPolicy(ctx, policy); err != nil { 2149 t.Fatalf("SetPolicy: %v", err) 2150 } 2151 // Confirm that the binding was added. 2152 policy, err = bkt.IAM().Policy(ctx) 2153 if err != nil { 2154 t.Fatalf("Getting policy: %v", err) 2155 } 2156 if !policy.HasRole(member, role) { 2157 t.Errorf("member %q does not have role %q", member, role) 2158 } 2159 2160 // Check TestPermissions. 2161 // This client should have all these permissions (and more). 2162 perms := []string{"storage.buckets.get", "storage.buckets.delete"} 2163 got, err := bkt.IAM().TestPermissions(ctx, perms) 2164 if err != nil { 2165 t.Fatalf("TestPermissions: %v", err) 2166 } 2167 sort.Strings(perms) 2168 sort.Strings(got) 2169 if !testutil.Equal(got, perms) { 2170 t.Errorf("got %v, want %v", got, perms) 2171 } 2172} 2173 2174func TestIntegration_RequesterPays(t *testing.T) { 2175 // This test needs a second project and user (token source) to test 2176 // all possibilities. Since we need these things for Firestore already, 2177 // we use them here. 2178 // 2179 // There are up to three entities involved in a requester-pays call: 2180 // 2181 // 1. The user making the request. Here, we use 2182 // a. The account used to create the token source used for all our 2183 // integration tests (see testutil.TokenSource). 2184 // b. The account used for the Firestore tests. 2185 // 2. The project that owns the requester-pays bucket. Here, that 2186 // is the test project ID (see testutil.ProjID). 2187 // 3. The project provided as the userProject parameter of the request; 2188 // the project to be billed. This test uses: 2189 // a. The project that owns the requester-pays bucket (same as (2)) 2190 // b. Another project (the Firestore project). 2191 // 2192 // The following must hold for this test to work: 2193 // - (1a) must have resourcemanager.projects.createBillingAssignment permission 2194 // (Owner role) on (2) (the project, not the bucket). 2195 // - (1b) must NOT have that permission on (2). 2196 // - (1b) must have serviceusage.services.use permission (Editor role) on (3b). 2197 // - (1b) must NOT have that permission on (3a). 2198 // - (1a) must NOT have that permission on (3b). 2199 2200 const wantErrorCode = 400 2201 2202 ctx := context.Background() 2203 client := testConfig(ctx, t) 2204 defer client.Close() 2205 h := testHelper{t} 2206 2207 bucketName2 := uidSpace.New() 2208 b1 := client.Bucket(bucketName2) 2209 projID := testutil.ProjID() 2210 // Use Firestore project as a project that does not contain the bucket. 2211 otherProjID := os.Getenv(envFirestoreProjID) 2212 if otherProjID == "" { 2213 t.Fatalf("need a second project (env var %s)", envFirestoreProjID) 2214 } 2215 ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl) 2216 if ts == nil { 2217 t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey) 2218 } 2219 otherClient, err := newTestClient(ctx, option.WithTokenSource(ts)) 2220 if err != nil { 2221 t.Fatal(err) 2222 } 2223 defer otherClient.Close() 2224 b2 := otherClient.Bucket(bucketName2) 2225 user, err := keyFileEmail(os.Getenv("GCLOUD_TESTS_GOLANG_KEY")) 2226 if err != nil { 2227 t.Fatal(err) 2228 } 2229 otherUser, err := keyFileEmail(os.Getenv(envFirestorePrivateKey)) 2230 if err != nil { 2231 t.Fatal(err) 2232 } 2233 2234 // Create a requester-pays bucket. The bucket is contained in the project projID. 2235 h.mustCreate(b1, projID, &BucketAttrs{RequesterPays: true}) 2236 if err := b1.ACL().Set(ctx, ACLEntity("user-"+otherUser), RoleOwner); err != nil { 2237 t.Fatal(err) 2238 } 2239 2240 // Extract the error code from err if it's a googleapi.Error. 2241 errCode := func(err error) int { 2242 if err == nil { 2243 return 0 2244 } 2245 if err, ok := err.(*googleapi.Error); ok { 2246 return err.Code 2247 } 2248 return -1 2249 } 2250 2251 // Call f under various conditions. 2252 // Here b1 and b2 refer to the same bucket, but b1 is bound to client, 2253 // while b2 is bound to otherClient. The clients differ in their credentials, 2254 // i.e. the identity of the user making the RPC: b1's user is an Owner on the 2255 // bucket's containing project, b2's is not. 2256 call := func(msg string, f func(*BucketHandle) error) { 2257 // user: an Owner on the containing project 2258 // userProject: absent 2259 // result: success, by the rule permitting access by owners of the containing bucket. 2260 if err := f(b1); err != nil { 2261 t.Errorf("%s: %v, want nil\n"+ 2262 "confirm that %s is an Owner on %s", 2263 msg, err, user, projID) 2264 } 2265 // user: an Owner on the containing project 2266 // userProject: containing project 2267 // result: success, by the same rule as above; userProject is unnecessary but allowed. 2268 if err := f(b1.UserProject(projID)); err != nil { 2269 t.Errorf("%s: got %v, want nil", msg, err) 2270 } 2271 // user: not an Owner on the containing project 2272 // userProject: absent 2273 // result: failure, by the standard requester-pays rule 2274 err := f(b2) 2275 if got, want := errCode(err), wantErrorCode; got != want { 2276 t.Errorf("%s: got error %v with code %d, want code %d\n"+ 2277 "confirm that %s is NOT an Owner on %s", 2278 msg, err, got, want, otherUser, projID) 2279 } 2280 // user: not an Owner on the containing project 2281 // userProject: not the containing one, but user has Editor role on it 2282 // result: success, by the standard requester-pays rule 2283 if err := f(b2.UserProject(otherProjID)); err != nil { 2284 t.Errorf("%s: got %v, want nil\n"+ 2285 "confirm that %s is an Editor on %s and that that project has billing enabled", 2286 msg, err, otherUser, otherProjID) 2287 } 2288 // user: not an Owner on the containing project 2289 // userProject: the containing one, on which the user does NOT have Editor permission. 2290 // result: failure 2291 err = f(b2.UserProject("veener-jba")) 2292 if got, want := errCode(err), 403; got != want { 2293 t.Errorf("%s: got error %v, want code %d\n"+ 2294 "confirm that %s is NOT an Editor on %s", 2295 msg, err, want, otherUser, "veener-jba") 2296 } 2297 } 2298 2299 // Getting its attributes requires a user project. 2300 var attrs *BucketAttrs 2301 call("Bucket attrs", func(b *BucketHandle) error { 2302 a, err := b.Attrs(ctx) 2303 if a != nil { 2304 attrs = a 2305 } 2306 return err 2307 }) 2308 if attrs != nil { 2309 if got, want := attrs.RequesterPays, true; got != want { 2310 t.Fatalf("attr.RequesterPays = %t, want %t", got, want) 2311 } 2312 } 2313 // Object operations. 2314 obj1 := "acl-go-test" + uidSpace.New() 2315 call("write object", func(b *BucketHandle) error { 2316 return writeObject(ctx, b.Object(obj1), "text/plain", []byte("hello")) 2317 }) 2318 call("read object", func(b *BucketHandle) error { 2319 _, err := readObject(ctx, b.Object(obj1)) 2320 return err 2321 }) 2322 call("object attrs", func(b *BucketHandle) error { 2323 _, err := b.Object(obj1).Attrs(ctx) 2324 return err 2325 }) 2326 call("update object", func(b *BucketHandle) error { 2327 _, err := b.Object(obj1).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 2328 return err 2329 }) 2330 2331 // ACL operations. 2332 // Create another object for these to avoid object rate limits. 2333 obj2 := "acl-go-test" + uidSpace.New() 2334 call("write object", func(b *BucketHandle) error { 2335 return writeObject(ctx, b.Object(obj2), "text/plain", []byte("hello")) 2336 }) 2337 entity := ACLEntity("domain-google.com") 2338 call("bucket acl set", func(b *BucketHandle) error { 2339 return b.ACL().Set(ctx, entity, RoleReader) 2340 }) 2341 call("bucket acl list", func(b *BucketHandle) error { 2342 _, err := b.ACL().List(ctx) 2343 return err 2344 }) 2345 call("bucket acl delete", func(b *BucketHandle) error { 2346 err := b.ACL().Delete(ctx, entity) 2347 if errCode(err) == 404 { 2348 // Since we call the function multiple times, it will 2349 // fail with NotFound for all but the first. 2350 return nil 2351 } 2352 return err 2353 }) 2354 call("default object acl set", func(b *BucketHandle) error { 2355 return b.DefaultObjectACL().Set(ctx, entity, RoleReader) 2356 }) 2357 call("default object acl list", func(b *BucketHandle) error { 2358 _, err := b.DefaultObjectACL().List(ctx) 2359 return err 2360 }) 2361 call("default object acl delete", func(b *BucketHandle) error { 2362 err := b.DefaultObjectACL().Delete(ctx, entity) 2363 if errCode(err) == 404 { 2364 return nil 2365 } 2366 return err 2367 }) 2368 call("object acl set", func(b *BucketHandle) error { 2369 return b.Object(obj2).ACL().Set(ctx, entity, RoleReader) 2370 }) 2371 call("object acl list", func(b *BucketHandle) error { 2372 _, err := b.Object(obj2).ACL().List(ctx) 2373 return err 2374 }) 2375 call("object acl delete", func(b *BucketHandle) error { 2376 err := b.Object(obj2).ACL().Delete(ctx, entity) 2377 if errCode(err) == 404 { 2378 return nil 2379 } 2380 return err 2381 }) 2382 2383 // Copy and compose. 2384 call("copy", func(b *BucketHandle) error { 2385 _, err := b.Object("copy").CopierFrom(b.Object(obj1)).Run(ctx) 2386 return err 2387 }) 2388 call("compose", func(b *BucketHandle) error { 2389 _, err := b.Object("compose").ComposerFrom(b.Object(obj1), b.Object("copy")).Run(ctx) 2390 return err 2391 }) 2392 call("delete object", func(b *BucketHandle) error { 2393 // Make sure the object exists, so we don't get confused by ErrObjectNotExist. 2394 // The storage service may perform validation in any order (perhaps in parallel), 2395 // so if we delete an object that doesn't exist and for which we lack permission, 2396 // we could see either of those two errors. (See Google-internal bug 78341001.) 2397 h.mustWrite(b1.Object(obj1).NewWriter(ctx), []byte("hello")) // note: b1, not b. 2398 return b.Object(obj1).Delete(ctx) 2399 }) 2400 b1.Object(obj1).Delete(ctx) // Clean up created objects. 2401 b1.Object(obj2).Delete(ctx) 2402 for _, obj := range []string{"copy", "compose"} { 2403 if err := b1.UserProject(projID).Object(obj).Delete(ctx); err != nil { 2404 t.Fatalf("could not delete %q: %v", obj, err) 2405 } 2406 } 2407 2408 h.mustDeleteBucket(b1) 2409} 2410 2411func TestIntegration_Notifications(t *testing.T) { 2412 ctx := context.Background() 2413 client := testConfig(ctx, t) 2414 defer client.Close() 2415 bkt := client.Bucket(bucketName) 2416 2417 checkNotifications := func(msg string, want map[string]*Notification) { 2418 got, err := bkt.Notifications(ctx) 2419 if err != nil { 2420 t.Fatal(err) 2421 } 2422 if diff := testutil.Diff(got, want); diff != "" { 2423 t.Errorf("%s: got=-, want=+:\n%s", msg, diff) 2424 } 2425 } 2426 checkNotifications("initial", map[string]*Notification{}) 2427 2428 nArg := &Notification{ 2429 TopicProjectID: testutil.ProjID(), 2430 TopicID: "go-storage-notification-test", 2431 PayloadFormat: NoPayload, 2432 } 2433 n, err := bkt.AddNotification(ctx, nArg) 2434 if err != nil { 2435 t.Fatal(err) 2436 } 2437 nArg.ID = n.ID 2438 if !testutil.Equal(n, nArg) { 2439 t.Errorf("got %+v, want %+v", n, nArg) 2440 } 2441 checkNotifications("after add", map[string]*Notification{n.ID: n}) 2442 2443 if err := bkt.DeleteNotification(ctx, n.ID); err != nil { 2444 t.Fatal(err) 2445 } 2446 checkNotifications("after delete", map[string]*Notification{}) 2447} 2448 2449func TestIntegration_PublicBucket(t *testing.T) { 2450 // Confirm that an unauthenticated client can access a public bucket. 2451 // See https://cloud.google.com/storage/docs/public-datasets/landsat 2452 if testing.Short() && !replaying { 2453 t.Skip("Integration tests skipped in short mode") 2454 } 2455 2456 const landsatBucket = "gcp-public-data-landsat" 2457 const landsatPrefix = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/" 2458 const landsatObject = landsatPrefix + "LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt" 2459 2460 // Create an unauthenticated client. 2461 ctx := context.Background() 2462 client, err := newTestClient(ctx, option.WithoutAuthentication()) 2463 if err != nil { 2464 t.Fatal(err) 2465 } 2466 defer client.Close() 2467 h := testHelper{t} 2468 bkt := client.Bucket(landsatBucket) 2469 obj := bkt.Object(landsatObject) 2470 2471 // Read a public object. 2472 bytes := h.mustRead(obj) 2473 if got, want := len(bytes), 117255; got != want { 2474 t.Errorf("len(bytes) = %d, want %d", got, want) 2475 } 2476 2477 // List objects in a public bucket. 2478 iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix}) 2479 gotCount := 0 2480 for { 2481 _, err := iter.Next() 2482 if err == iterator.Done { 2483 break 2484 } 2485 if err != nil { 2486 t.Fatal(err) 2487 } 2488 gotCount++ 2489 } 2490 if wantCount := 14; gotCount != wantCount { 2491 t.Errorf("object count: got %d, want %d", gotCount, wantCount) 2492 } 2493 2494 errCode := func(err error) int { 2495 err2, ok := err.(*googleapi.Error) 2496 if !ok { 2497 return -1 2498 } 2499 return err2.Code 2500 } 2501 2502 // Reading from or writing to a non-public bucket fails. 2503 c := testConfig(ctx, t) 2504 defer c.Close() 2505 nonPublicObj := client.Bucket(bucketName).Object("noauth") 2506 // Oddly, reading returns 403 but writing returns 401. 2507 _, err = readObject(ctx, nonPublicObj) 2508 if got, want := errCode(err), 403; got != want { 2509 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 2510 } 2511 err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b")) 2512 if got, want := errCode(err), 401; got != want { 2513 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 2514 } 2515} 2516 2517func TestIntegration_ReadCRC(t *testing.T) { 2518 // Test that the checksum is handled correctly when reading files. 2519 // For gzipped files, see https://github.com/GoogleCloudPlatform/google-cloud-dotnet/issues/1641. 2520 if testing.Short() && !replaying { 2521 t.Skip("Integration tests skipped in short mode") 2522 } 2523 2524 const ( 2525 // This is an uncompressed file. 2526 // See https://cloud.google.com/storage/docs/public-datasets/landsat 2527 uncompressedBucket = "gcp-public-data-landsat" 2528 uncompressedObject = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt" 2529 2530 gzippedObject = "gzipped-text.txt" 2531 ) 2532 ctx := context.Background() 2533 client, err := newTestClient(ctx) 2534 if err != nil { 2535 t.Fatal(err) 2536 } 2537 h := testHelper{t} 2538 defer client.Close() 2539 2540 // Create gzipped object. 2541 var buf bytes.Buffer 2542 zw := gzip.NewWriter(&buf) 2543 zw.Name = gzippedObject 2544 if _, err := zw.Write([]byte("gzipped object data")); err != nil { 2545 t.Fatalf("creating gzip: %v", err) 2546 } 2547 if err := zw.Close(); err != nil { 2548 t.Fatalf("closing gzip writer: %v", err) 2549 } 2550 w := client.Bucket(bucketName).Object(gzippedObject).NewWriter(ctx) 2551 w.ContentEncoding = "gzip" 2552 w.ContentType = "text/plain" 2553 h.mustWrite(w, buf.Bytes()) 2554 2555 for _, test := range []struct { 2556 desc string 2557 obj *ObjectHandle 2558 offset, length int64 2559 readCompressed bool // don't decompress a gzipped file 2560 2561 wantErr bool 2562 wantCheck bool // Should Reader try to check the CRC? 2563 }{ 2564 { 2565 desc: "uncompressed, entire file", 2566 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2567 offset: 0, 2568 length: -1, 2569 readCompressed: false, 2570 wantCheck: true, 2571 }, 2572 { 2573 desc: "uncompressed, entire file, don't decompress", 2574 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2575 offset: 0, 2576 length: -1, 2577 readCompressed: true, 2578 wantCheck: true, 2579 }, 2580 { 2581 desc: "uncompressed, suffix", 2582 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2583 offset: 1, 2584 length: -1, 2585 readCompressed: false, 2586 wantCheck: false, 2587 }, 2588 { 2589 desc: "uncompressed, prefix", 2590 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2591 offset: 0, 2592 length: 18, 2593 readCompressed: false, 2594 wantCheck: false, 2595 }, 2596 { 2597 // When a gzipped file is unzipped on read, we can't verify the checksum 2598 // because it was computed against the zipped contents. We can detect 2599 // this case using http.Response.Uncompressed. 2600 desc: "compressed, entire file, unzipped", 2601 obj: client.Bucket(bucketName).Object(gzippedObject), 2602 offset: 0, 2603 length: -1, 2604 readCompressed: false, 2605 wantCheck: false, 2606 }, 2607 { 2608 // When we read a gzipped file uncompressed, it's like reading a regular file: 2609 // the served content and the CRC match. 2610 desc: "compressed, entire file, read compressed", 2611 obj: client.Bucket(bucketName).Object(gzippedObject), 2612 offset: 0, 2613 length: -1, 2614 readCompressed: true, 2615 wantCheck: true, 2616 }, 2617 { 2618 desc: "compressed, partial, server unzips", 2619 obj: client.Bucket(bucketName).Object(gzippedObject), 2620 offset: 1, 2621 length: 8, 2622 readCompressed: false, 2623 wantErr: true, // GCS can't serve part of a gzipped object 2624 wantCheck: false, 2625 }, 2626 { 2627 desc: "compressed, partial, read compressed", 2628 obj: client.Bucket(bucketName).Object(gzippedObject), 2629 offset: 1, 2630 length: 8, 2631 readCompressed: true, 2632 wantCheck: false, 2633 }, 2634 } { 2635 obj := test.obj.ReadCompressed(test.readCompressed) 2636 r, err := obj.NewRangeReader(ctx, test.offset, test.length) 2637 if err != nil { 2638 if test.wantErr { 2639 continue 2640 } 2641 t.Fatalf("%s: %v", test.desc, err) 2642 } 2643 if got, want := r.checkCRC, test.wantCheck; got != want { 2644 t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want) 2645 } 2646 _, err = ioutil.ReadAll(r) 2647 _ = r.Close() 2648 if err != nil { 2649 t.Fatalf("%s: %v", test.desc, err) 2650 } 2651 } 2652} 2653 2654func TestIntegration_CancelWrite(t *testing.T) { 2655 // Verify that canceling the writer's context immediately stops uploading an object. 2656 ctx := context.Background() 2657 client := testConfig(ctx, t) 2658 defer client.Close() 2659 bkt := client.Bucket(bucketName) 2660 2661 cctx, cancel := context.WithCancel(ctx) 2662 defer cancel() 2663 obj := bkt.Object("cancel-write") 2664 w := obj.NewWriter(cctx) 2665 w.ChunkSize = googleapi.MinUploadChunkSize 2666 buf := make([]byte, w.ChunkSize) 2667 // Write the first chunk. This is read in its entirety before sending the request 2668 // (see google.golang.org/api/gensupport.PrepareUpload), so we expect it to return 2669 // without error. 2670 _, err := w.Write(buf) 2671 if err != nil { 2672 t.Fatal(err) 2673 } 2674 // Now cancel the context. 2675 cancel() 2676 // The next Write should return context.Canceled. 2677 _, err = w.Write(buf) 2678 // TODO: Once we drop support for Go versions < 1.13, use errors.Is() to 2679 // check for context cancellation instead. 2680 if err != context.Canceled && !strings.Contains(err.Error(), "context canceled") { 2681 t.Fatalf("got %v, wanted context.Canceled", err) 2682 } 2683 // The Close should too. 2684 err = w.Close() 2685 if err != context.Canceled && !strings.Contains(err.Error(), "context canceled") { 2686 t.Fatalf("got %v, wanted context.Canceled", err) 2687 } 2688} 2689 2690func TestIntegration_UpdateCORS(t *testing.T) { 2691 ctx := context.Background() 2692 client := testConfig(ctx, t) 2693 defer client.Close() 2694 h := testHelper{t} 2695 2696 initialSettings := []CORS{ 2697 { 2698 MaxAge: time.Hour, 2699 Methods: []string{"POST"}, 2700 Origins: []string{"some-origin.com"}, 2701 ResponseHeaders: []string{"foo-bar"}, 2702 }, 2703 } 2704 2705 for _, test := range []struct { 2706 input []CORS 2707 want []CORS 2708 }{ 2709 { 2710 input: []CORS{ 2711 { 2712 MaxAge: time.Hour, 2713 Methods: []string{"GET"}, 2714 Origins: []string{"*"}, 2715 ResponseHeaders: []string{"some-header"}, 2716 }, 2717 }, 2718 want: []CORS{ 2719 { 2720 MaxAge: time.Hour, 2721 Methods: []string{"GET"}, 2722 Origins: []string{"*"}, 2723 ResponseHeaders: []string{"some-header"}, 2724 }, 2725 }, 2726 }, 2727 { 2728 input: []CORS{}, 2729 want: nil, 2730 }, 2731 { 2732 input: nil, 2733 want: []CORS{ 2734 { 2735 MaxAge: time.Hour, 2736 Methods: []string{"POST"}, 2737 Origins: []string{"some-origin.com"}, 2738 ResponseHeaders: []string{"foo-bar"}, 2739 }, 2740 }, 2741 }, 2742 } { 2743 bkt := client.Bucket(uidSpace.New()) 2744 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings}) 2745 defer h.mustDeleteBucket(bkt) 2746 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input}) 2747 attrs := h.mustBucketAttrs(bkt) 2748 if diff := testutil.Diff(attrs.CORS, test.want); diff != "" { 2749 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2750 } 2751 } 2752} 2753 2754func TestIntegration_UpdateDefaultEventBasedHold(t *testing.T) { 2755 ctx := context.Background() 2756 client := testConfig(ctx, t) 2757 defer client.Close() 2758 h := testHelper{t} 2759 2760 bkt := client.Bucket(uidSpace.New()) 2761 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2762 defer h.mustDeleteBucket(bkt) 2763 attrs := h.mustBucketAttrs(bkt) 2764 if attrs.DefaultEventBasedHold != false { 2765 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, false) 2766 } 2767 2768 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{DefaultEventBasedHold: true}) 2769 attrs = h.mustBucketAttrs(bkt) 2770 if attrs.DefaultEventBasedHold != true { 2771 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 2772 } 2773 2774 // Omitting it should leave the value unchanged. 2775 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RequesterPays: true}) 2776 attrs = h.mustBucketAttrs(bkt) 2777 if attrs.DefaultEventBasedHold != true { 2778 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 2779 } 2780} 2781 2782func TestIntegration_UpdateEventBasedHold(t *testing.T) { 2783 ctx := context.Background() 2784 client := testConfig(ctx, t) 2785 defer client.Close() 2786 h := testHelper{t} 2787 2788 bkt := client.Bucket(uidSpace.New()) 2789 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2790 obj := bkt.Object("some-obj") 2791 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2792 2793 defer func() { 2794 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: false}) 2795 h.mustDeleteObject(obj) 2796 h.mustDeleteBucket(bkt) 2797 }() 2798 2799 attrs := h.mustObjectAttrs(obj) 2800 if attrs.EventBasedHold != false { 2801 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, false) 2802 } 2803 2804 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: true}) 2805 attrs = h.mustObjectAttrs(obj) 2806 if attrs.EventBasedHold != true { 2807 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 2808 } 2809 2810 // Omitting it should leave the value unchanged. 2811 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}) 2812 attrs = h.mustObjectAttrs(obj) 2813 if attrs.EventBasedHold != true { 2814 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 2815 } 2816} 2817 2818func TestIntegration_UpdateTemporaryHold(t *testing.T) { 2819 ctx := context.Background() 2820 client := testConfig(ctx, t) 2821 defer client.Close() 2822 h := testHelper{t} 2823 2824 bkt := client.Bucket(uidSpace.New()) 2825 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2826 obj := bkt.Object("some-obj") 2827 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2828 2829 defer func() { 2830 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: false}) 2831 h.mustDeleteObject(obj) 2832 h.mustDeleteBucket(bkt) 2833 }() 2834 2835 attrs := h.mustObjectAttrs(obj) 2836 if attrs.TemporaryHold != false { 2837 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, false) 2838 } 2839 2840 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: true}) 2841 attrs = h.mustObjectAttrs(obj) 2842 if attrs.TemporaryHold != true { 2843 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 2844 } 2845 2846 // Omitting it should leave the value unchanged. 2847 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}) 2848 attrs = h.mustObjectAttrs(obj) 2849 if attrs.TemporaryHold != true { 2850 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 2851 } 2852} 2853 2854func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) { 2855 ctx := context.Background() 2856 client := testConfig(ctx, t) 2857 defer client.Close() 2858 h := testHelper{t} 2859 2860 bkt := client.Bucket(uidSpace.New()) 2861 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 2862 obj := bkt.Object("some-obj") 2863 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2864 2865 defer func() { 2866 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}) 2867 2868 // RetentionPeriod of less than a day is explicitly called out 2869 // as best effort and not guaranteed, so let's log problems deleting 2870 // objects instead of failing. 2871 if err := obj.Delete(context.Background()); err != nil { 2872 t.Logf("%s: object delete: %v", loc(), err) 2873 } 2874 if err := bkt.Delete(context.Background()); err != nil { 2875 t.Logf("%s: bucket delete: %v", loc(), err) 2876 } 2877 }() 2878 2879 attrs := h.mustObjectAttrs(obj) 2880 if attrs.RetentionExpirationTime == (time.Time{}) { 2881 t.Fatalf("got=%v, wanted a non-zero value", attrs.RetentionExpirationTime) 2882 } 2883} 2884 2885func TestIntegration_CustomTime(t *testing.T) { 2886 ctx := context.Background() 2887 client := testConfig(ctx, t) 2888 defer client.Close() 2889 h := testHelper{t} 2890 2891 // Create object with CustomTime. 2892 bkt := client.Bucket(bucketName) 2893 obj := bkt.Object("custom-time-obj") 2894 w := obj.NewWriter(ctx) 2895 ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC) 2896 w.ObjectAttrs.CustomTime = ct 2897 h.mustWrite(w, randomContents()) 2898 2899 // Validate that CustomTime has been set 2900 checkCustomTime := func(want time.Time) error { 2901 attrs, err := obj.Attrs(ctx) 2902 if err != nil { 2903 return fmt.Errorf("failed to get object attrs: %v", err) 2904 } 2905 if got := attrs.CustomTime; got != want { 2906 return fmt.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct) 2907 } 2908 return nil 2909 } 2910 2911 if err := checkCustomTime(ct); err != nil { 2912 t.Fatalf("checking CustomTime: %v", err) 2913 } 2914 2915 // Update CustomTime to the future should succeed. 2916 laterTime := ct.Add(10 * time.Hour) 2917 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: laterTime}); err != nil { 2918 t.Fatalf("updating CustomTime: %v", err) 2919 } 2920 2921 // Update CustomTime to the past should give error. 2922 earlierTime := ct.Add(5 * time.Hour) 2923 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: earlierTime}); err == nil { 2924 t.Fatalf("backdating CustomTime: expected error, got none") 2925 } 2926 2927 // Zero value for CustomTime should be ignored. 2928 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{}); err != nil { 2929 t.Fatalf("empty update: %v", err) 2930 } 2931 if err := checkCustomTime(laterTime); err != nil { 2932 t.Fatalf("after sending zero value: %v", err) 2933 } 2934} 2935 2936func TestIntegration_UpdateRetentionPolicy(t *testing.T) { 2937 ctx := context.Background() 2938 client := testConfig(ctx, t) 2939 defer client.Close() 2940 h := testHelper{t} 2941 2942 initial := &RetentionPolicy{RetentionPeriod: time.Minute} 2943 2944 for _, test := range []struct { 2945 input *RetentionPolicy 2946 want *RetentionPolicy 2947 }{ 2948 { // Update 2949 input: &RetentionPolicy{RetentionPeriod: time.Hour}, 2950 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2951 }, 2952 { // Update even with timestamp (EffectiveTime should be ignored) 2953 input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()}, 2954 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2955 }, 2956 { // Remove 2957 input: &RetentionPolicy{}, 2958 want: nil, 2959 }, 2960 { // Remove even with timestamp (EffectiveTime should be ignored) 2961 input: &RetentionPolicy{EffectiveTime: time.Now()}, 2962 want: nil, 2963 }, 2964 { // Ignore 2965 input: nil, 2966 want: initial, 2967 }, 2968 } { 2969 bkt := client.Bucket(uidSpace.New()) 2970 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial}) 2971 defer h.mustDeleteBucket(bkt) 2972 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input}) 2973 attrs := h.mustBucketAttrs(bkt) 2974 if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 { 2975 // Should be set by the server and parsed by the client 2976 t.Fatal("EffectiveTime should be set, but it was not") 2977 } 2978 if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" { 2979 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2980 } 2981 } 2982} 2983 2984func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) { 2985 ctx := context.Background() 2986 client := testConfig(ctx, t) 2987 defer client.Close() 2988 h := testHelper{t} 2989 2990 bkt := client.Bucket(uidSpace.New()) 2991 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}}) 2992 2993 oh := bkt.Object("some-object") 2994 if err := writeObject(ctx, oh, "text/plain", []byte("hello world")); err != nil { 2995 t.Fatal(err) 2996 } 2997 2998 if err := oh.Delete(ctx); err == nil { 2999 t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil") 3000 } 3001 3002 // Remove the retention period 3003 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{}}) 3004 // Deleting with retry, as bucket metadata changes 3005 // can take some time to propagate. 3006 err := retry(ctx, func() error { 3007 return oh.Delete(ctx) 3008 }, nil) 3009 if err != nil { 3010 h.t.Fatalf("%s: object delete: %v", loc(), err) 3011 } 3012 h.mustDeleteBucket(bkt) 3013} 3014 3015func TestIntegration_LockBucket(t *testing.T) { 3016 ctx := context.Background() 3017 client := testConfig(ctx, t) 3018 defer client.Close() 3019 h := testHelper{t} 3020 3021 bkt := client.Bucket(uidSpace.New()) 3022 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}}) 3023 attrs := h.mustBucketAttrs(bkt) 3024 if attrs.RetentionPolicy.IsLocked { 3025 t.Fatal("Expected bucket to begin unlocked, but it was not") 3026 } 3027 err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx) 3028 if err != nil { 3029 t.Fatal("could not lock", err) 3030 } 3031 3032 attrs = h.mustBucketAttrs(bkt) 3033 if !attrs.RetentionPolicy.IsLocked { 3034 t.Fatal("Expected bucket to be locked, but it was not") 3035 } 3036 3037 _, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 3038 if err == nil { 3039 t.Fatal("Expected error updating locked bucket, got nil") 3040 } 3041} 3042 3043func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) { 3044 ctx := context.Background() 3045 client := testConfig(ctx, t) 3046 defer client.Close() 3047 h := testHelper{t} 3048 3049 bkt := client.Bucket(uidSpace.New()) 3050 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 3051 RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}, 3052 }) 3053 err := bkt.LockRetentionPolicy(ctx) 3054 if err == nil { 3055 t.Fatal("expected error locking bucket without metageneration condition, got nil") 3056 } 3057} 3058 3059func TestIntegration_KMS(t *testing.T) { 3060 ctx := context.Background() 3061 client := testConfig(ctx, t) 3062 defer client.Close() 3063 h := testHelper{t} 3064 3065 keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING") 3066 if keyRingName == "" { 3067 t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details") 3068 } 3069 keyName1 := keyRingName + "/cryptoKeys/key1" 3070 keyName2 := keyRingName + "/cryptoKeys/key2" 3071 contents := []byte("my secret") 3072 3073 write := func(obj *ObjectHandle, setKey bool) { 3074 w := obj.NewWriter(ctx) 3075 if setKey { 3076 w.KMSKeyName = keyName1 3077 } 3078 h.mustWrite(w, contents) 3079 } 3080 3081 checkRead := func(obj *ObjectHandle) { 3082 got := h.mustRead(obj) 3083 if !bytes.Equal(got, contents) { 3084 t.Errorf("got %v, want %v", got, contents) 3085 } 3086 attrs := h.mustObjectAttrs(obj) 3087 if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 { 3088 t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1) 3089 } 3090 } 3091 3092 // Write an object with a key, then read it to verify its contents and the presence of the key name. 3093 bkt := client.Bucket(bucketName) 3094 obj := bkt.Object("kms") 3095 write(obj, true) 3096 checkRead(obj) 3097 h.mustDeleteObject(obj) 3098 3099 // Encrypt an object with a CSEK, then copy it using a CMEK. 3100 src := bkt.Object("csek").Key(testEncryptionKey) 3101 if err := writeObject(ctx, src, "text/plain", contents); err != nil { 3102 t.Fatal(err) 3103 } 3104 dest := bkt.Object("cmek") 3105 c := dest.CopierFrom(src) 3106 c.DestinationKMSKeyName = keyName1 3107 if _, err := c.Run(ctx); err != nil { 3108 t.Fatal(err) 3109 } 3110 checkRead(dest) 3111 src.Delete(ctx) 3112 dest.Delete(ctx) 3113 3114 // Create a bucket with a default key, then write and read an object. 3115 bkt = client.Bucket(uidSpace.New()) 3116 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 3117 Location: "US", 3118 Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1}, 3119 }) 3120 defer h.mustDeleteBucket(bkt) 3121 3122 attrs := h.mustBucketAttrs(bkt) 3123 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want { 3124 t.Fatalf("got %q, want %q", got, want) 3125 } 3126 obj = bkt.Object("kms") 3127 write(obj, false) 3128 checkRead(obj) 3129 h.mustDeleteObject(obj) 3130 3131 // Update the bucket's default key to a different name. 3132 // (This key doesn't have to exist.) 3133 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}) 3134 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 3135 t.Fatalf("got %q, want %q", got, want) 3136 } 3137 attrs = h.mustBucketAttrs(bkt) 3138 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 3139 t.Fatalf("got %q, want %q", got, want) 3140 } 3141 3142 // Remove the default KMS key. 3143 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}) 3144 if attrs.Encryption != nil { 3145 t.Fatalf("got %#v, want nil", attrs.Encryption) 3146 } 3147} 3148 3149func TestIntegration_PredefinedACLs(t *testing.T) { 3150 check := func(msg string, rs []ACLRule, i int, wantEntity ACLEntity, wantRole ACLRole) { 3151 if i >= len(rs) { 3152 t.Errorf("%s: no rule at index %d", msg, i) 3153 return 3154 } 3155 got := rs[i] 3156 if got.Entity != wantEntity || got.Role != wantRole { 3157 t.Errorf("%s[%d]: got %+v, want Entity %s and Role %s", 3158 msg, i, got, wantEntity, wantRole) 3159 } 3160 } 3161 checkPrefix := func(msg string, rs []ACLRule, i int, wantPrefix string, wantRole ACLRole) { 3162 if i >= len(rs) { 3163 t.Errorf("%s: no rule at index %d", msg, i) 3164 return 3165 } 3166 got := rs[i] 3167 if !strings.HasPrefix(string(got.Entity), wantPrefix) || got.Role != wantRole { 3168 t.Errorf("%s[%d]: got %+v, want Entity %s... and Role %s", 3169 msg, i, got, wantPrefix, wantRole) 3170 } 3171 } 3172 3173 ctx := context.Background() 3174 client := testConfig(ctx, t) 3175 defer client.Close() 3176 h := testHelper{t} 3177 3178 bkt := client.Bucket(uidSpace.New()) 3179 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 3180 PredefinedACL: "authenticatedRead", 3181 PredefinedDefaultObjectACL: "publicRead", 3182 }) 3183 defer h.mustDeleteBucket(bkt) 3184 attrs := h.mustBucketAttrs(bkt) 3185 checkPrefix("Bucket.ACL", attrs.ACL, 0, "project-owners", RoleOwner) 3186 check("Bucket.ACL", attrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 3187 check("DefaultObjectACL", attrs.DefaultObjectACL, 0, AllUsers, RoleReader) 3188 3189 // Bucket update 3190 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{ 3191 PredefinedACL: "private", 3192 PredefinedDefaultObjectACL: "authenticatedRead", 3193 }) 3194 checkPrefix("Bucket.ACL update", attrs.ACL, 0, "project-owners", RoleOwner) 3195 check("DefaultObjectACL update", attrs.DefaultObjectACL, 0, AllAuthenticatedUsers, RoleReader) 3196 3197 // Object creation 3198 obj := bkt.Object("private") 3199 w := obj.NewWriter(ctx) 3200 w.PredefinedACL = "authenticatedRead" 3201 h.mustWrite(w, []byte("hello")) 3202 defer h.mustDeleteObject(obj) 3203 checkPrefix("Object.ACL", w.Attrs().ACL, 0, "user", RoleOwner) 3204 check("Object.ACL", w.Attrs().ACL, 1, AllAuthenticatedUsers, RoleReader) 3205 3206 // Object update 3207 oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}) 3208 checkPrefix("Object.ACL update", oattrs.ACL, 0, "user", RoleOwner) 3209 if got := len(oattrs.ACL); got != 1 { 3210 t.Errorf("got %d ACLs, want 1", got) 3211 } 3212 3213 // Copy 3214 dst := bkt.Object("dst") 3215 copier := dst.CopierFrom(obj) 3216 copier.PredefinedACL = "publicRead" 3217 oattrs, err := copier.Run(ctx) 3218 if err != nil { 3219 t.Fatal(err) 3220 } 3221 defer h.mustDeleteObject(dst) 3222 // The copied object still retains the "private" ACL of the source object. 3223 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 3224 check("Copy dest", oattrs.ACL, 1, AllUsers, RoleReader) 3225 3226 // Compose 3227 comp := bkt.Object("comp") 3228 composer := comp.ComposerFrom(obj, dst) 3229 composer.PredefinedACL = "authenticatedRead" 3230 oattrs, err = composer.Run(ctx) 3231 if err != nil { 3232 t.Fatal(err) 3233 } 3234 defer h.mustDeleteObject(comp) 3235 // The composed object still retains the "private" ACL. 3236 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 3237 check("Copy dest", oattrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 3238} 3239 3240func TestIntegration_ServiceAccount(t *testing.T) { 3241 ctx := context.Background() 3242 client := testConfig(ctx, t) 3243 defer client.Close() 3244 3245 s, err := client.ServiceAccount(ctx, testutil.ProjID()) 3246 if err != nil { 3247 t.Fatal(err) 3248 } 3249 want := "@gs-project-accounts.iam.gserviceaccount.com" 3250 if !strings.Contains(s, want) { 3251 t.Fatalf("got %v, want to contain %v", s, want) 3252 } 3253} 3254 3255func TestIntegration_ReaderAttrs(t *testing.T) { 3256 ctx := context.Background() 3257 client := testConfig(ctx, t) 3258 defer client.Close() 3259 bkt := client.Bucket(bucketName) 3260 3261 const defaultType = "text/plain" 3262 obj := "some-object" 3263 c := randomContents() 3264 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 3265 t.Errorf("Write for %v failed with %v", obj, err) 3266 } 3267 oh := bkt.Object(obj) 3268 3269 rc, err := oh.NewReader(ctx) 3270 if err != nil { 3271 t.Fatal(err) 3272 } 3273 3274 attrs, err := oh.Attrs(ctx) 3275 if err != nil { 3276 t.Fatal(err) 3277 } 3278 3279 got := rc.Attrs 3280 want := ReaderObjectAttrs{ 3281 Size: attrs.Size, 3282 ContentType: attrs.ContentType, 3283 ContentEncoding: attrs.ContentEncoding, 3284 CacheControl: attrs.CacheControl, 3285 LastModified: got.LastModified, // ignored, tested separately 3286 Generation: attrs.Generation, 3287 Metageneration: attrs.Metageneration, 3288 } 3289 if got != want { 3290 t.Fatalf("got %v, wanted %v", got, want) 3291 } 3292 3293 if got.LastModified.IsZero() { 3294 t.Fatal("LastModified is 0, should be >0") 3295 } 3296} 3297 3298// Test that context cancellation correctly stops a download before completion. 3299func TestIntegration_ReaderCancel(t *testing.T) { 3300 ctx := context.Background() 3301 client := testConfig(ctx, t) 3302 defer client.Close() 3303 3304 bkt := client.Bucket(bucketName) 3305 3306 // Upload a 1MB object. 3307 obj := bkt.Object("reader-cancel-obj") 3308 w := obj.NewWriter(ctx) 3309 c := randomContents() 3310 for i := 0; i < 62500; i++ { 3311 if _, err := w.Write(c); err != nil { 3312 t.Fatalf("writer.Write: %v", err) 3313 } 3314 3315 } 3316 w.Close() 3317 3318 // Create a reader (which makes a GET request to GCS and opens the body to 3319 // read the object) and then cancel the context before reading. 3320 readerCtx, cancel := context.WithCancel(ctx) 3321 r, err := obj.NewReader(readerCtx) 3322 if err != nil { 3323 t.Fatalf("obj.NewReader: %v", err) 3324 } 3325 defer r.Close() 3326 3327 cancel() 3328 3329 // Read the object 1KB a time. We cannot guarantee that Reads will return a 3330 // context canceled error immediately, but they should always do so before we 3331 // reach EOF. 3332 var readErr error 3333 for i := 0; i < 1000; i++ { 3334 buf := make([]byte, 1000) 3335 _, readErr = r.Read(buf) 3336 if readErr != nil { 3337 if readErr == context.Canceled { 3338 return 3339 } 3340 break 3341 } 3342 } 3343 t.Fatalf("Reader.Read: got %v, want context.Canceled", readErr) 3344} 3345 3346// Ensures that a file stored with a: 3347// * Content-Encoding of "gzip" 3348// * Content-Type of "text/plain" 3349// will be properly served back. 3350// See: 3351// * https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip 3352// * https://github.com/googleapis/google-cloud-go/issues/1800 3353func TestIntegration_NewReaderWithContentEncodingGzip(t *testing.T) { 3354 ctx := context.Background() 3355 client := testConfig(ctx, t) 3356 defer client.Close() 3357 3358 h := testHelper{t} 3359 3360 projectID := testutil.ProjID() 3361 bkt := client.Bucket(uidSpace.New()) 3362 h.mustCreate(bkt, projectID, nil) 3363 defer h.mustDeleteBucket(bkt) 3364 obj := bkt.Object("decompressive-transcoding") 3365 original := bytes.Repeat([]byte("a"), 4<<10) 3366 3367 // Wrap the file upload in a retry. 3368 // TODO: Investigate removing retry after resolving 3369 // https://github.com/googleapis/google-api-go-client/issues/392. 3370 err := retry(ctx, func() error { 3371 // Firstly upload the gzip compressed file. 3372 w := obj.NewWriter(ctx) 3373 // Compress and upload the content. 3374 gzw := gzip.NewWriter(w) 3375 if _, err := gzw.Write(original); err != nil { 3376 return fmt.Errorf("Failed to compress content: %v", err) 3377 } 3378 if err := gzw.Close(); err != nil { 3379 return fmt.Errorf("Failed to compress content: %v", err) 3380 } 3381 if err := w.Close(); err != nil { 3382 return fmt.Errorf("Failed to finish uploading the file: %v", err) 3383 } 3384 return nil 3385 }, 3386 nil) 3387 3388 defer h.mustDeleteObject(obj) 3389 3390 // Now update the Content-Encoding and Content-Type to enable 3391 // decompressive transcoding. 3392 updatedAttrs, err := obj.Update(ctx, ObjectAttrsToUpdate{ 3393 ContentEncoding: "gzip", 3394 ContentType: "text/plain", 3395 }) 3396 if err != nil { 3397 t.Fatalf("Attribute update failure: %v", err) 3398 } 3399 if g, w := updatedAttrs.ContentEncoding, "gzip"; g != w { 3400 t.Fatalf("ContentEncoding mismtach:\nGot: %q\nWant: %q", g, w) 3401 } 3402 if g, w := updatedAttrs.ContentType, "text/plain"; g != w { 3403 t.Fatalf("ContentType mismtach:\nGot: %q\nWant: %q", g, w) 3404 } 3405 3406 rWhole, err := obj.NewReader(ctx) 3407 if err != nil { 3408 t.Fatalf("Failed to create wholesome reader: %v", err) 3409 } 3410 blobWhole, err := ioutil.ReadAll(rWhole) 3411 rWhole.Close() 3412 if err != nil { 3413 t.Fatalf("Failed to read the whole body: %v", err) 3414 } 3415 if g, w := blobWhole, original; !bytes.Equal(g, w) { 3416 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 3417 } 3418 3419 // Now try a range read, which should return the whole body anyways since 3420 // for decompressive transcoding, range requests ARE IGNORED by Cloud Storage. 3421 r2kBTo3kB, err := obj.NewRangeReader(ctx, 2<<10, 3<<10) 3422 if err != nil { 3423 t.Fatalf("Failed to create range reader: %v", err) 3424 } 3425 blob2kBTo3kB, err := ioutil.ReadAll(r2kBTo3kB) 3426 r2kBTo3kB.Close() 3427 if err != nil { 3428 t.Fatalf("Failed to read with the 2kB to 3kB range request: %v", err) 3429 } 3430 // The ENTIRE body MUST be served back regardless of the requested range. 3431 if g, w := blob2kBTo3kB, original; !bytes.Equal(g, w) { 3432 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 3433 } 3434} 3435 3436func TestIntegration_HMACKey(t *testing.T) { 3437 ctx := context.Background() 3438 client := testConfig(ctx, t) 3439 defer client.Close() 3440 3441 projectID := testutil.ProjID() 3442 3443 // Use the service account email from the user's credentials. Requires that the 3444 // credentials are set via a JSON credentials file. 3445 // Note that a service account may only have up to 5 active HMAC keys at once; if 3446 // we see flakes because of this, we should consider switching to using a project 3447 // pool. 3448 credentials := testutil.CredentialsEnv(ctx, "GCLOUD_TESTS_GOLANG_KEY") 3449 if credentials == nil { 3450 t.Fatal("credentials could not be determined, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 3451 } 3452 if credentials.JSON == nil { 3453 t.Fatal("could not read the JSON key file, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 3454 } 3455 conf, err := google.JWTConfigFromJSON(credentials.JSON) 3456 if err != nil { 3457 t.Fatal(err) 3458 } 3459 serviceAccountEmail := conf.Email 3460 3461 hmacKey, err := client.CreateHMACKey(ctx, projectID, serviceAccountEmail) 3462 if err != nil { 3463 t.Fatalf("Failed to create HMACKey: %v", err) 3464 } 3465 if hmacKey == nil { 3466 t.Fatal("Unexpectedly got back a nil HMAC key") 3467 } 3468 3469 if hmacKey.State != Active { 3470 t.Fatalf("Unexpected state %q, expected %q", hmacKey.State, Active) 3471 } 3472 3473 hkh := client.HMACKeyHandle(projectID, hmacKey.AccessID) 3474 // 1. Ensure that we CANNOT delete an ACTIVE key. 3475 if err := hkh.Delete(ctx); err == nil { 3476 t.Fatal("Unexpectedly deleted key whose state is ACTIVE: No error from Delete.") 3477 } 3478 3479 invalidStates := []HMACState{"", Deleted, "active", "inactive", "foo_bar"} 3480 for _, invalidState := range invalidStates { 3481 t.Run("invalid-"+string(invalidState), func(t *testing.T) { 3482 _, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3483 State: invalidState, 3484 }) 3485 if err == nil { 3486 t.Fatal("Unexpectedly succeeded") 3487 } 3488 invalidStateMsg := fmt.Sprintf(`storage: invalid state %q for update, must be either "ACTIVE" or "INACTIVE"`, invalidState) 3489 if err.Error() != invalidStateMsg { 3490 t.Fatalf("Mismatched error: got: %q\nwant: %q", err, invalidStateMsg) 3491 } 3492 }) 3493 } 3494 3495 // 2.1. Setting the State to Inactive should succeed. 3496 hu, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3497 State: Inactive, 3498 }) 3499 if err != nil { 3500 t.Fatalf("Unexpected Update failure: %v", err) 3501 } 3502 if got, want := hu.State, Inactive; got != want { 3503 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 3504 } 3505 3506 // 2.2. Setting the State back to Active should succeed. 3507 hu, err = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3508 State: Active, 3509 }) 3510 if err != nil { 3511 t.Fatalf("Unexpected Update failure: %v", err) 3512 } 3513 if got, want := hu.State, Active; got != want { 3514 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 3515 } 3516 3517 // 3. Verify that keys are listed as expected. 3518 iter := client.ListHMACKeys(ctx, projectID) 3519 count := 0 3520 for ; ; count++ { 3521 _, err := iter.Next() 3522 if err == iterator.Done { 3523 break 3524 } 3525 if err != nil { 3526 t.Fatalf("Failed to ListHMACKeys: %v", err) 3527 } 3528 } 3529 if count == 0 { 3530 t.Fatal("Failed to list any HMACKeys") 3531 } 3532 3533 // 4. Finally set it to back to Inactive and 3534 // then retry the deletion which should now succeed. 3535 _, _ = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3536 State: Inactive, 3537 }) 3538 if err := hkh.Delete(ctx); err != nil { 3539 t.Fatalf("Unexpected deletion failure: %v", err) 3540 } 3541 3542 _, err = hkh.Get(ctx) 3543 if err != nil && !strings.Contains(err.Error(), "404") { 3544 // If the deleted key has already been garbage collected, a 404 is expected. 3545 // Other errors should cause a failure and are not expected. 3546 t.Fatalf("Unexpected error: %v", err) 3547 } 3548} 3549 3550func TestIntegration_PostPolicyV4(t *testing.T) { 3551 jwtConf, err := testutil.JWTConfig() 3552 if err != nil { 3553 t.Fatal(err) 3554 } 3555 if jwtConf == nil { 3556 t.Skip("JSON key file is not present") 3557 } 3558 3559 ctx := context.Background() 3560 client := testConfig(ctx, t) 3561 defer client.Close() 3562 3563 projectID := testutil.ProjID() 3564 newBucketName := uidSpace.New() 3565 b := client.Bucket(newBucketName) 3566 h := testHelper{t} 3567 h.mustCreate(b, projectID, nil) 3568 defer h.mustDeleteBucket(b) 3569 3570 statusCodeToRespond := 200 3571 opts := &PostPolicyV4Options{ 3572 GoogleAccessID: jwtConf.Email, 3573 PrivateKey: jwtConf.PrivateKey, 3574 3575 Expires: time.Now().Add(30 * time.Minute), 3576 3577 Fields: &PolicyV4Fields{ 3578 StatusCodeOnSuccess: statusCodeToRespond, 3579 ContentType: "text/plain", 3580 ACL: "public-read", 3581 }, 3582 3583 // The conditions that the uploaded file will be expected to conform to. 3584 Conditions: []PostPolicyV4Condition{ 3585 // Make the file a maximum of 10mB. 3586 ConditionContentLengthRange(0, 10<<20), 3587 ConditionStartsWith("$acl", "public"), 3588 }, 3589 } 3590 3591 objectName := "my-object.txt" 3592 pv4, err := GenerateSignedPostPolicyV4(newBucketName, objectName, opts) 3593 if err != nil { 3594 t.Fatal(err) 3595 } 3596 3597 formBuf := new(bytes.Buffer) 3598 mw := multipart.NewWriter(formBuf) 3599 for fieldName, value := range pv4.Fields { 3600 if err := mw.WriteField(fieldName, value); err != nil { 3601 t.Fatalf("Failed to write form field: %q: %v", fieldName, err) 3602 } 3603 } 3604 3605 // Now let's perform the upload. 3606 fileBody := bytes.Repeat([]byte("a"), 25) 3607 mf, err := mw.CreateFormFile("file", "myfile.txt") 3608 if err != nil { 3609 t.Fatal(err) 3610 } 3611 if _, err := mf.Write(fileBody); err != nil { 3612 t.Fatal(err) 3613 } 3614 if err := mw.Close(); err != nil { 3615 t.Fatal(err) 3616 } 3617 3618 // Compose the HTTP request. 3619 req, err := http.NewRequest("POST", pv4.URL, formBuf) 3620 if err != nil { 3621 t.Fatalf("Failed to compose HTTP request: %v", err) 3622 } 3623 // Ensure the Content-Type is derived from the writer. 3624 req.Header.Set("Content-Type", mw.FormDataContentType()) 3625 res, err := http.DefaultClient.Do(req) 3626 if err != nil { 3627 t.Fatal(err) 3628 } 3629 if g, w := res.StatusCode, statusCodeToRespond; g != w { 3630 blob, _ := httputil.DumpResponse(res, true) 3631 t.Fatalf("Status code in response mismatch: got %d want %d\nBody: %s", g, w, blob) 3632 } 3633 io.Copy(ioutil.Discard, res.Body) 3634 3635 // Verify that the file was properly uploaded, by 3636 // reading back its attributes and content! 3637 obj := b.Object(objectName) 3638 defer h.mustDeleteObject(obj) 3639 3640 attrs, err := obj.Attrs(ctx) 3641 if err != nil { 3642 t.Fatalf("Failed to retrieve attributes: %v", err) 3643 } 3644 if g, w := attrs.Size, int64(len(fileBody)); g != w { 3645 t.Errorf("ContentLength mismatch: got %d want %d", g, w) 3646 } 3647 if g, w := attrs.MD5, md5.Sum(fileBody); !bytes.Equal(g, w[:]) { 3648 t.Errorf("MD5Checksum mismatch\nGot: %x\nWant: %x", g, w) 3649 } 3650 3651 // Compare the uploaded body with the expected. 3652 rd, err := obj.NewReader(ctx) 3653 if err != nil { 3654 t.Fatalf("Failed to create a reader: %v", err) 3655 } 3656 gotBody, err := ioutil.ReadAll(rd) 3657 if err != nil { 3658 t.Fatalf("Failed to read the body: %v", err) 3659 } 3660 if diff := testutil.Diff(string(gotBody), string(fileBody)); diff != "" { 3661 t.Fatalf("Body mismatch: got - want +\n%s", diff) 3662 } 3663} 3664 3665// Verify that custom scopes passed in by the user are applied correctly. 3666func TestIntegration_Scopes(t *testing.T) { 3667 // A default client should be able to write objects since it has scope of 3668 // FullControl 3669 ctx := context.Background() 3670 clientFullControl := testConfig(ctx, t) 3671 defer clientFullControl.Close() 3672 3673 bkt := clientFullControl.Bucket(bucketName) 3674 obj := "FakeObj1" 3675 contents := []byte("This object should be written successfully\n") 3676 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 3677 t.Fatalf("writing: %v", err) 3678 } 3679 3680 // A client with ReadOnly scope should not be able to write successfully. 3681 clientReadOnly, err := NewClient(ctx, option.WithScopes(ScopeReadOnly)) 3682 defer clientReadOnly.Close() 3683 if err != nil { 3684 t.Fatalf("error creating client: %v", err) 3685 } 3686 3687 bkt = clientReadOnly.Bucket(bucketName) 3688 obj = "FakeObj2" 3689 contents = []byte("This object should not be written.\n") 3690 3691 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err == nil { 3692 t.Fatal("client with ScopeReadOnly was able to write an object unexpectedly.") 3693 } 3694 3695} 3696 3697type testHelper struct { 3698 t *testing.T 3699} 3700 3701func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) { 3702 if err := b.Create(context.Background(), projID, attrs); err != nil { 3703 h.t.Fatalf("%s: bucket create: %v", loc(), err) 3704 } 3705} 3706 3707func (h testHelper) mustDeleteBucket(b *BucketHandle) { 3708 if err := b.Delete(context.Background()); err != nil { 3709 h.t.Fatalf("%s: bucket delete: %v", loc(), err) 3710 } 3711} 3712 3713func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs { 3714 attrs, err := b.Attrs(context.Background()) 3715 if err != nil { 3716 h.t.Fatalf("%s: bucket attrs: %v", loc(), err) 3717 } 3718 return attrs 3719} 3720 3721func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate) *BucketAttrs { 3722 attrs, err := b.Update(context.Background(), ua) 3723 if err != nil { 3724 h.t.Fatalf("%s: update: %v", loc(), err) 3725 } 3726 return attrs 3727} 3728 3729func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs { 3730 attrs, err := o.Attrs(context.Background()) 3731 if err != nil { 3732 h.t.Fatalf("%s: object attrs: %v", loc(), err) 3733 } 3734 return attrs 3735} 3736 3737func (h testHelper) mustDeleteObject(o *ObjectHandle) { 3738 if err := o.Delete(context.Background()); err != nil { 3739 h.t.Fatalf("%s: object delete: %v", loc(), err) 3740 } 3741} 3742 3743func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate) *ObjectAttrs { 3744 attrs, err := o.Update(context.Background(), ua) 3745 if err != nil { 3746 h.t.Fatalf("%s: update: %v", loc(), err) 3747 } 3748 return attrs 3749} 3750 3751func (h testHelper) mustWrite(w *Writer, data []byte) { 3752 if _, err := w.Write(data); err != nil { 3753 w.Close() 3754 h.t.Fatalf("%s: write: %v", loc(), err) 3755 } 3756 if err := w.Close(); err != nil { 3757 h.t.Fatalf("%s: close write: %v", loc(), err) 3758 } 3759} 3760 3761func (h testHelper) mustRead(obj *ObjectHandle) []byte { 3762 data, err := readObject(context.Background(), obj) 3763 if err != nil { 3764 h.t.Fatalf("%s: read: %v", loc(), err) 3765 } 3766 return data 3767} 3768 3769func (h testHelper) mustNewReader(obj *ObjectHandle) *Reader { 3770 r, err := obj.NewReader(context.Background()) 3771 if err != nil { 3772 h.t.Fatalf("%s: new reader: %v", loc(), err) 3773 } 3774 return r 3775} 3776 3777func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error { 3778 w := obj.NewWriter(ctx) 3779 w.ContentType = contentType 3780 w.CacheControl = "public, max-age=60" 3781 if contents != nil { 3782 if _, err := w.Write(contents); err != nil { 3783 _ = w.Close() 3784 return err 3785 } 3786 } 3787 return w.Close() 3788} 3789 3790// loc returns a string describing the file and line of its caller's call site. In 3791// other words, if a test function calls a helper, and the helper calls loc, then the 3792// string will refer to the line on which the test function called the helper. 3793// TODO(jba): use t.Helper once we drop go 1.6. 3794func loc() string { 3795 _, file, line, ok := runtime.Caller(2) 3796 if !ok { 3797 return "???" 3798 } 3799 return fmt.Sprintf("%s:%d", filepath.Base(file), line) 3800} 3801 3802func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) { 3803 r, err := obj.NewReader(ctx) 3804 if err != nil { 3805 return nil, err 3806 } 3807 defer r.Close() 3808 return ioutil.ReadAll(r) 3809} 3810 3811// cleanupBuckets deletes the bucket used for testing, as well as old 3812// testing buckets that weren't cleaned previously. 3813func cleanupBuckets() error { 3814 if testing.Short() { 3815 return nil // Don't clean up in short mode. 3816 } 3817 ctx := context.Background() 3818 client := config(ctx) 3819 if client == nil { 3820 return nil // Don't cleanup if we're not configured correctly. 3821 } 3822 defer client.Close() 3823 if err := killBucket(ctx, client, bucketName); err != nil { 3824 return err 3825 } 3826 3827 // Delete buckets whose name begins with our test prefix, and which were 3828 // created a while ago. (Unfortunately GCS doesn't provide last-modified 3829 // time, which would be a better way to check for staleness.) 3830 const expireAge = 24 * time.Hour 3831 projectID := testutil.ProjID() 3832 it := client.Buckets(ctx, projectID) 3833 it.Prefix = testPrefix 3834 for { 3835 bktAttrs, err := it.Next() 3836 if err == iterator.Done { 3837 break 3838 } 3839 if err != nil { 3840 return err 3841 } 3842 if time.Since(bktAttrs.Created) > expireAge { 3843 log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge) 3844 if err := killBucket(ctx, client, bktAttrs.Name); err != nil { 3845 return err 3846 } 3847 } 3848 } 3849 return nil 3850} 3851 3852// killBucket deletes a bucket and all its objects. 3853func killBucket(ctx context.Context, client *Client, bucketName string) error { 3854 bkt := client.Bucket(bucketName) 3855 // Bucket must be empty to delete. 3856 it := bkt.Objects(ctx, nil) 3857 for { 3858 objAttrs, err := it.Next() 3859 if err == iterator.Done { 3860 break 3861 } 3862 if err != nil { 3863 return err 3864 } 3865 // Objects with a hold must have the hold released. 3866 if objAttrs.EventBasedHold || objAttrs.TemporaryHold { 3867 obj := bkt.Object(objAttrs.Name) 3868 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{EventBasedHold: false, TemporaryHold: false}); err != nil { 3869 return fmt.Errorf("removing hold from %q: %v", bucketName+"/"+objAttrs.Name, err) 3870 } 3871 } 3872 if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil { 3873 return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err) 3874 } 3875 } 3876 // GCS is eventually consistent, so this delete may fail because the 3877 // replica still sees an object in the bucket. We log the error and expect 3878 // a later test run to delete the bucket. 3879 if err := bkt.Delete(ctx); err != nil { 3880 log.Printf("deleting %q: %v", bucketName, err) 3881 } 3882 return nil 3883} 3884 3885func randomContents() []byte { 3886 h := md5.New() 3887 io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000))) 3888 return h.Sum(nil) 3889} 3890 3891type zeros struct{} 3892 3893func (zeros) Read(p []byte) (int, error) { return len(p), nil } 3894 3895// Make a GET request to a URL using an unauthenticated client, and return its contents. 3896func getURL(url string, headers map[string][]string) ([]byte, error) { 3897 req, err := http.NewRequest("GET", url, nil) 3898 if err != nil { 3899 return nil, err 3900 } 3901 req.Header = headers 3902 res, err := http.DefaultClient.Do(req) 3903 if err != nil { 3904 return nil, err 3905 } 3906 defer res.Body.Close() 3907 bytes, err := ioutil.ReadAll(res.Body) 3908 if err != nil { 3909 return nil, err 3910 } 3911 if res.StatusCode != 200 { 3912 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 3913 } 3914 return bytes, nil 3915} 3916 3917// Make a PUT request to a URL using an unauthenticated client, and return its contents. 3918func putURL(url string, headers map[string][]string, payload io.Reader) ([]byte, error) { 3919 req, err := http.NewRequest("PUT", url, payload) 3920 if err != nil { 3921 return nil, err 3922 } 3923 req.Header = headers 3924 res, err := http.DefaultClient.Do(req) 3925 if err != nil { 3926 return nil, err 3927 } 3928 defer res.Body.Close() 3929 bytes, err := ioutil.ReadAll(res.Body) 3930 if err != nil { 3931 return nil, err 3932 } 3933 if res.StatusCode != 200 { 3934 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 3935 } 3936 return bytes, nil 3937} 3938 3939func namesEqual(obj *ObjectAttrs, bucketName, objectName string) bool { 3940 return obj.Bucket == bucketName && obj.Name == objectName 3941} 3942 3943func keyFileEmail(filename string) (string, error) { 3944 bytes, err := ioutil.ReadFile(filename) 3945 if err != nil { 3946 return "", err 3947 } 3948 var v struct { 3949 ClientEmail string `json:"client_email"` 3950 } 3951 if err := json.Unmarshal(bytes, &v); err != nil { 3952 return "", err 3953 } 3954 return v.ClientEmail, nil 3955} 3956 3957func containsACL(acls []ACLRule, e ACLEntity, r ACLRole) bool { 3958 for _, a := range acls { 3959 if a.Entity == e && a.Role == r { 3960 return true 3961 } 3962 } 3963 return false 3964} 3965 3966func hasRule(acl []ACLRule, rule ACLRule) bool { 3967 for _, r := range acl { 3968 if cmp.Equal(r, rule) { 3969 return true 3970 } 3971 } 3972 return false 3973} 3974 3975// retry retries a function call as well as an (optional) correctness check for up 3976// to 11 seconds. Both call and check must run without error in order to succeed. 3977// If the timeout is hit, the most recent error from call or check will be returned. 3978// This function should be used to wrap calls that might cause integration test 3979// flakes due to delays in propagation (for example, metadata updates). 3980func retry(ctx context.Context, call func() error, check func() error) error { 3981 timeout := time.After(11 * time.Second) 3982 var err error 3983 for { 3984 select { 3985 case <-timeout: 3986 return err 3987 default: 3988 } 3989 err = call() 3990 if err == nil { 3991 if check == nil || check() == nil { 3992 return nil 3993 } 3994 err = check() 3995 } 3996 time.Sleep(200 * time.Millisecond) 3997 } 3998} 3999