1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "compress/gzip" 20 "context" 21 "crypto/md5" 22 "crypto/sha256" 23 "encoding/base64" 24 "encoding/json" 25 "flag" 26 "fmt" 27 "hash/crc32" 28 "io" 29 "io/ioutil" 30 "log" 31 "math/rand" 32 "mime/multipart" 33 "net/http" 34 "net/http/httputil" 35 "os" 36 "path/filepath" 37 "runtime" 38 "sort" 39 "strconv" 40 "strings" 41 "testing" 42 "time" 43 44 "cloud.google.com/go/httpreplay" 45 "cloud.google.com/go/iam" 46 "cloud.google.com/go/internal/testutil" 47 "cloud.google.com/go/internal/uid" 48 "github.com/google/go-cmp/cmp" 49 "github.com/google/go-cmp/cmp/cmpopts" 50 "golang.org/x/oauth2/google" 51 "google.golang.org/api/googleapi" 52 "google.golang.org/api/iterator" 53 itesting "google.golang.org/api/iterator/testing" 54 "google.golang.org/api/option" 55) 56 57const ( 58 testPrefix = "go-integration-test" 59 replayFilename = "storage.replay" 60 // TODO(jba): move to testutil, factor out from firestore/integration_test.go. 61 envFirestoreProjID = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID" 62 envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY" 63) 64 65var ( 66 record = flag.Bool("record", false, "record RPCs") 67 68 uidSpace *uid.Space 69 bucketName string 70 // Use our own random number generator to isolate the sequence of random numbers from 71 // other packages. This makes it possible to use HTTP replay and draw the same sequence 72 // of numbers as during recording. 73 rng *rand.Rand 74 newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error) 75 replaying bool 76 testTime time.Time 77) 78 79func TestMain(m *testing.M) { 80 cleanup := initIntegrationTest() 81 exit := m.Run() 82 if err := cleanup(); err != nil { 83 // Don't fail the test if cleanup fails. 84 log.Printf("Post-test cleanup failed: %v", err) 85 } 86 os.Exit(exit) 87} 88 89// If integration tests will be run, create a unique bucket for them. 90// Also, set newTestClient to handle record/replay. 91// Return a cleanup function. 92func initIntegrationTest() func() error { 93 flag.Parse() // needed for testing.Short() 94 switch { 95 case testing.Short() && *record: 96 log.Fatal("cannot combine -short and -record") 97 return nil 98 99 case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "": 100 // go test -short with a replay file will replay the integration tests, if 101 // the appropriate environment variables have been set. 102 replaying = true 103 httpreplay.DebugHeaders() 104 replayer, err := httpreplay.NewReplayer(replayFilename) 105 if err != nil { 106 log.Fatal(err) 107 } 108 var t time.Time 109 if err := json.Unmarshal(replayer.Initial(), &t); err != nil { 110 log.Fatal(err) 111 } 112 initUIDsAndRand(t) 113 newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) { 114 hc, err := replayer.Client(ctx) // no creds needed 115 if err != nil { 116 return nil, err 117 } 118 return NewClient(ctx, option.WithHTTPClient(hc)) 119 } 120 log.Printf("replaying from %s", replayFilename) 121 return func() error { return replayer.Close() } 122 123 case testing.Short(): 124 // go test -short without a replay file skips the integration tests. 125 if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" { 126 log.Print("replay not supported for Go versions before 1.8") 127 } 128 newTestClient = nil 129 return func() error { return nil } 130 131 default: // Run integration tests against a real backend. 132 now := time.Now().UTC() 133 initUIDsAndRand(now) 134 var cleanup func() error 135 if *record && httpreplay.Supported() { 136 // Remember the time for replay. 137 nowBytes, err := json.Marshal(now) 138 if err != nil { 139 log.Fatal(err) 140 } 141 recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes) 142 if err != nil { 143 log.Fatalf("could not record: %v", err) 144 } 145 newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 146 hc, err := recorder.Client(ctx, opts...) 147 if err != nil { 148 return nil, err 149 } 150 return NewClient(ctx, option.WithHTTPClient(hc)) 151 } 152 cleanup = func() error { 153 err1 := cleanupBuckets() 154 err2 := recorder.Close() 155 if err1 != nil { 156 return err1 157 } 158 return err2 159 } 160 log.Printf("recording to %s", replayFilename) 161 } else { 162 if *record { 163 log.Print("record not supported for Go versions before 1.8") 164 } 165 newTestClient = NewClient 166 cleanup = cleanupBuckets 167 } 168 ctx := context.Background() 169 client := config(ctx) 170 if client == nil { 171 return func() error { return nil } 172 } 173 defer client.Close() 174 if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil { 175 log.Fatalf("creating bucket %q: %v", bucketName, err) 176 } 177 return cleanup 178 } 179} 180 181func initUIDsAndRand(t time.Time) { 182 uidSpace = uid.NewSpace(testPrefix, &uid.Options{Time: t}) 183 bucketName = uidSpace.New() 184 // Use our own random source, to avoid other parts of the program taking 185 // random numbers from the global source and putting record and replay 186 // out of sync. 187 rng = testutil.NewRand(t) 188 testTime = t 189} 190 191// testConfig returns the Client used to access GCS. testConfig skips 192// the current test if credentials are not available or when being run 193// in Short mode. 194func testConfig(ctx context.Context, t *testing.T) *Client { 195 if testing.Short() && !replaying { 196 t.Skip("Integration tests skipped in short mode") 197 } 198 client := config(ctx) 199 if client == nil { 200 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 201 } 202 return client 203} 204 205// config is like testConfig, but it doesn't need a *testing.T. 206func config(ctx context.Context) *Client { 207 ts := testutil.TokenSource(ctx, ScopeFullControl) 208 if ts == nil { 209 return nil 210 } 211 client, err := newTestClient(ctx, option.WithTokenSource(ts)) 212 if err != nil { 213 log.Fatalf("NewClient: %v", err) 214 } 215 return client 216} 217 218func TestIntegration_BucketMethods(t *testing.T) { 219 ctx := context.Background() 220 client := testConfig(ctx, t) 221 defer client.Close() 222 h := testHelper{t} 223 224 projectID := testutil.ProjID() 225 newBucketName := uidSpace.New() 226 b := client.Bucket(newBucketName) 227 // Test Create and Delete. 228 h.mustCreate(b, projectID, nil) 229 attrs := h.mustBucketAttrs(b) 230 if got, want := attrs.MetaGeneration, int64(1); got != want { 231 t.Errorf("got metagen %d, want %d", got, want) 232 } 233 if got, want := attrs.StorageClass, "STANDARD"; got != want { 234 t.Errorf("got storage class %q, want %q", got, want) 235 } 236 if attrs.VersioningEnabled { 237 t.Error("got versioning enabled, wanted it disabled") 238 } 239 if attrs.LocationType == "" { 240 t.Error("got an empty LocationType") 241 } 242 h.mustDeleteBucket(b) 243 244 // Test Create and Delete with attributes. 245 labels := map[string]string{ 246 "l1": "v1", 247 "empty": "", 248 } 249 attrs = &BucketAttrs{ 250 StorageClass: "NEARLINE", 251 VersioningEnabled: true, 252 Labels: labels, 253 Lifecycle: Lifecycle{ 254 Rules: []LifecycleRule{{ 255 Action: LifecycleAction{ 256 Type: SetStorageClassAction, 257 StorageClass: "NEARLINE", 258 }, 259 Condition: LifecycleCondition{ 260 AgeInDays: 10, 261 Liveness: Archived, 262 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 263 MatchesStorageClasses: []string{"STANDARD"}, 264 NumNewerVersions: 3, 265 }, 266 }, { 267 Action: LifecycleAction{ 268 Type: SetStorageClassAction, 269 StorageClass: "ARCHIVE", 270 }, 271 Condition: LifecycleCondition{ 272 CustomTimeBefore: time.Date(2020, 1, 2, 3, 0, 0, 0, time.UTC), 273 DaysSinceCustomTime: 20, 274 Liveness: Live, 275 MatchesStorageClasses: []string{"STANDARD"}, 276 }, 277 }, { 278 Action: LifecycleAction{ 279 Type: DeleteAction, 280 }, 281 Condition: LifecycleCondition{ 282 DaysSinceNoncurrentTime: 30, 283 Liveness: Live, 284 NoncurrentTimeBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 285 MatchesStorageClasses: []string{"NEARLINE"}, 286 NumNewerVersions: 10, 287 }, 288 }}, 289 }, 290 } 291 h.mustCreate(b, projectID, attrs) 292 attrs = h.mustBucketAttrs(b) 293 if got, want := attrs.MetaGeneration, int64(1); got != want { 294 t.Errorf("got metagen %d, want %d", got, want) 295 } 296 if got, want := attrs.StorageClass, "NEARLINE"; got != want { 297 t.Errorf("got storage class %q, want %q", got, want) 298 } 299 if !attrs.VersioningEnabled { 300 t.Error("got versioning disabled, wanted it enabled") 301 } 302 if got, want := attrs.Labels, labels; !testutil.Equal(got, want) { 303 t.Errorf("labels: got %v, want %v", got, want) 304 } 305 if attrs.LocationType == "" { 306 t.Error("got an empty LocationType") 307 } 308 h.mustDeleteBucket(b) 309} 310 311func TestIntegration_BucketUpdate(t *testing.T) { 312 ctx := context.Background() 313 client := testConfig(ctx, t) 314 defer client.Close() 315 h := testHelper{t} 316 317 b := client.Bucket(uidSpace.New()) 318 h.mustCreate(b, testutil.ProjID(), nil) 319 defer h.mustDeleteBucket(b) 320 321 attrs := h.mustBucketAttrs(b) 322 if attrs.VersioningEnabled { 323 t.Fatal("bucket should not have versioning by default") 324 } 325 if len(attrs.Labels) > 0 { 326 t.Fatal("bucket should not have labels initially") 327 } 328 329 // Using empty BucketAttrsToUpdate should be a no-nop. 330 attrs = h.mustUpdateBucket(b, BucketAttrsToUpdate{}) 331 if attrs.VersioningEnabled { 332 t.Fatal("should not have versioning") 333 } 334 if len(attrs.Labels) > 0 { 335 t.Fatal("should not have labels") 336 } 337 338 // Turn on versioning, add some labels. 339 ua := BucketAttrsToUpdate{VersioningEnabled: true} 340 ua.SetLabel("l1", "v1") 341 ua.SetLabel("empty", "") 342 attrs = h.mustUpdateBucket(b, ua) 343 if !attrs.VersioningEnabled { 344 t.Fatal("should have versioning now") 345 } 346 wantLabels := map[string]string{ 347 "l1": "v1", 348 "empty": "", 349 } 350 if !testutil.Equal(attrs.Labels, wantLabels) { 351 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 352 } 353 354 // Turn off versioning again; add and remove some more labels. 355 ua = BucketAttrsToUpdate{VersioningEnabled: false} 356 ua.SetLabel("l1", "v2") // update 357 ua.SetLabel("new", "new") // create 358 ua.DeleteLabel("empty") // delete 359 ua.DeleteLabel("absent") // delete non-existent 360 attrs = h.mustUpdateBucket(b, ua) 361 if attrs.VersioningEnabled { 362 t.Fatal("should have versioning off") 363 } 364 wantLabels = map[string]string{ 365 "l1": "v2", 366 "new": "new", 367 } 368 if !testutil.Equal(attrs.Labels, wantLabels) { 369 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 370 } 371 372 // Configure a lifecycle 373 wantLifecycle := Lifecycle{ 374 Rules: []LifecycleRule{ 375 { 376 Action: LifecycleAction{Type: "Delete"}, 377 Condition: LifecycleCondition{AgeInDays: 30}, 378 }, 379 }, 380 } 381 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle} 382 attrs = h.mustUpdateBucket(b, ua) 383 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 384 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 385 } 386} 387 388func TestIntegration_BucketPolicyOnly(t *testing.T) { 389 ctx := context.Background() 390 client := testConfig(ctx, t) 391 defer client.Close() 392 h := testHelper{t} 393 bkt := client.Bucket(bucketName) 394 395 // Insert an object with custom ACL. 396 o := bkt.Object("bucketPolicyOnly") 397 defer func() { 398 if err := o.Delete(ctx); err != nil { 399 log.Printf("failed to delete test object: %v", err) 400 } 401 }() 402 wc := o.NewWriter(ctx) 403 wc.ContentType = "text/plain" 404 h.mustWrite(wc, []byte("test")) 405 a := o.ACL() 406 aclEntity := ACLEntity("user-test@example.com") 407 err := a.Set(ctx, aclEntity, RoleReader) 408 if err != nil { 409 t.Fatalf("set ACL failed: %v", err) 410 } 411 412 // Enable BucketPolicyOnly. 413 ua := BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: true}} 414 attrs := h.mustUpdateBucket(bkt, ua) 415 if got, want := attrs.BucketPolicyOnly.Enabled, true; got != want { 416 t.Fatalf("got %v, want %v", got, want) 417 } 418 if got := attrs.BucketPolicyOnly.LockedTime; got.IsZero() { 419 t.Fatal("got a zero time value, want a populated value") 420 } 421 422 // Confirm BucketAccessControl returns error. 423 err = retry(ctx, func() error { 424 _, err = bkt.ACL().List(ctx) 425 return nil 426 }, func() error { 427 if err == nil { 428 return fmt.Errorf("ACL.List: expected bucket ACL list to fail") 429 } 430 return nil 431 }) 432 if err != nil { 433 t.Fatal(err) 434 } 435 436 // Confirm ObjectAccessControl returns error. 437 err = retry(ctx, func() error { 438 _, err = o.ACL().List(ctx) 439 return nil 440 }, func() error { 441 if err == nil { 442 return fmt.Errorf("ACL.List: expected object ACL list to fail") 443 } 444 return nil 445 }) 446 if err != nil { 447 t.Fatal(err) 448 } 449 450 // Disable BucketPolicyOnly. 451 ua = BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: false}} 452 attrs = h.mustUpdateBucket(bkt, ua) 453 if got, want := attrs.BucketPolicyOnly.Enabled, false; got != want { 454 t.Fatalf("got %v, want %v", got, want) 455 } 456 457 // Check that the object ACLs are the same. 458 var acls []ACLRule 459 err = retry(ctx, func() error { 460 acls, err = o.ACL().List(ctx) 461 if err != nil { 462 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 463 } 464 return nil 465 }, func() error { 466 if !containsACL(acls, aclEntity, RoleReader) { 467 return fmt.Errorf("containsACL: expected ACLs %v to include custom ACL entity %v", acls, aclEntity) 468 } 469 return nil 470 }) 471 if err != nil { 472 t.Fatal(err) 473 } 474} 475 476func TestIntegration_UniformBucketLevelAccess(t *testing.T) { 477 ctx := context.Background() 478 client := testConfig(ctx, t) 479 defer client.Close() 480 h := testHelper{t} 481 bkt := client.Bucket(uidSpace.New()) 482 h.mustCreate(bkt, testutil.ProjID(), nil) 483 defer h.mustDeleteBucket(bkt) 484 485 // Insert an object with custom ACL. 486 o := bkt.Object("uniformBucketLevelAccess") 487 defer func() { 488 if err := o.Delete(ctx); err != nil { 489 log.Printf("failed to delete test object: %v", err) 490 } 491 }() 492 wc := o.NewWriter(ctx) 493 wc.ContentType = "text/plain" 494 h.mustWrite(wc, []byte("test")) 495 a := o.ACL() 496 aclEntity := ACLEntity("user-test@example.com") 497 err := a.Set(ctx, aclEntity, RoleReader) 498 if err != nil { 499 t.Fatalf("set ACL failed: %v", err) 500 } 501 502 // Enable UniformBucketLevelAccess. 503 ua := BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}} 504 attrs := h.mustUpdateBucket(bkt, ua) 505 if got, want := attrs.UniformBucketLevelAccess.Enabled, true; got != want { 506 t.Fatalf("got %v, want %v", got, want) 507 } 508 if got := attrs.UniformBucketLevelAccess.LockedTime; got.IsZero() { 509 t.Fatal("got a zero time value, want a populated value") 510 } 511 512 // Confirm BucketAccessControl returns error. 513 err = retry(ctx, func() error { 514 _, err = bkt.ACL().List(ctx) 515 return nil 516 }, func() error { 517 if err == nil { 518 return fmt.Errorf("ACL.List: expected bucket ACL list to fail") 519 } 520 return nil 521 }) 522 if err != nil { 523 t.Fatal(err) 524 } 525 526 // Confirm ObjectAccessControl returns error. 527 err = retry(ctx, func() error { 528 _, err = o.ACL().List(ctx) 529 return nil 530 }, func() error { 531 if err == nil { 532 return fmt.Errorf("ACL.List: expected object ACL list to fail") 533 } 534 return nil 535 }) 536 if err != nil { 537 t.Fatal(err) 538 } 539 540 // Disable UniformBucketLevelAccess. 541 ua = BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: false}} 542 attrs = h.mustUpdateBucket(bkt, ua) 543 if got, want := attrs.UniformBucketLevelAccess.Enabled, false; got != want { 544 t.Fatalf("got %v, want %v", got, want) 545 } 546 547 // Check that the object ACLs are the same. 548 var acls []ACLRule 549 err = retry(ctx, func() error { 550 acls, err = o.ACL().List(ctx) 551 if err != nil { 552 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 553 } 554 return nil 555 }, func() error { 556 if !containsACL(acls, aclEntity, RoleReader) { 557 return fmt.Errorf("containsACL: expected ACLs %v to include custom ACL entity %v", acls, aclEntity) 558 } 559 return nil 560 }) 561 if err != nil { 562 t.Fatal(err) 563 } 564} 565 566func TestIntegration_ConditionalDelete(t *testing.T) { 567 ctx := context.Background() 568 client := testConfig(ctx, t) 569 defer client.Close() 570 h := testHelper{t} 571 572 o := client.Bucket(bucketName).Object("conddel") 573 574 wc := o.NewWriter(ctx) 575 wc.ContentType = "text/plain" 576 h.mustWrite(wc, []byte("foo")) 577 578 gen := wc.Attrs().Generation 579 metaGen := wc.Attrs().Metageneration 580 581 if err := o.Generation(gen - 1).Delete(ctx); err == nil { 582 t.Fatalf("Unexpected successful delete with Generation") 583 } 584 if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil { 585 t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch") 586 } 587 if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil { 588 t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch") 589 } 590 if err := o.Generation(gen).Delete(ctx); err != nil { 591 t.Fatalf("final delete failed: %v", err) 592 } 593} 594 595func TestIntegration_ObjectsRangeReader(t *testing.T) { 596 ctx := context.Background() 597 client := testConfig(ctx, t) 598 defer client.Close() 599 bkt := client.Bucket(bucketName) 600 601 objName := uidSpace.New() 602 obj := bkt.Object(objName) 603 contents := []byte("Hello, world this is a range request") 604 605 if err := retry(ctx, func() error { 606 w := obj.NewWriter(ctx) 607 if _, err := w.Write(contents); err != nil { 608 return fmt.Errorf("Failed to write contents: %v", err) 609 } 610 if err := w.Close(); err != nil { 611 return fmt.Errorf("Failed to close writer: %v", err) 612 } 613 return nil 614 }, nil); err != nil { 615 t.Fatal(err) 616 } 617 618 last5s := []struct { 619 name string 620 start int64 621 length int64 622 }{ 623 {name: "negative offset", start: -5, length: -1}, 624 {name: "offset with specified length", start: int64(len(contents)) - 5, length: 5}, 625 {name: "offset and read till end", start: int64(len(contents)) - 5, length: -1}, 626 } 627 628 for _, last5 := range last5s { 629 t.Run(last5.name, func(t *testing.T) { 630 r, err := obj.NewRangeReader(ctx, last5.start, last5.length) 631 if err != nil { 632 t.Fatalf("Failed to make range read: %v", err) 633 } 634 defer r.Close() 635 636 if got, want := r.Attrs.StartOffset, int64(len(contents))-5; got != want { 637 t.Fatalf("StartOffset mismatch, got %d want %d", got, want) 638 } 639 640 nr, _ := io.Copy(ioutil.Discard, r) 641 if got, want := nr, int64(5); got != want { 642 t.Fatalf("Body length mismatch, got %d want %d", got, want) 643 } 644 }) 645 } 646} 647 648func TestIntegration_Objects(t *testing.T) { 649 // TODO(jba): Use subtests (Go 1.7). 650 ctx := context.Background() 651 client := testConfig(ctx, t) 652 defer client.Close() 653 // Reset testTime, 'cause object last modification time should be within 5 min 654 // from test (test iteration if -count passed) start time. 655 testTime = time.Now().UTC() 656 newBucketName := uidSpace.New() 657 h := testHelper{t} 658 bkt := client.Bucket(newBucketName) 659 660 h.mustCreate(bkt, testutil.ProjID(), nil) 661 defer func() { 662 if err := killBucket(ctx, client, newBucketName); err != nil { 663 log.Printf("deleting %q: %v", newBucketName, err) 664 } 665 }() 666 const defaultType = "text/plain" 667 668 // Populate object names and make a map for their contents. 669 objects := []string{ 670 "obj1", 671 "obj2", 672 "obj/with/slashes", 673 } 674 contents := make(map[string][]byte) 675 676 // Test Writer. 677 for _, obj := range objects { 678 c := randomContents() 679 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 680 t.Errorf("Write for %v failed with %v", obj, err) 681 } 682 contents[obj] = c 683 } 684 685 testObjectIterator(t, bkt, objects) 686 testObjectsIterateSelectedAttrs(t, bkt, objects) 687 testObjectsIterateAllSelectedAttrs(t, bkt, objects) 688 testObjectIteratorWithOffset(t, bkt, objects) 689 t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) { 690 query := &Query{Prefix: "", Delimiter: "/"} 691 if err := query.SetAttrSelection([]string{"Name"}); err != nil { 692 t.Fatalf("selecting query attrs: %v", err) 693 } 694 695 var gotNames []string 696 var gotPrefixes []string 697 it := bkt.Objects(context.Background(), query) 698 for { 699 attrs, err := it.Next() 700 if err == iterator.Done { 701 break 702 } 703 if err != nil { 704 t.Fatalf("iterator.Next: %v", err) 705 } 706 if attrs.Name != "" { 707 gotNames = append(gotNames, attrs.Name) 708 } else if attrs.Prefix != "" { 709 gotPrefixes = append(gotPrefixes, attrs.Prefix) 710 } 711 712 if attrs.Bucket != "" { 713 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 714 } 715 } 716 717 sortedNames := []string{"obj1", "obj2"} 718 if !cmp.Equal(sortedNames, gotNames) { 719 t.Errorf("names = %v, want %v", gotNames, sortedNames) 720 } 721 sortedPrefixes := []string{"obj/"} 722 if !cmp.Equal(sortedPrefixes, gotPrefixes) { 723 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes) 724 } 725 }) 726 727 // Test Reader. 728 for _, obj := range objects { 729 rc, err := bkt.Object(obj).NewReader(ctx) 730 if err != nil { 731 t.Errorf("Can't create a reader for %v, errored with %v", obj, err) 732 continue 733 } 734 if !rc.checkCRC { 735 t.Errorf("%v: not checking CRC", obj) 736 } 737 slurp, err := ioutil.ReadAll(rc) 738 if err != nil { 739 t.Errorf("Can't ReadAll object %v, errored with %v", obj, err) 740 } 741 if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { 742 t.Errorf("Contents (%q) = %q; want %q", obj, got, want) 743 } 744 if got, want := rc.Size(), len(contents[obj]); got != int64(want) { 745 t.Errorf("Size (%q) = %d; want %d", obj, got, want) 746 } 747 if got, want := rc.ContentType(), "text/plain"; got != want { 748 t.Errorf("ContentType (%q) = %q; want %q", obj, got, want) 749 } 750 if got, want := rc.CacheControl(), "public, max-age=60"; got != want { 751 t.Errorf("CacheControl (%q) = %q; want %q", obj, got, want) 752 } 753 // We just wrote these objects, so they should have a recent last-modified time. 754 lm, err := rc.LastModified() 755 // Accept a time within +/- of the test time, to account for natural 756 // variation and the fact that testTime is set at the start of the test run. 757 expectedVariance := 5 * time.Minute 758 if err != nil { 759 t.Errorf("LastModified (%q): got error %v", obj, err) 760 } else if lm.Before(testTime.Add(-expectedVariance)) || lm.After(testTime.Add(expectedVariance)) { 761 t.Errorf("LastModified (%q): got %s, which not the %v from now (%v)", obj, lm, expectedVariance, testTime) 762 } 763 rc.Close() 764 765 // Check early close. 766 buf := make([]byte, 1) 767 rc, err = bkt.Object(obj).NewReader(ctx) 768 if err != nil { 769 t.Fatalf("%v: %v", obj, err) 770 } 771 _, err = rc.Read(buf) 772 if err != nil { 773 t.Fatalf("%v: %v", obj, err) 774 } 775 if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) { 776 t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want) 777 } 778 if err := rc.Close(); err != nil { 779 t.Errorf("%v Close: %v", obj, err) 780 } 781 } 782 783 obj := objects[0] 784 objlen := int64(len(contents[obj])) 785 // Test Range Reader. 786 for i, r := range []struct { 787 offset, length, want int64 788 }{ 789 {0, objlen, objlen}, 790 {0, objlen / 2, objlen / 2}, 791 {objlen / 2, objlen, objlen / 2}, 792 {0, 0, 0}, 793 {objlen / 2, 0, 0}, 794 {objlen / 2, -1, objlen / 2}, 795 {0, objlen * 2, objlen}, 796 {-2, -1, 2}, 797 {-objlen, -1, objlen}, 798 {-(objlen / 2), -1, objlen / 2}, 799 } { 800 rc, err := bkt.Object(obj).NewRangeReader(ctx, r.offset, r.length) 801 if err != nil { 802 t.Errorf("%+v: Can't create a range reader for %v, errored with %v", i, obj, err) 803 continue 804 } 805 if rc.Size() != objlen { 806 t.Errorf("%+v: Reader has a content-size of %d, want %d", i, rc.Size(), objlen) 807 } 808 if rc.Remain() != r.want { 809 t.Errorf("%+v: Reader's available bytes reported as %d, want %d", i, rc.Remain(), r.want) 810 } 811 slurp, err := ioutil.ReadAll(rc) 812 if err != nil { 813 t.Errorf("%+v: can't ReadAll object %v, errored with %v", r, obj, err) 814 continue 815 } 816 if len(slurp) != int(r.want) { 817 t.Errorf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", i, r.offset, r.length, len(slurp), r.want) 818 continue 819 } 820 821 switch { 822 case r.offset < 0: // The case of reading the last N bytes. 823 start := objlen + r.offset 824 if got, want := slurp, contents[obj][start:]; !bytes.Equal(got, want) { 825 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 826 } 827 828 default: 829 if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) { 830 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 831 } 832 } 833 rc.Close() 834 } 835 836 objName := objects[0] 837 838 // Test NewReader googleapi.Error. 839 // Since a 429 or 5xx is hard to cause, we trigger a 416. 840 realLen := len(contents[objName]) 841 _, err := bkt.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10) 842 if err, ok := err.(*googleapi.Error); !ok { 843 t.Error("NewRangeReader did not return a googleapi.Error") 844 } else { 845 if err.Code != 416 { 846 t.Errorf("Code = %d; want %d", err.Code, 416) 847 } 848 if len(err.Header) == 0 { 849 t.Error("Missing googleapi.Error.Header") 850 } 851 if len(err.Body) == 0 { 852 t.Error("Missing googleapi.Error.Body") 853 } 854 } 855 856 // Test StatObject. 857 o := h.mustObjectAttrs(bkt.Object(objName)) 858 if got, want := o.Name, objName; got != want { 859 t.Errorf("Name (%v) = %q; want %q", objName, got, want) 860 } 861 if got, want := o.ContentType, defaultType; got != want { 862 t.Errorf("ContentType (%v) = %q; want %q", objName, got, want) 863 } 864 created := o.Created 865 // Check that the object is newer than its containing bucket. 866 bAttrs := h.mustBucketAttrs(bkt) 867 if o.Created.Before(bAttrs.Created) { 868 t.Errorf("Object %v is older than its containing bucket, %v", o, bAttrs) 869 } 870 871 // Test object copy. 872 copyName := "copy-" + objName 873 copyObj, err := bkt.Object(copyName).CopierFrom(bkt.Object(objName)).Run(ctx) 874 if err != nil { 875 t.Errorf("Copier.Run failed with %v", err) 876 } else if !namesEqual(copyObj, newBucketName, copyName) { 877 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 878 copyObj.Bucket, copyObj.Name, newBucketName, copyName) 879 } 880 881 // Copying with attributes. 882 const contentEncoding = "identity" 883 copier := bkt.Object(copyName).CopierFrom(bkt.Object(objName)) 884 copier.ContentEncoding = contentEncoding 885 copyObj, err = copier.Run(ctx) 886 if err != nil { 887 t.Errorf("Copier.Run failed with %v", err) 888 } else { 889 if !namesEqual(copyObj, newBucketName, copyName) { 890 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 891 copyObj.Bucket, copyObj.Name, newBucketName, copyName) 892 } 893 if copyObj.ContentEncoding != contentEncoding { 894 t.Errorf("Copy ContentEncoding: got %q, want %q", copyObj.ContentEncoding, contentEncoding) 895 } 896 } 897 898 // Test UpdateAttrs. 899 metadata := map[string]string{"key": "value"} 900 updated := h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 901 ContentType: "text/html", 902 ContentLanguage: "en", 903 Metadata: metadata, 904 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, 905 }) 906 if got, want := updated.ContentType, "text/html"; got != want { 907 t.Errorf("updated.ContentType == %q; want %q", got, want) 908 } 909 if got, want := updated.ContentLanguage, "en"; got != want { 910 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 911 } 912 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) { 913 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want) 914 } 915 if got, want := updated.Created, created; got != want { 916 t.Errorf("updated.Created == %q; want %q", got, want) 917 } 918 if !updated.Created.Before(updated.Updated) { 919 t.Errorf("updated.Updated should be newer than update.Created") 920 } 921 922 // Delete ContentType and ContentLanguage. 923 updated = h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 924 ContentType: "", 925 ContentLanguage: "", 926 Metadata: map[string]string{}, 927 }) 928 if got, want := updated.ContentType, ""; got != want { 929 t.Errorf("updated.ContentType == %q; want %q", got, want) 930 } 931 if got, want := updated.ContentLanguage, ""; got != want { 932 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 933 } 934 if updated.Metadata != nil { 935 t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata) 936 } 937 if got, want := updated.Created, created; got != want { 938 t.Errorf("updated.Created == %q; want %q", got, want) 939 } 940 if !updated.Created.Before(updated.Updated) { 941 t.Errorf("updated.Updated should be newer than update.Created") 942 } 943 944 // Test checksums. 945 checksumCases := []struct { 946 name string 947 contents [][]byte 948 size int64 949 md5 string 950 crc32c uint32 951 }{ 952 { 953 name: "checksum-object", 954 contents: [][]byte{[]byte("hello"), []byte("world")}, 955 size: 10, 956 md5: "fc5e038d38a57032085441e7fe7010b0", 957 crc32c: 1456190592, 958 }, 959 { 960 name: "zero-object", 961 contents: [][]byte{}, 962 size: 0, 963 md5: "d41d8cd98f00b204e9800998ecf8427e", 964 crc32c: 0, 965 }, 966 } 967 for _, c := range checksumCases { 968 wc := bkt.Object(c.name).NewWriter(ctx) 969 for _, data := range c.contents { 970 if _, err := wc.Write(data); err != nil { 971 t.Errorf("Write(%q) failed with %q", data, err) 972 } 973 } 974 if err = wc.Close(); err != nil { 975 t.Errorf("%q: close failed with %q", c.name, err) 976 } 977 obj := wc.Attrs() 978 if got, want := obj.Size, c.size; got != want { 979 t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want) 980 } 981 if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want { 982 t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want) 983 } 984 if got, want := obj.CRC32C, c.crc32c; got != want { 985 t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want) 986 } 987 } 988 989 // Test public ACL. 990 publicObj := objects[0] 991 if err = bkt.Object(publicObj).ACL().Set(ctx, AllUsers, RoleReader); err != nil { 992 t.Errorf("PutACLEntry failed with %v", err) 993 } 994 publicClient, err := newTestClient(ctx, option.WithoutAuthentication()) 995 if err != nil { 996 t.Fatal(err) 997 } 998 999 slurp := h.mustRead(publicClient.Bucket(newBucketName).Object(publicObj)) 1000 if !bytes.Equal(slurp, contents[publicObj]) { 1001 t.Errorf("Public object's content: got %q, want %q", slurp, contents[publicObj]) 1002 } 1003 1004 // Test writer error handling. 1005 wc := publicClient.Bucket(newBucketName).Object(publicObj).NewWriter(ctx) 1006 if _, err := wc.Write([]byte("hello")); err != nil { 1007 t.Errorf("Write unexpectedly failed with %v", err) 1008 } 1009 if err = wc.Close(); err == nil { 1010 t.Error("Close expected an error, found none") 1011 } 1012 1013 // Test deleting the copy object. 1014 h.mustDeleteObject(bkt.Object(copyName)) 1015 // Deleting it a second time should return ErrObjectNotExist. 1016 if err := bkt.Object(copyName).Delete(ctx); err != ErrObjectNotExist { 1017 t.Errorf("second deletion of %v = %v; want ErrObjectNotExist", copyName, err) 1018 } 1019 _, err = bkt.Object(copyName).Attrs(ctx) 1020 if err != ErrObjectNotExist { 1021 t.Errorf("Copy is expected to be deleted, stat errored with %v", err) 1022 } 1023 1024 // Test object composition. 1025 var compSrcs []*ObjectHandle 1026 var wantContents []byte 1027 for _, obj := range objects { 1028 compSrcs = append(compSrcs, bkt.Object(obj)) 1029 wantContents = append(wantContents, contents[obj]...) 1030 } 1031 checkCompose := func(obj *ObjectHandle, wantContentType string) { 1032 rc := h.mustNewReader(obj) 1033 slurp, err = ioutil.ReadAll(rc) 1034 if err != nil { 1035 t.Fatalf("ioutil.ReadAll: %v", err) 1036 } 1037 defer rc.Close() 1038 if !bytes.Equal(slurp, wantContents) { 1039 t.Errorf("Composed object contents\ngot: %q\nwant: %q", slurp, wantContents) 1040 } 1041 if got := rc.ContentType(); got != wantContentType { 1042 t.Errorf("Composed object content-type = %q, want %q", got, wantContentType) 1043 } 1044 } 1045 1046 // Compose should work even if the user sets no destination attributes. 1047 compDst := bkt.Object("composed1") 1048 c := compDst.ComposerFrom(compSrcs...) 1049 if _, err := c.Run(ctx); err != nil { 1050 t.Fatalf("ComposeFrom error: %v", err) 1051 } 1052 checkCompose(compDst, "application/octet-stream") 1053 1054 // It should also work if we do. 1055 compDst = bkt.Object("composed2") 1056 c = compDst.ComposerFrom(compSrcs...) 1057 c.ContentType = "text/json" 1058 if _, err := c.Run(ctx); err != nil { 1059 t.Fatalf("ComposeFrom error: %v", err) 1060 } 1061 checkCompose(compDst, "text/json") 1062} 1063 1064func TestIntegration_Encoding(t *testing.T) { 1065 ctx := context.Background() 1066 client := testConfig(ctx, t) 1067 defer client.Close() 1068 bkt := client.Bucket(bucketName) 1069 1070 // Test content encoding 1071 const zeroCount = 20 << 1 // TODO: should be 20 << 20 1072 obj := bkt.Object("gzip-test") 1073 w := obj.NewWriter(ctx) 1074 w.ContentEncoding = "gzip" 1075 gw := gzip.NewWriter(w) 1076 if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil { 1077 t.Fatalf("io.Copy, upload: %v", err) 1078 } 1079 if err := gw.Close(); err != nil { 1080 t.Errorf("gzip.Close(): %v", err) 1081 } 1082 if err := w.Close(); err != nil { 1083 t.Errorf("w.Close(): %v", err) 1084 } 1085 r, err := obj.NewReader(ctx) 1086 if err != nil { 1087 t.Fatalf("NewReader(gzip-test): %v", err) 1088 } 1089 n, err := io.Copy(ioutil.Discard, r) 1090 if err != nil { 1091 t.Errorf("io.Copy, download: %v", err) 1092 } 1093 if n != zeroCount { 1094 t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount) 1095 } 1096 1097 // Test NotFound. 1098 _, err = bkt.Object("obj-not-exists").NewReader(ctx) 1099 if err != ErrObjectNotExist { 1100 t.Errorf("Object should not exist, err found to be %v", err) 1101 } 1102} 1103 1104func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) { 1105 ctx := context.Background() 1106 h := testHelper{t} 1107 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1108 names := make([]string, len(objects)) 1109 copy(names, objects) 1110 sort.Strings(names) 1111 var attrs []*ObjectAttrs 1112 for _, name := range names { 1113 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1114 } 1115 msg, ok := itesting.TestIterator(attrs, 1116 func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) }, 1117 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1118 if !ok { 1119 t.Errorf("ObjectIterator.Next: %s", msg) 1120 } 1121 // TODO(jba): test query.Delimiter != "" 1122} 1123 1124func testObjectIteratorWithOffset(t *testing.T, bkt *BucketHandle, objects []string) { 1125 ctx := context.Background() 1126 h := testHelper{t} 1127 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1128 names := make([]string, len(objects)) 1129 copy(names, objects) 1130 sort.Strings(names) 1131 var attrs []*ObjectAttrs 1132 for _, name := range names { 1133 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1134 } 1135 m := make(map[string][]*ObjectAttrs) 1136 for i, name := range names { 1137 // StartOffset takes the value of object names, the result must be for: 1138 // ― obj/with/slashes: obj/with/slashes, obj1, obj2 1139 // ― obj1: obj1, obj2 1140 // ― obj2: obj2. 1141 m[name] = attrs[i:] 1142 msg, ok := itesting.TestIterator(m[name], 1143 func() interface{} { return bkt.Objects(ctx, &Query{StartOffset: name}) }, 1144 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1145 if !ok { 1146 t.Errorf("ObjectIterator.Next: %s", msg) 1147 } 1148 // EndOffset takes the value of object names, the result must be for: 1149 // ― obj/with/slashes: "" 1150 // ― obj1: obj/with/slashes 1151 // ― obj2: obj/with/slashes, obj1. 1152 m[name] = attrs[:i] 1153 msg, ok = itesting.TestIterator(m[name], 1154 func() interface{} { return bkt.Objects(ctx, &Query{EndOffset: name}) }, 1155 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1156 if !ok { 1157 t.Errorf("ObjectIterator.Next: %s", msg) 1158 } 1159 } 1160} 1161 1162func testObjectsIterateSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1163 // Create a query that will only select the "Name" attr of objects, and 1164 // invoke object listing. 1165 query := &Query{Prefix: ""} 1166 query.SetAttrSelection([]string{"Name"}) 1167 1168 var gotNames []string 1169 it := bkt.Objects(context.Background(), query) 1170 for { 1171 attrs, err := it.Next() 1172 if err == iterator.Done { 1173 break 1174 } 1175 if err != nil { 1176 t.Fatalf("iterator.Next: %v", err) 1177 } 1178 gotNames = append(gotNames, attrs.Name) 1179 1180 if len(attrs.Bucket) > 0 { 1181 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 1182 } 1183 } 1184 1185 sortedNames := make([]string, len(objects)) 1186 copy(sortedNames, objects) 1187 sort.Strings(sortedNames) 1188 sort.Strings(gotNames) 1189 1190 if !cmp.Equal(sortedNames, gotNames) { 1191 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1192 } 1193} 1194 1195func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1196 // Tests that all selected attributes work - query succeeds (without actually 1197 // verifying the returned results). 1198 query := &Query{ 1199 Prefix: "", 1200 StartOffset: "obj/with/slashes", 1201 EndOffset: "obj2", 1202 } 1203 var selectedAttrs []string 1204 for k := range attrToFieldMap { 1205 selectedAttrs = append(selectedAttrs, k) 1206 } 1207 query.SetAttrSelection(selectedAttrs) 1208 1209 count := 0 1210 it := bkt.Objects(context.Background(), query) 1211 for { 1212 _, err := it.Next() 1213 if err == iterator.Done { 1214 break 1215 } 1216 if err != nil { 1217 t.Fatalf("iterator.Next: %v", err) 1218 } 1219 count++ 1220 } 1221 1222 if count != len(objects)-1 { 1223 t.Errorf("count = %v, want %v", count, len(objects)-1) 1224 } 1225} 1226 1227func TestIntegration_SignedURL(t *testing.T) { 1228 if testing.Short() { // do not test during replay 1229 t.Skip("Integration tests skipped in short mode") 1230 } 1231 // To test SignedURL, we need a real user email and private key. Extract them 1232 // from the JSON key file. 1233 jwtConf, err := testutil.JWTConfig() 1234 if err != nil { 1235 t.Fatal(err) 1236 } 1237 if jwtConf == nil { 1238 t.Skip("JSON key file is not present") 1239 } 1240 1241 ctx := context.Background() 1242 client := testConfig(ctx, t) 1243 defer client.Close() 1244 1245 bkt := client.Bucket(bucketName) 1246 obj := "signedURL" 1247 contents := []byte("This is a test of SignedURL.\n") 1248 md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents 1249 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 1250 t.Fatalf("writing: %v", err) 1251 } 1252 for _, test := range []struct { 1253 desc string 1254 opts SignedURLOptions 1255 headers map[string][]string 1256 fail bool 1257 }{ 1258 { 1259 desc: "basic v2", 1260 }, 1261 { 1262 desc: "basic v4", 1263 opts: SignedURLOptions{Scheme: SigningSchemeV4}, 1264 }, 1265 { 1266 desc: "MD5 sent and matches", 1267 opts: SignedURLOptions{MD5: md5}, 1268 headers: map[string][]string{"Content-MD5": {md5}}, 1269 }, 1270 { 1271 desc: "MD5 not sent", 1272 opts: SignedURLOptions{MD5: md5}, 1273 fail: true, 1274 }, 1275 { 1276 desc: "Content-Type sent and matches", 1277 opts: SignedURLOptions{ContentType: "text/plain"}, 1278 headers: map[string][]string{"Content-Type": {"text/plain"}}, 1279 }, 1280 { 1281 desc: "Content-Type sent but does not match", 1282 opts: SignedURLOptions{ContentType: "text/plain"}, 1283 headers: map[string][]string{"Content-Type": {"application/json"}}, 1284 fail: true, 1285 }, 1286 { 1287 desc: "Canonical headers sent and match", 1288 opts: SignedURLOptions{Headers: []string{ 1289 " X-Goog-Foo: Bar baz ", 1290 "X-Goog-Novalue", // ignored: no value 1291 "X-Google-Foo", // ignored: wrong prefix 1292 }}, 1293 headers: map[string][]string{"X-Goog-foo": {"Bar baz "}}, 1294 }, 1295 { 1296 desc: "Canonical headers sent but don't match", 1297 opts: SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}}, 1298 headers: map[string][]string{"X-Goog-Foo": {"bar baz"}}, 1299 fail: true, 1300 }, 1301 } { 1302 opts := test.opts 1303 opts.GoogleAccessID = jwtConf.Email 1304 opts.PrivateKey = jwtConf.PrivateKey 1305 opts.Method = "GET" 1306 opts.Expires = time.Now().Add(time.Hour) 1307 u, err := SignedURL(bucketName, obj, &opts) 1308 if err != nil { 1309 t.Errorf("%s: SignedURL: %v", test.desc, err) 1310 continue 1311 } 1312 got, err := getURL(u, test.headers) 1313 if err != nil && !test.fail { 1314 t.Errorf("%s: getURL %q: %v", test.desc, u, err) 1315 } else if err == nil && !bytes.Equal(got, contents) { 1316 t.Errorf("%s: got %q, want %q", test.desc, got, contents) 1317 } 1318 } 1319} 1320 1321func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { 1322 if testing.Short() { // do not test during replay 1323 t.Skip("Integration tests skipped in short mode") 1324 } 1325 // To test SignedURL, we need a real user email and private key. Extract 1326 // them from the JSON key file. 1327 jwtConf, err := testutil.JWTConfig() 1328 if err != nil { 1329 t.Fatal(err) 1330 } 1331 if jwtConf == nil { 1332 t.Skip("JSON key file is not present") 1333 } 1334 1335 ctx := context.Background() 1336 client := testConfig(ctx, t) 1337 defer client.Close() 1338 1339 // TODO(deklerk): document how these were generated and their significance 1340 encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=" 1341 encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=" 1342 headers := map[string][]string{ 1343 "x-goog-encryption-algorithm": {"AES256"}, 1344 "x-goog-encryption-key": {encryptionKey}, 1345 "x-goog-encryption-key-sha256": {encryptionKeySha256}, 1346 } 1347 contents := []byte(`{"message":"encryption with csek works"}`) 1348 tests := []struct { 1349 desc string 1350 opts *SignedURLOptions 1351 }{ 1352 { 1353 desc: "v4 URL with customer supplied encryption keys for PUT", 1354 opts: &SignedURLOptions{ 1355 Method: "PUT", 1356 Headers: []string{ 1357 "x-goog-encryption-algorithm:AES256", 1358 "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=", 1359 "x-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=", 1360 }, 1361 Scheme: SigningSchemeV4, 1362 }, 1363 }, 1364 { 1365 desc: "v4 URL with customer supplied encryption keys for GET", 1366 opts: &SignedURLOptions{ 1367 Method: "GET", 1368 Headers: []string{ 1369 "x-goog-encryption-algorithm:AES256", 1370 fmt.Sprintf("x-goog-encryption-key:%s", encryptionKey), 1371 fmt.Sprintf("x-goog-encryption-key-sha256:%s", encryptionKeySha256), 1372 }, 1373 Scheme: SigningSchemeV4, 1374 }, 1375 }, 1376 } 1377 defer func() { 1378 // Delete encrypted object. 1379 bkt := client.Bucket(bucketName) 1380 err := bkt.Object("csek.json").Delete(ctx) 1381 if err != nil { 1382 log.Printf("failed to deleted encrypted file: %v", err) 1383 } 1384 }() 1385 1386 for _, test := range tests { 1387 opts := test.opts 1388 opts.GoogleAccessID = jwtConf.Email 1389 opts.PrivateKey = jwtConf.PrivateKey 1390 opts.Expires = time.Now().Add(time.Hour) 1391 1392 u, err := SignedURL(bucketName, "csek.json", test.opts) 1393 if err != nil { 1394 t.Fatalf("%s: %v", test.desc, err) 1395 } 1396 1397 if test.opts.Method == "PUT" { 1398 if _, err := putURL(u, headers, bytes.NewReader(contents)); err != nil { 1399 t.Fatalf("%s: %v", test.desc, err) 1400 } 1401 } 1402 1403 if test.opts.Method == "GET" { 1404 got, err := getURL(u, headers) 1405 if err != nil { 1406 t.Fatalf("%s: %v", test.desc, err) 1407 } 1408 if !bytes.Equal(got, contents) { 1409 t.Fatalf("%s: got %q, want %q", test.desc, got, contents) 1410 } 1411 } 1412 } 1413} 1414 1415func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) { 1416 if testing.Short() { // do not test during replay 1417 t.Skip("Integration tests skipped in short mode") 1418 } 1419 1420 // To test SignedURL, we need a real user email and private key. Extract them 1421 // from the JSON key file. 1422 jwtConf, err := testutil.JWTConfig() 1423 if err != nil { 1424 t.Fatal(err) 1425 } 1426 if jwtConf == nil { 1427 t.Skip("JSON key file is not present") 1428 } 1429 1430 ctx := context.Background() 1431 client := testConfig(ctx, t) 1432 defer client.Close() 1433 1434 opts := &SignedURLOptions{ 1435 Scheme: SigningSchemeV4, 1436 Method: "GET", 1437 GoogleAccessID: jwtConf.Email, 1438 PrivateKey: jwtConf.PrivateKey, 1439 Expires: time.Now().Add(time.Hour), 1440 } 1441 1442 u, err := SignedURL(bucketName, "", opts) 1443 if err != nil { 1444 t.Fatal(err) 1445 } 1446 1447 // Should be some ListBucketResult response. 1448 _, err = getURL(u, nil) 1449 if err != nil { 1450 t.Fatal(err) 1451 } 1452} 1453 1454func TestIntegration_ACL(t *testing.T) { 1455 ctx := context.Background() 1456 client := testConfig(ctx, t) 1457 defer client.Close() 1458 1459 bkt := client.Bucket(bucketName) 1460 1461 entity := ACLEntity("domain-google.com") 1462 rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"} 1463 if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil { 1464 t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err) 1465 } 1466 1467 acl, err := bkt.DefaultObjectACL().List(ctx) 1468 if err != nil { 1469 t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucketName, err) 1470 } else if !hasRule(acl, rule) { 1471 t.Errorf("default ACL missing %#v", rule) 1472 } 1473 aclObjects := []string{"acl1", "acl2"} 1474 name := aclObjects[0] 1475 o := bkt.Object(name) 1476 err = retry(ctx, func() error { 1477 for _, obj := range aclObjects { 1478 c := randomContents() 1479 if err := writeObject(ctx, bkt.Object(obj), "", c); err != nil { 1480 t.Errorf("Write for %v failed with %v", obj, err) 1481 } 1482 } 1483 acl, err = o.ACL().List(ctx) 1484 if err != nil { 1485 return fmt.Errorf("ACL.List: can't retrieve ACL of %v", name) 1486 } 1487 return nil 1488 }, func() error { 1489 if !hasRule(acl, rule) { 1490 return fmt.Errorf("hasRule: object ACL missing %+v", rule) 1491 } 1492 return nil 1493 }) 1494 if err != nil { 1495 t.Error(err) 1496 } 1497 if err := o.ACL().Delete(ctx, entity); err != nil { 1498 t.Errorf("object ACL: could not delete entity %s", entity) 1499 } 1500 // Delete the default ACL rule. We can't move this code earlier in the 1501 // test, because the test depends on the fact that the object ACL inherits 1502 // it. 1503 if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil { 1504 t.Errorf("default ACL: could not delete entity %s", entity) 1505 } 1506 1507 entity2 := ACLEntity("user-jbd@google.com") 1508 rule2 := ACLRule{Entity: entity2, Role: RoleReader, Email: "jbd@google.com"} 1509 if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil { 1510 t.Errorf("Error while putting bucket ACL rule: %v", err) 1511 } 1512 var bACL []ACLRule 1513 err = retry(ctx, func() error { 1514 bACL, err = bkt.ACL().List(ctx) 1515 if err != nil { 1516 return fmt.Errorf("ACL.List: error while getting the ACL of the bucket: %v", err) 1517 } 1518 return nil 1519 }, func() error { 1520 if !hasRule(bACL, rule2) { 1521 return fmt.Errorf("hasRule: bucket ACL missing %+v", rule2) 1522 } 1523 return nil 1524 }) 1525 if err != nil { 1526 t.Error(err) 1527 } 1528 if err := bkt.ACL().Delete(ctx, entity2); err != nil { 1529 t.Errorf("Error while deleting bucket ACL rule: %v", err) 1530 } 1531 1532} 1533 1534func TestIntegration_ValidObjectNames(t *testing.T) { 1535 ctx := context.Background() 1536 client := testConfig(ctx, t) 1537 defer client.Close() 1538 1539 bkt := client.Bucket(bucketName) 1540 1541 validNames := []string{ 1542 "gopher", 1543 "Гоферови", 1544 "a", 1545 strings.Repeat("a", 1024), 1546 } 1547 for _, name := range validNames { 1548 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 1549 t.Errorf("Object %q write failed: %v. Want success", name, err) 1550 continue 1551 } 1552 defer bkt.Object(name).Delete(ctx) 1553 } 1554 1555 invalidNames := []string{ 1556 "", // Too short. 1557 strings.Repeat("a", 1025), // Too long. 1558 "new\nlines", 1559 "bad\xffunicode", 1560 } 1561 for _, name := range invalidNames { 1562 // Invalid object names will either cause failure during Write or Close. 1563 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 1564 continue 1565 } 1566 defer bkt.Object(name).Delete(ctx) 1567 t.Errorf("%q should have failed. Didn't", name) 1568 } 1569} 1570 1571func TestIntegration_WriterContentType(t *testing.T) { 1572 ctx := context.Background() 1573 client := testConfig(ctx, t) 1574 defer client.Close() 1575 1576 obj := client.Bucket(bucketName).Object("content") 1577 testCases := []struct { 1578 content string 1579 setType, wantType string 1580 }{ 1581 { 1582 content: "It was the best of times, it was the worst of times.", 1583 wantType: "text/plain; charset=utf-8", 1584 }, 1585 { 1586 content: "<html><head><title>My first page</title></head></html>", 1587 wantType: "text/html; charset=utf-8", 1588 }, 1589 { 1590 content: "<html><head><title>My first page</title></head></html>", 1591 setType: "text/html", 1592 wantType: "text/html", 1593 }, 1594 { 1595 content: "<html><head><title>My first page</title></head></html>", 1596 setType: "image/jpeg", 1597 wantType: "image/jpeg", 1598 }, 1599 } 1600 for i, tt := range testCases { 1601 if err := writeObject(ctx, obj, tt.setType, []byte(tt.content)); err != nil { 1602 t.Errorf("writing #%d: %v", i, err) 1603 } 1604 attrs, err := obj.Attrs(ctx) 1605 if err != nil { 1606 t.Errorf("obj.Attrs: %v", err) 1607 continue 1608 } 1609 if got := attrs.ContentType; got != tt.wantType { 1610 t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType) 1611 } 1612 } 1613} 1614 1615func TestIntegration_ZeroSizedObject(t *testing.T) { 1616 t.Parallel() 1617 ctx := context.Background() 1618 client := testConfig(ctx, t) 1619 defer client.Close() 1620 h := testHelper{t} 1621 1622 obj := client.Bucket(bucketName).Object("zero") 1623 1624 // Check writing it works as expected. 1625 w := obj.NewWriter(ctx) 1626 if err := w.Close(); err != nil { 1627 t.Fatalf("Writer.Close: %v", err) 1628 } 1629 defer obj.Delete(ctx) 1630 1631 // Check we can read it too. 1632 body := h.mustRead(obj) 1633 if len(body) != 0 { 1634 t.Errorf("Body is %v, want empty []byte{}", body) 1635 } 1636} 1637 1638func TestIntegration_Encryption(t *testing.T) { 1639 // This function tests customer-supplied encryption keys for all operations 1640 // involving objects. Bucket and ACL operations aren't tested because they 1641 // aren't affected by customer encryption. Neither is deletion. 1642 ctx := context.Background() 1643 client := testConfig(ctx, t) 1644 defer client.Close() 1645 h := testHelper{t} 1646 1647 obj := client.Bucket(bucketName).Object("customer-encryption") 1648 key := []byte("my-secret-AES-256-encryption-key") 1649 keyHash := sha256.Sum256(key) 1650 keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:]) 1651 key2 := []byte("My-Secret-AES-256-Encryption-Key") 1652 contents := "top secret." 1653 1654 checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) { 1655 // Performing a metadata operation without the key should succeed. 1656 attrs, err := f(obj) 1657 if err != nil { 1658 t.Fatalf("%s: %v", msg, err) 1659 } 1660 // The key hash should match... 1661 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1662 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1663 } 1664 // ...but CRC and MD5 should not be present. 1665 if attrs.CRC32C != 0 { 1666 t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C) 1667 } 1668 if len(attrs.MD5) > 0 { 1669 t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5) 1670 } 1671 1672 // Performing a metadata operation with the key should succeed. 1673 attrs, err = f(obj.Key(key)) 1674 if err != nil { 1675 t.Fatalf("%s: %v", msg, err) 1676 } 1677 // Check the key and content hashes. 1678 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1679 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1680 } 1681 if attrs.CRC32C == 0 { 1682 t.Errorf("%s: CRC: got 0, want non-zero", msg) 1683 } 1684 if len(attrs.MD5) == 0 { 1685 t.Errorf("%s: MD5: got len == 0, want len > 0", msg) 1686 } 1687 } 1688 1689 checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) { 1690 // Reading the object without the key should fail. 1691 if _, err := readObject(ctx, o); err == nil { 1692 t.Errorf("%s: reading without key: want error, got nil", msg) 1693 } 1694 // Reading the object with the key should succeed. 1695 got := h.mustRead(o.Key(k)) 1696 gotContents := string(got) 1697 // And the contents should match what we wrote. 1698 if gotContents != wantContents { 1699 t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents) 1700 } 1701 } 1702 1703 checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) { 1704 got := h.mustRead(obj) 1705 gotContents := string(got) 1706 if gotContents != wantContents { 1707 t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents) 1708 } 1709 } 1710 1711 // Write to obj using our own encryption key, which is a valid 32-byte 1712 // AES-256 key. 1713 h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents)) 1714 1715 checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) { 1716 return o.Attrs(ctx) 1717 }) 1718 1719 checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) { 1720 return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 1721 }) 1722 1723 checkRead("first object", obj, key, contents) 1724 1725 obj2 := client.Bucket(bucketName).Object("customer-encryption-2") 1726 // Copying an object without the key should fail. 1727 if _, err := obj2.CopierFrom(obj).Run(ctx); err == nil { 1728 t.Fatal("want error, got nil") 1729 } 1730 // Copying an object with the key should succeed. 1731 if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1732 t.Fatal(err) 1733 } 1734 // The destination object is not encrypted; we can read it without a key. 1735 checkReadUnencrypted("copy dest", obj2, contents) 1736 1737 // Providing a key on the destination but not the source should fail, 1738 // since the source is encrypted. 1739 if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil { 1740 t.Fatal("want error, got nil") 1741 } 1742 1743 // But copying with keys for both source and destination should succeed. 1744 if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1745 t.Fatal(err) 1746 } 1747 // And the destination should be encrypted, meaning we can only read it 1748 // with a key. 1749 checkRead("copy destination", obj2, key2, contents) 1750 1751 // Change obj2's key to prepare for compose, where all objects must have 1752 // the same key. Also illustrates key rotation: copy an object to itself 1753 // with a different key. 1754 if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil { 1755 t.Fatal(err) 1756 } 1757 obj3 := client.Bucket(bucketName).Object("customer-encryption-3") 1758 // Composing without keys should fail. 1759 if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil { 1760 t.Fatal("want error, got nil") 1761 } 1762 // Keys on the source objects result in an error. 1763 if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil { 1764 t.Fatal("want error, got nil") 1765 } 1766 // A key on the destination object both decrypts the source objects 1767 // and encrypts the destination. 1768 if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil { 1769 t.Fatalf("got %v, want nil", err) 1770 } 1771 // Check that the destination in encrypted. 1772 checkRead("compose destination", obj3, key, contents+contents) 1773 1774 // You can't compose one or more unencrypted source objects into an 1775 // encrypted destination object. 1776 _, err := obj2.CopierFrom(obj2.Key(key)).Run(ctx) // unencrypt obj2 1777 if err != nil { 1778 t.Fatal(err) 1779 } 1780 if _, err := obj3.Key(key).ComposerFrom(obj2).Run(ctx); err == nil { 1781 t.Fatal("got nil, want error") 1782 } 1783} 1784 1785func TestIntegration_NonexistentBucket(t *testing.T) { 1786 t.Parallel() 1787 ctx := context.Background() 1788 client := testConfig(ctx, t) 1789 defer client.Close() 1790 1791 bkt := client.Bucket(uidSpace.New()) 1792 if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist { 1793 t.Errorf("Attrs: got %v, want ErrBucketNotExist", err) 1794 } 1795 it := bkt.Objects(ctx, nil) 1796 if _, err := it.Next(); err != ErrBucketNotExist { 1797 t.Errorf("Objects: got %v, want ErrBucketNotExist", err) 1798 } 1799} 1800 1801func TestIntegration_PerObjectStorageClass(t *testing.T) { 1802 const ( 1803 defaultStorageClass = "STANDARD" 1804 newStorageClass = "NEARLINE" 1805 ) 1806 ctx := context.Background() 1807 client := testConfig(ctx, t) 1808 defer client.Close() 1809 h := testHelper{t} 1810 1811 bkt := client.Bucket(bucketName) 1812 1813 // The bucket should have the default storage class. 1814 battrs := h.mustBucketAttrs(bkt) 1815 if battrs.StorageClass != defaultStorageClass { 1816 t.Fatalf("bucket storage class: got %q, want %q", 1817 battrs.StorageClass, defaultStorageClass) 1818 } 1819 // Write an object; it should start with the bucket's storage class. 1820 obj := bkt.Object("posc") 1821 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 1822 oattrs, err := obj.Attrs(ctx) 1823 if err != nil { 1824 t.Fatal(err) 1825 } 1826 if oattrs.StorageClass != defaultStorageClass { 1827 t.Fatalf("object storage class: got %q, want %q", 1828 oattrs.StorageClass, defaultStorageClass) 1829 } 1830 // Now use Copy to change the storage class. 1831 copier := obj.CopierFrom(obj) 1832 copier.StorageClass = newStorageClass 1833 oattrs2, err := copier.Run(ctx) 1834 if err != nil { 1835 log.Fatal(err) 1836 } 1837 if oattrs2.StorageClass != newStorageClass { 1838 t.Fatalf("new object storage class: got %q, want %q", 1839 oattrs2.StorageClass, newStorageClass) 1840 } 1841 1842 // We can also write a new object using a non-default storage class. 1843 obj2 := bkt.Object("posc2") 1844 w := obj2.NewWriter(ctx) 1845 w.StorageClass = newStorageClass 1846 h.mustWrite(w, []byte("xxx")) 1847 if w.Attrs().StorageClass != newStorageClass { 1848 t.Fatalf("new object storage class: got %q, want %q", 1849 w.Attrs().StorageClass, newStorageClass) 1850 } 1851} 1852 1853func TestIntegration_BucketInCopyAttrs(t *testing.T) { 1854 // Confirm that if bucket is included in the object attributes of a rewrite 1855 // call, but object name and content-type aren't, then we get an error. See 1856 // the comment in Copier.Run. 1857 ctx := context.Background() 1858 client := testConfig(ctx, t) 1859 defer client.Close() 1860 h := testHelper{t} 1861 1862 bkt := client.Bucket(bucketName) 1863 obj := bkt.Object("bucketInCopyAttrs") 1864 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 1865 copier := obj.CopierFrom(obj) 1866 rawObject := copier.ObjectAttrs.toRawObject(bucketName) 1867 _, err := copier.callRewrite(ctx, rawObject) 1868 if err == nil { 1869 t.Errorf("got nil, want error") 1870 } 1871} 1872 1873func TestIntegration_NoUnicodeNormalization(t *testing.T) { 1874 t.Parallel() 1875 ctx := context.Background() 1876 client := testConfig(ctx, t) 1877 defer client.Close() 1878 bkt := client.Bucket("storage-library-test-bucket") 1879 h := testHelper{t} 1880 1881 for _, tst := range []struct { 1882 nameQuoted, content string 1883 }{ 1884 {`"Caf\u00e9"`, "Normalization Form C"}, 1885 {`"Cafe\u0301"`, "Normalization Form D"}, 1886 } { 1887 name, err := strconv.Unquote(tst.nameQuoted) 1888 if err != nil { 1889 t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err) 1890 } 1891 if got := string(h.mustRead(bkt.Object(name))); got != tst.content { 1892 t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content) 1893 } 1894 } 1895} 1896 1897func TestIntegration_HashesOnUpload(t *testing.T) { 1898 // Check that the user can provide hashes on upload, and that these are checked. 1899 ctx := context.Background() 1900 client := testConfig(ctx, t) 1901 defer client.Close() 1902 obj := client.Bucket(bucketName).Object("hashesOnUpload-1") 1903 data := []byte("I can't wait to be verified") 1904 1905 write := func(w *Writer) error { 1906 if _, err := w.Write(data); err != nil { 1907 _ = w.Close() 1908 return err 1909 } 1910 return w.Close() 1911 } 1912 1913 crc32c := crc32.Checksum(data, crc32cTable) 1914 // The correct CRC should succeed. 1915 w := obj.NewWriter(ctx) 1916 w.CRC32C = crc32c 1917 w.SendCRC32C = true 1918 if err := write(w); err != nil { 1919 t.Fatal(err) 1920 } 1921 1922 // If we change the CRC, validation should fail. 1923 w = obj.NewWriter(ctx) 1924 w.CRC32C = crc32c + 1 1925 w.SendCRC32C = true 1926 if err := write(w); err == nil { 1927 t.Fatal("write with bad CRC32c: want error, got nil") 1928 } 1929 1930 // If we have the wrong CRC but forget to send it, we succeed. 1931 w = obj.NewWriter(ctx) 1932 w.CRC32C = crc32c + 1 1933 if err := write(w); err != nil { 1934 t.Fatal(err) 1935 } 1936 1937 // MD5 1938 md5 := md5.Sum(data) 1939 // The correct MD5 should succeed. 1940 w = obj.NewWriter(ctx) 1941 w.MD5 = md5[:] 1942 if err := write(w); err != nil { 1943 t.Fatal(err) 1944 } 1945 1946 // If we change the MD5, validation should fail. 1947 w = obj.NewWriter(ctx) 1948 w.MD5 = append([]byte(nil), md5[:]...) 1949 w.MD5[0]++ 1950 if err := write(w); err == nil { 1951 t.Fatal("write with bad MD5: want error, got nil") 1952 } 1953} 1954 1955func TestIntegration_BucketIAM(t *testing.T) { 1956 ctx := context.Background() 1957 client := testConfig(ctx, t) 1958 defer client.Close() 1959 h := testHelper{t} 1960 bkt := client.Bucket(uidSpace.New()) 1961 h.mustCreate(bkt, testutil.ProjID(), nil) 1962 defer h.mustDeleteBucket(bkt) 1963 // This bucket is unique to this test run. So we don't have 1964 // to worry about other runs interfering with our IAM policy 1965 // changes. 1966 1967 member := "projectViewer:" + testutil.ProjID() 1968 role := iam.RoleName("roles/storage.objectViewer") 1969 // Get the bucket's IAM policy. 1970 policy, err := bkt.IAM().Policy(ctx) 1971 if err != nil { 1972 t.Fatalf("Getting policy: %v", err) 1973 } 1974 // The member should not have the role. 1975 if policy.HasRole(member, role) { 1976 t.Errorf("member %q has role %q", member, role) 1977 } 1978 // Change the policy. 1979 policy.Add(member, role) 1980 if err := bkt.IAM().SetPolicy(ctx, policy); err != nil { 1981 t.Fatalf("SetPolicy: %v", err) 1982 } 1983 // Confirm that the binding was added. 1984 policy, err = bkt.IAM().Policy(ctx) 1985 if err != nil { 1986 t.Fatalf("Getting policy: %v", err) 1987 } 1988 if !policy.HasRole(member, role) { 1989 t.Errorf("member %q does not have role %q", member, role) 1990 } 1991 1992 // Check TestPermissions. 1993 // This client should have all these permissions (and more). 1994 perms := []string{"storage.buckets.get", "storage.buckets.delete"} 1995 got, err := bkt.IAM().TestPermissions(ctx, perms) 1996 if err != nil { 1997 t.Fatalf("TestPermissions: %v", err) 1998 } 1999 sort.Strings(perms) 2000 sort.Strings(got) 2001 if !testutil.Equal(got, perms) { 2002 t.Errorf("got %v, want %v", got, perms) 2003 } 2004} 2005 2006func TestIntegration_RequesterPays(t *testing.T) { 2007 // This test needs a second project and user (token source) to test 2008 // all possibilities. Since we need these things for Firestore already, 2009 // we use them here. 2010 // 2011 // There are up to three entities involved in a requester-pays call: 2012 // 2013 // 1. The user making the request. Here, we use 2014 // a. The account used to create the token source used for all our 2015 // integration tests (see testutil.TokenSource). 2016 // b. The account used for the Firestore tests. 2017 // 2. The project that owns the requester-pays bucket. Here, that 2018 // is the test project ID (see testutil.ProjID). 2019 // 3. The project provided as the userProject parameter of the request; 2020 // the project to be billed. This test uses: 2021 // a. The project that owns the requester-pays bucket (same as (2)) 2022 // b. Another project (the Firestore project). 2023 // 2024 // The following must hold for this test to work: 2025 // - (1a) must have resourcemanager.projects.createBillingAssignment permission 2026 // (Owner role) on (2) (the project, not the bucket). 2027 // - (1b) must NOT have that permission on (2). 2028 // - (1b) must have serviceusage.services.use permission (Editor role) on (3b). 2029 // - (1b) must NOT have that permission on (3a). 2030 // - (1a) must NOT have that permission on (3b). 2031 2032 t.Skip("https://github.com/googleapis/google-cloud-go/issues/1753") 2033 const wantErrorCode = 400 2034 2035 ctx := context.Background() 2036 client := testConfig(ctx, t) 2037 defer client.Close() 2038 h := testHelper{t} 2039 2040 bucketName2 := uidSpace.New() 2041 b1 := client.Bucket(bucketName2) 2042 projID := testutil.ProjID() 2043 // Use Firestore project as a project that does not contain the bucket. 2044 otherProjID := os.Getenv(envFirestoreProjID) 2045 if otherProjID == "" { 2046 t.Fatalf("need a second project (env var %s)", envFirestoreProjID) 2047 } 2048 ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl) 2049 if ts == nil { 2050 t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey) 2051 } 2052 otherClient, err := newTestClient(ctx, option.WithTokenSource(ts)) 2053 if err != nil { 2054 t.Fatal(err) 2055 } 2056 defer otherClient.Close() 2057 b2 := otherClient.Bucket(bucketName2) 2058 user, err := keyFileEmail(os.Getenv("GCLOUD_TESTS_GOLANG_KEY")) 2059 if err != nil { 2060 t.Fatal(err) 2061 } 2062 otherUser, err := keyFileEmail(os.Getenv(envFirestorePrivateKey)) 2063 if err != nil { 2064 t.Fatal(err) 2065 } 2066 2067 // Create a requester-pays bucket. The bucket is contained in the project projID. 2068 h.mustCreate(b1, projID, &BucketAttrs{RequesterPays: true}) 2069 if err := b1.ACL().Set(ctx, ACLEntity("user-"+otherUser), RoleOwner); err != nil { 2070 t.Fatal(err) 2071 } 2072 2073 // Extract the error code from err if it's a googleapi.Error. 2074 errCode := func(err error) int { 2075 if err == nil { 2076 return 0 2077 } 2078 if err, ok := err.(*googleapi.Error); ok { 2079 return err.Code 2080 } 2081 return -1 2082 } 2083 2084 // Call f under various conditions. 2085 // Here b1 and b2 refer to the same bucket, but b1 is bound to client, 2086 // while b2 is bound to otherClient. The clients differ in their credentials, 2087 // i.e. the identity of the user making the RPC: b1's user is an Owner on the 2088 // bucket's containing project, b2's is not. 2089 call := func(msg string, f func(*BucketHandle) error) { 2090 // user: an Owner on the containing project 2091 // userProject: absent 2092 // result: success, by the rule permitting access by owners of the containing bucket. 2093 if err := f(b1); err != nil { 2094 t.Errorf("%s: %v, want nil\n"+ 2095 "confirm that %s is an Owner on %s", 2096 msg, err, user, projID) 2097 } 2098 // user: an Owner on the containing project 2099 // userProject: containing project 2100 // result: success, by the same rule as above; userProject is unnecessary but allowed. 2101 if err := f(b1.UserProject(projID)); err != nil { 2102 t.Errorf("%s: got %v, want nil", msg, err) 2103 } 2104 // user: not an Owner on the containing project 2105 // userProject: absent 2106 // result: failure, by the standard requester-pays rule 2107 err := f(b2) 2108 if got, want := errCode(err), wantErrorCode; got != want { 2109 t.Errorf("%s: got error %v with code %d, want code %d\n"+ 2110 "confirm that %s is NOT an Owner on %s", 2111 msg, err, got, want, otherUser, projID) 2112 } 2113 // user: not an Owner on the containing project 2114 // userProject: not the containing one, but user has Editor role on it 2115 // result: success, by the standard requester-pays rule 2116 if err := f(b2.UserProject(otherProjID)); err != nil { 2117 t.Errorf("%s: got %v, want nil\n"+ 2118 "confirm that %s is an Editor on %s and that that project has billing enabled", 2119 msg, err, otherUser, otherProjID) 2120 } 2121 // user: not an Owner on the containing project 2122 // userProject: the containing one, on which the user does NOT have Editor permission. 2123 // result: failure 2124 err = f(b2.UserProject("veener-jba")) 2125 if got, want := errCode(err), 403; got != want { 2126 t.Errorf("%s: got error %v, want code %d\n"+ 2127 "confirm that %s is NOT an Editor on %s", 2128 msg, err, want, otherUser, "veener-jba") 2129 } 2130 } 2131 2132 // Getting its attributes requires a user project. 2133 var attrs *BucketAttrs 2134 call("Bucket attrs", func(b *BucketHandle) error { 2135 a, err := b.Attrs(ctx) 2136 if a != nil { 2137 attrs = a 2138 } 2139 return err 2140 }) 2141 if attrs != nil { 2142 if got, want := attrs.RequesterPays, true; got != want { 2143 t.Fatalf("attr.RequesterPays = %t, want %t", got, want) 2144 } 2145 } 2146 // Object operations. 2147 call("write object", func(b *BucketHandle) error { 2148 return writeObject(ctx, b.Object("foo"), "text/plain", []byte("hello")) 2149 }) 2150 call("read object", func(b *BucketHandle) error { 2151 _, err := readObject(ctx, b.Object("foo")) 2152 return err 2153 }) 2154 call("object attrs", func(b *BucketHandle) error { 2155 _, err := b.Object("foo").Attrs(ctx) 2156 return err 2157 }) 2158 call("update object", func(b *BucketHandle) error { 2159 _, err := b.Object("foo").Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 2160 return err 2161 }) 2162 2163 // ACL operations. 2164 entity := ACLEntity("domain-google.com") 2165 call("bucket acl set", func(b *BucketHandle) error { 2166 return b.ACL().Set(ctx, entity, RoleReader) 2167 }) 2168 call("bucket acl list", func(b *BucketHandle) error { 2169 _, err := b.ACL().List(ctx) 2170 return err 2171 }) 2172 call("bucket acl delete", func(b *BucketHandle) error { 2173 err := b.ACL().Delete(ctx, entity) 2174 if errCode(err) == 404 { 2175 // Since we call the function multiple times, it will 2176 // fail with NotFound for all but the first. 2177 return nil 2178 } 2179 return err 2180 }) 2181 call("default object acl set", func(b *BucketHandle) error { 2182 return b.DefaultObjectACL().Set(ctx, entity, RoleReader) 2183 }) 2184 call("default object acl list", func(b *BucketHandle) error { 2185 _, err := b.DefaultObjectACL().List(ctx) 2186 return err 2187 }) 2188 call("default object acl delete", func(b *BucketHandle) error { 2189 err := b.DefaultObjectACL().Delete(ctx, entity) 2190 if errCode(err) == 404 { 2191 return nil 2192 } 2193 return err 2194 }) 2195 call("object acl set", func(b *BucketHandle) error { 2196 return b.Object("foo").ACL().Set(ctx, entity, RoleReader) 2197 }) 2198 call("object acl list", func(b *BucketHandle) error { 2199 _, err := b.Object("foo").ACL().List(ctx) 2200 return err 2201 }) 2202 call("object acl delete", func(b *BucketHandle) error { 2203 err := b.Object("foo").ACL().Delete(ctx, entity) 2204 if errCode(err) == 404 { 2205 return nil 2206 } 2207 return err 2208 }) 2209 2210 // Copy and compose. 2211 call("copy", func(b *BucketHandle) error { 2212 _, err := b.Object("copy").CopierFrom(b.Object("foo")).Run(ctx) 2213 return err 2214 }) 2215 call("compose", func(b *BucketHandle) error { 2216 _, err := b.Object("compose").ComposerFrom(b.Object("foo"), b.Object("copy")).Run(ctx) 2217 return err 2218 }) 2219 call("delete object", func(b *BucketHandle) error { 2220 // Make sure the object exists, so we don't get confused by ErrObjectNotExist. 2221 // The storage service may perform validation in any order (perhaps in parallel), 2222 // so if we delete an object that doesn't exist and for which we lack permission, 2223 // we could see either of those two errors. (See Google-internal bug 78341001.) 2224 h.mustWrite(b1.Object("foo").NewWriter(ctx), []byte("hello")) // note: b1, not b. 2225 return b.Object("foo").Delete(ctx) 2226 }) 2227 b1.Object("foo").Delete(ctx) // Make sure object is deleted. 2228 for _, obj := range []string{"copy", "compose"} { 2229 if err := b1.UserProject(projID).Object(obj).Delete(ctx); err != nil { 2230 t.Fatalf("could not delete %q: %v", obj, err) 2231 } 2232 } 2233 2234 h.mustDeleteBucket(b1) 2235} 2236 2237func TestIntegration_Notifications(t *testing.T) { 2238 ctx := context.Background() 2239 client := testConfig(ctx, t) 2240 defer client.Close() 2241 bkt := client.Bucket(bucketName) 2242 2243 checkNotifications := func(msg string, want map[string]*Notification) { 2244 got, err := bkt.Notifications(ctx) 2245 if err != nil { 2246 t.Fatal(err) 2247 } 2248 if diff := testutil.Diff(got, want); diff != "" { 2249 t.Errorf("%s: got=-, want=+:\n%s", msg, diff) 2250 } 2251 } 2252 checkNotifications("initial", map[string]*Notification{}) 2253 2254 nArg := &Notification{ 2255 TopicProjectID: testutil.ProjID(), 2256 TopicID: "go-storage-notification-test", 2257 PayloadFormat: NoPayload, 2258 } 2259 n, err := bkt.AddNotification(ctx, nArg) 2260 if err != nil { 2261 t.Fatal(err) 2262 } 2263 nArg.ID = n.ID 2264 if !testutil.Equal(n, nArg) { 2265 t.Errorf("got %+v, want %+v", n, nArg) 2266 } 2267 checkNotifications("after add", map[string]*Notification{n.ID: n}) 2268 2269 if err := bkt.DeleteNotification(ctx, n.ID); err != nil { 2270 t.Fatal(err) 2271 } 2272 checkNotifications("after delete", map[string]*Notification{}) 2273} 2274 2275func TestIntegration_PublicBucket(t *testing.T) { 2276 // Confirm that an unauthenticated client can access a public bucket. 2277 // See https://cloud.google.com/storage/docs/public-datasets/landsat 2278 if testing.Short() && !replaying { 2279 t.Skip("Integration tests skipped in short mode") 2280 } 2281 2282 const landsatBucket = "gcp-public-data-landsat" 2283 const landsatPrefix = "LC08/PRE/044/034/LC80440342016259LGN00/" 2284 const landsatObject = landsatPrefix + "LC80440342016259LGN00_MTL.txt" 2285 2286 // Create an unauthenticated client. 2287 ctx := context.Background() 2288 client, err := newTestClient(ctx, option.WithoutAuthentication()) 2289 if err != nil { 2290 t.Fatal(err) 2291 } 2292 defer client.Close() 2293 h := testHelper{t} 2294 bkt := client.Bucket(landsatBucket) 2295 obj := bkt.Object(landsatObject) 2296 2297 // Read a public object. 2298 bytes := h.mustRead(obj) 2299 if got, want := len(bytes), 7903; got != want { 2300 t.Errorf("len(bytes) = %d, want %d", got, want) 2301 } 2302 2303 // List objects in a public bucket. 2304 iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix}) 2305 gotCount := 0 2306 for { 2307 _, err := iter.Next() 2308 if err == iterator.Done { 2309 break 2310 } 2311 if err != nil { 2312 t.Fatal(err) 2313 } 2314 gotCount++ 2315 } 2316 if wantCount := 13; gotCount != wantCount { 2317 t.Errorf("object count: got %d, want %d", gotCount, wantCount) 2318 } 2319 2320 errCode := func(err error) int { 2321 err2, ok := err.(*googleapi.Error) 2322 if !ok { 2323 return -1 2324 } 2325 return err2.Code 2326 } 2327 2328 // Reading from or writing to a non-public bucket fails. 2329 c := testConfig(ctx, t) 2330 defer c.Close() 2331 nonPublicObj := client.Bucket(bucketName).Object("noauth") 2332 // Oddly, reading returns 403 but writing returns 401. 2333 _, err = readObject(ctx, nonPublicObj) 2334 if got, want := errCode(err), 403; got != want { 2335 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 2336 } 2337 err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b")) 2338 if got, want := errCode(err), 401; got != want { 2339 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 2340 } 2341} 2342 2343func TestIntegration_ReadCRC(t *testing.T) { 2344 // Test that the checksum is handled correctly when reading files. 2345 // For gzipped files, see https://github.com/GoogleCloudPlatform/google-cloud-dotnet/issues/1641. 2346 if testing.Short() && !replaying { 2347 t.Skip("Integration tests skipped in short mode") 2348 } 2349 2350 const ( 2351 // This is an uncompressed file. 2352 // See https://cloud.google.com/storage/docs/public-datasets/landsat 2353 uncompressedBucket = "gcp-public-data-landsat" 2354 uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt" 2355 2356 gzippedBucket = "storage-library-test-bucket" 2357 gzippedObject = "gzipped-text.txt" 2358 ) 2359 ctx := context.Background() 2360 client, err := newTestClient(ctx, option.WithoutAuthentication()) 2361 if err != nil { 2362 t.Fatal(err) 2363 } 2364 defer client.Close() 2365 2366 for _, test := range []struct { 2367 desc string 2368 obj *ObjectHandle 2369 offset, length int64 2370 readCompressed bool // don't decompress a gzipped file 2371 2372 wantErr bool 2373 wantCheck bool // Should Reader try to check the CRC? 2374 }{ 2375 { 2376 desc: "uncompressed, entire file", 2377 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2378 offset: 0, 2379 length: -1, 2380 readCompressed: false, 2381 wantCheck: true, 2382 }, 2383 { 2384 desc: "uncompressed, entire file, don't decompress", 2385 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2386 offset: 0, 2387 length: -1, 2388 readCompressed: true, 2389 wantCheck: true, 2390 }, 2391 { 2392 desc: "uncompressed, suffix", 2393 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2394 offset: 1, 2395 length: -1, 2396 readCompressed: false, 2397 wantCheck: false, 2398 }, 2399 { 2400 desc: "uncompressed, prefix", 2401 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 2402 offset: 0, 2403 length: 18, 2404 readCompressed: false, 2405 wantCheck: false, 2406 }, 2407 { 2408 // When a gzipped file is unzipped on read, we can't verify the checksum 2409 // because it was computed against the zipped contents. We can detect 2410 // this case using http.Response.Uncompressed. 2411 desc: "compressed, entire file, unzipped", 2412 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 2413 offset: 0, 2414 length: -1, 2415 readCompressed: false, 2416 wantCheck: false, 2417 }, 2418 { 2419 // When we read a gzipped file uncompressed, it's like reading a regular file: 2420 // the served content and the CRC match. 2421 desc: "compressed, entire file, read compressed", 2422 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 2423 offset: 0, 2424 length: -1, 2425 readCompressed: true, 2426 wantCheck: true, 2427 }, 2428 { 2429 desc: "compressed, partial, server unzips", 2430 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 2431 offset: 1, 2432 length: 8, 2433 readCompressed: false, 2434 wantErr: true, // GCS can't serve part of a gzipped object 2435 wantCheck: false, 2436 }, 2437 { 2438 desc: "compressed, partial, read compressed", 2439 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 2440 offset: 1, 2441 length: 8, 2442 readCompressed: true, 2443 wantCheck: false, 2444 }, 2445 } { 2446 obj := test.obj.ReadCompressed(test.readCompressed) 2447 r, err := obj.NewRangeReader(ctx, test.offset, test.length) 2448 if err != nil { 2449 if test.wantErr { 2450 continue 2451 } 2452 t.Fatalf("%s: %v", test.desc, err) 2453 } 2454 if got, want := r.checkCRC, test.wantCheck; got != want { 2455 t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want) 2456 } 2457 _, err = ioutil.ReadAll(r) 2458 _ = r.Close() 2459 if err != nil { 2460 t.Fatalf("%s: %v", test.desc, err) 2461 } 2462 } 2463} 2464 2465func TestIntegration_CancelWrite(t *testing.T) { 2466 // Verify that canceling the writer's context immediately stops uploading an object. 2467 ctx := context.Background() 2468 client := testConfig(ctx, t) 2469 defer client.Close() 2470 bkt := client.Bucket(bucketName) 2471 2472 cctx, cancel := context.WithCancel(ctx) 2473 defer cancel() 2474 obj := bkt.Object("cancel-write") 2475 w := obj.NewWriter(cctx) 2476 w.ChunkSize = googleapi.MinUploadChunkSize 2477 buf := make([]byte, w.ChunkSize) 2478 // Write the first chunk. This is read in its entirety before sending the request 2479 // (see google.golang.org/api/gensupport.PrepareUpload), so we expect it to return 2480 // without error. 2481 _, err := w.Write(buf) 2482 if err != nil { 2483 t.Fatal(err) 2484 } 2485 // Now cancel the context. 2486 cancel() 2487 // The next Write should return context.Canceled. 2488 _, err = w.Write(buf) 2489 if err != context.Canceled { 2490 t.Fatalf("got %v, wanted context.Canceled", err) 2491 } 2492 // The Close should too. 2493 err = w.Close() 2494 if err != context.Canceled { 2495 t.Fatalf("got %v, wanted context.Canceled", err) 2496 } 2497} 2498 2499func TestIntegration_UpdateCORS(t *testing.T) { 2500 ctx := context.Background() 2501 client := testConfig(ctx, t) 2502 defer client.Close() 2503 h := testHelper{t} 2504 2505 initialSettings := []CORS{ 2506 { 2507 MaxAge: time.Hour, 2508 Methods: []string{"POST"}, 2509 Origins: []string{"some-origin.com"}, 2510 ResponseHeaders: []string{"foo-bar"}, 2511 }, 2512 } 2513 2514 for _, test := range []struct { 2515 input []CORS 2516 want []CORS 2517 }{ 2518 { 2519 input: []CORS{ 2520 { 2521 MaxAge: time.Hour, 2522 Methods: []string{"GET"}, 2523 Origins: []string{"*"}, 2524 ResponseHeaders: []string{"some-header"}, 2525 }, 2526 }, 2527 want: []CORS{ 2528 { 2529 MaxAge: time.Hour, 2530 Methods: []string{"GET"}, 2531 Origins: []string{"*"}, 2532 ResponseHeaders: []string{"some-header"}, 2533 }, 2534 }, 2535 }, 2536 { 2537 input: []CORS{}, 2538 want: nil, 2539 }, 2540 { 2541 input: nil, 2542 want: []CORS{ 2543 { 2544 MaxAge: time.Hour, 2545 Methods: []string{"POST"}, 2546 Origins: []string{"some-origin.com"}, 2547 ResponseHeaders: []string{"foo-bar"}, 2548 }, 2549 }, 2550 }, 2551 } { 2552 bkt := client.Bucket(uidSpace.New()) 2553 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings}) 2554 defer h.mustDeleteBucket(bkt) 2555 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input}) 2556 attrs := h.mustBucketAttrs(bkt) 2557 if diff := testutil.Diff(attrs.CORS, test.want); diff != "" { 2558 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2559 } 2560 } 2561} 2562 2563func TestIntegration_UpdateDefaultEventBasedHold(t *testing.T) { 2564 ctx := context.Background() 2565 client := testConfig(ctx, t) 2566 defer client.Close() 2567 h := testHelper{t} 2568 2569 bkt := client.Bucket(uidSpace.New()) 2570 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2571 defer h.mustDeleteBucket(bkt) 2572 attrs := h.mustBucketAttrs(bkt) 2573 if attrs.DefaultEventBasedHold != false { 2574 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, false) 2575 } 2576 2577 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{DefaultEventBasedHold: true}) 2578 attrs = h.mustBucketAttrs(bkt) 2579 if attrs.DefaultEventBasedHold != true { 2580 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 2581 } 2582 2583 // Omitting it should leave the value unchanged. 2584 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RequesterPays: true}) 2585 attrs = h.mustBucketAttrs(bkt) 2586 if attrs.DefaultEventBasedHold != true { 2587 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 2588 } 2589} 2590 2591func TestIntegration_UpdateEventBasedHold(t *testing.T) { 2592 ctx := context.Background() 2593 client := testConfig(ctx, t) 2594 defer client.Close() 2595 h := testHelper{t} 2596 2597 bkt := client.Bucket(uidSpace.New()) 2598 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2599 obj := bkt.Object("some-obj") 2600 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2601 2602 defer func() { 2603 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: false}) 2604 h.mustDeleteObject(obj) 2605 h.mustDeleteBucket(bkt) 2606 }() 2607 2608 attrs := h.mustObjectAttrs(obj) 2609 if attrs.EventBasedHold != false { 2610 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, false) 2611 } 2612 2613 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: true}) 2614 attrs = h.mustObjectAttrs(obj) 2615 if attrs.EventBasedHold != true { 2616 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 2617 } 2618 2619 // Omitting it should leave the value unchanged. 2620 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}) 2621 attrs = h.mustObjectAttrs(obj) 2622 if attrs.EventBasedHold != true { 2623 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 2624 } 2625} 2626 2627func TestIntegration_UpdateTemporaryHold(t *testing.T) { 2628 ctx := context.Background() 2629 client := testConfig(ctx, t) 2630 defer client.Close() 2631 h := testHelper{t} 2632 2633 bkt := client.Bucket(uidSpace.New()) 2634 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 2635 obj := bkt.Object("some-obj") 2636 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2637 2638 defer func() { 2639 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: false}) 2640 h.mustDeleteObject(obj) 2641 h.mustDeleteBucket(bkt) 2642 }() 2643 2644 attrs := h.mustObjectAttrs(obj) 2645 if attrs.TemporaryHold != false { 2646 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, false) 2647 } 2648 2649 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: true}) 2650 attrs = h.mustObjectAttrs(obj) 2651 if attrs.TemporaryHold != true { 2652 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 2653 } 2654 2655 // Omitting it should leave the value unchanged. 2656 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}) 2657 attrs = h.mustObjectAttrs(obj) 2658 if attrs.TemporaryHold != true { 2659 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 2660 } 2661} 2662 2663func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) { 2664 ctx := context.Background() 2665 client := testConfig(ctx, t) 2666 defer client.Close() 2667 h := testHelper{t} 2668 2669 bkt := client.Bucket(uidSpace.New()) 2670 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 2671 obj := bkt.Object("some-obj") 2672 h.mustWrite(obj.NewWriter(ctx), randomContents()) 2673 2674 defer func() { 2675 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}) 2676 2677 // RetentionPeriod of less than a day is explicitly called out 2678 // as best effort and not guaranteed, so let's log problems deleting 2679 // objects instead of failing. 2680 if err := obj.Delete(context.Background()); err != nil { 2681 t.Logf("%s: object delete: %v", loc(), err) 2682 } 2683 if err := bkt.Delete(context.Background()); err != nil { 2684 t.Logf("%s: bucket delete: %v", loc(), err) 2685 } 2686 }() 2687 2688 attrs := h.mustObjectAttrs(obj) 2689 if attrs.RetentionExpirationTime == (time.Time{}) { 2690 t.Fatalf("got=%v, wanted a non-zero value", attrs.RetentionExpirationTime) 2691 } 2692} 2693 2694func TestIntegration_CustomTime(t *testing.T) { 2695 ctx := context.Background() 2696 client := testConfig(ctx, t) 2697 defer client.Close() 2698 h := testHelper{t} 2699 2700 // Create object with CustomTime. 2701 bkt := client.Bucket(bucketName) 2702 obj := bkt.Object("custom-time-obj") 2703 w := obj.NewWriter(ctx) 2704 ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC) 2705 w.ObjectAttrs.CustomTime = ct 2706 h.mustWrite(w, randomContents()) 2707 2708 // Validate that CustomTime has been set 2709 checkCustomTime := func(want time.Time) error { 2710 attrs, err := obj.Attrs(ctx) 2711 if err != nil { 2712 return fmt.Errorf("failed to get object attrs: %v", err) 2713 } 2714 if got := attrs.CustomTime; got != want { 2715 return fmt.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct) 2716 } 2717 return nil 2718 } 2719 2720 if err := checkCustomTime(ct); err != nil { 2721 t.Fatalf("checking CustomTime: %v", err) 2722 } 2723 2724 // Update CustomTime to the future should succeed. 2725 laterTime := ct.Add(10 * time.Hour) 2726 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: laterTime}); err != nil { 2727 t.Fatalf("updating CustomTime: %v", err) 2728 } 2729 2730 // Update CustomTime to the past should give error. 2731 earlierTime := ct.Add(5 * time.Hour) 2732 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: earlierTime}); err == nil { 2733 t.Fatalf("backdating CustomTime: expected error, got none") 2734 } 2735 2736 // Zero value for CustomTime should be ignored. 2737 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{}); err != nil { 2738 t.Fatalf("empty update: %v", err) 2739 } 2740 if err := checkCustomTime(laterTime); err != nil { 2741 t.Fatalf("after sending zero value: %v", err) 2742 } 2743} 2744 2745func TestIntegration_UpdateRetentionPolicy(t *testing.T) { 2746 ctx := context.Background() 2747 client := testConfig(ctx, t) 2748 defer client.Close() 2749 h := testHelper{t} 2750 2751 initial := &RetentionPolicy{RetentionPeriod: time.Minute} 2752 2753 for _, test := range []struct { 2754 input *RetentionPolicy 2755 want *RetentionPolicy 2756 }{ 2757 { // Update 2758 input: &RetentionPolicy{RetentionPeriod: time.Hour}, 2759 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2760 }, 2761 { // Update even with timestamp (EffectiveTime should be ignored) 2762 input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()}, 2763 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2764 }, 2765 { // Remove 2766 input: &RetentionPolicy{}, 2767 want: nil, 2768 }, 2769 { // Remove even with timestamp (EffectiveTime should be ignored) 2770 input: &RetentionPolicy{EffectiveTime: time.Now()}, 2771 want: nil, 2772 }, 2773 { // Ignore 2774 input: nil, 2775 want: initial, 2776 }, 2777 } { 2778 bkt := client.Bucket(uidSpace.New()) 2779 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial}) 2780 defer h.mustDeleteBucket(bkt) 2781 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input}) 2782 attrs := h.mustBucketAttrs(bkt) 2783 if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 { 2784 // Should be set by the server and parsed by the client 2785 t.Fatal("EffectiveTime should be set, but it was not") 2786 } 2787 if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" { 2788 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2789 } 2790 } 2791} 2792 2793func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) { 2794 ctx := context.Background() 2795 client := testConfig(ctx, t) 2796 defer client.Close() 2797 h := testHelper{t} 2798 2799 bkt := client.Bucket(uidSpace.New()) 2800 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}}) 2801 2802 oh := bkt.Object("some-object") 2803 if err := writeObject(ctx, oh, "text/plain", []byte("hello world")); err != nil { 2804 t.Fatal(err) 2805 } 2806 2807 if err := oh.Delete(ctx); err == nil { 2808 t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil") 2809 } 2810 2811 // Remove the retention period 2812 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{}}) 2813 // Deleting with retry, as bucket metadata changes 2814 // can take some time to propagate. 2815 err := retry(ctx, func() error { 2816 return oh.Delete(ctx) 2817 }, nil) 2818 if err != nil { 2819 h.t.Fatalf("%s: object delete: %v", loc(), err) 2820 } 2821 h.mustDeleteBucket(bkt) 2822} 2823 2824func TestIntegration_LockBucket(t *testing.T) { 2825 ctx := context.Background() 2826 client := testConfig(ctx, t) 2827 defer client.Close() 2828 h := testHelper{t} 2829 2830 bkt := client.Bucket(uidSpace.New()) 2831 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}}) 2832 attrs := h.mustBucketAttrs(bkt) 2833 if attrs.RetentionPolicy.IsLocked { 2834 t.Fatal("Expected bucket to begin unlocked, but it was not") 2835 } 2836 err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx) 2837 if err != nil { 2838 t.Fatal("could not lock", err) 2839 } 2840 2841 attrs = h.mustBucketAttrs(bkt) 2842 if !attrs.RetentionPolicy.IsLocked { 2843 t.Fatal("Expected bucket to be locked, but it was not") 2844 } 2845 2846 _, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 2847 if err == nil { 2848 t.Fatal("Expected error updating locked bucket, got nil") 2849 } 2850} 2851 2852func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) { 2853 ctx := context.Background() 2854 client := testConfig(ctx, t) 2855 defer client.Close() 2856 h := testHelper{t} 2857 2858 bkt := client.Bucket(uidSpace.New()) 2859 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2860 RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}, 2861 }) 2862 err := bkt.LockRetentionPolicy(ctx) 2863 if err == nil { 2864 t.Fatal("expected error locking bucket without metageneration condition, got nil") 2865 } 2866} 2867 2868func TestIntegration_KMS(t *testing.T) { 2869 ctx := context.Background() 2870 client := testConfig(ctx, t) 2871 defer client.Close() 2872 h := testHelper{t} 2873 2874 keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING") 2875 if keyRingName == "" { 2876 t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details") 2877 } 2878 keyName1 := keyRingName + "/cryptoKeys/key1" 2879 keyName2 := keyRingName + "/cryptoKeys/key2" 2880 contents := []byte("my secret") 2881 2882 write := func(obj *ObjectHandle, setKey bool) { 2883 w := obj.NewWriter(ctx) 2884 if setKey { 2885 w.KMSKeyName = keyName1 2886 } 2887 h.mustWrite(w, contents) 2888 } 2889 2890 checkRead := func(obj *ObjectHandle) { 2891 got := h.mustRead(obj) 2892 if !bytes.Equal(got, contents) { 2893 t.Errorf("got %v, want %v", got, contents) 2894 } 2895 attrs := h.mustObjectAttrs(obj) 2896 if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 { 2897 t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1) 2898 } 2899 } 2900 2901 // Write an object with a key, then read it to verify its contents and the presence of the key name. 2902 bkt := client.Bucket(bucketName) 2903 obj := bkt.Object("kms") 2904 write(obj, true) 2905 checkRead(obj) 2906 h.mustDeleteObject(obj) 2907 2908 // Encrypt an object with a CSEK, then copy it using a CMEK. 2909 src := bkt.Object("csek").Key(testEncryptionKey) 2910 if err := writeObject(ctx, src, "text/plain", contents); err != nil { 2911 t.Fatal(err) 2912 } 2913 dest := bkt.Object("cmek") 2914 c := dest.CopierFrom(src) 2915 c.DestinationKMSKeyName = keyName1 2916 if _, err := c.Run(ctx); err != nil { 2917 t.Fatal(err) 2918 } 2919 checkRead(dest) 2920 src.Delete(ctx) 2921 dest.Delete(ctx) 2922 2923 // Create a bucket with a default key, then write and read an object. 2924 bkt = client.Bucket(uidSpace.New()) 2925 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2926 Location: "US", 2927 Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1}, 2928 }) 2929 defer h.mustDeleteBucket(bkt) 2930 2931 attrs := h.mustBucketAttrs(bkt) 2932 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want { 2933 t.Fatalf("got %q, want %q", got, want) 2934 } 2935 obj = bkt.Object("kms") 2936 write(obj, false) 2937 checkRead(obj) 2938 h.mustDeleteObject(obj) 2939 2940 // Update the bucket's default key to a different name. 2941 // (This key doesn't have to exist.) 2942 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}) 2943 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 2944 t.Fatalf("got %q, want %q", got, want) 2945 } 2946 attrs = h.mustBucketAttrs(bkt) 2947 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 2948 t.Fatalf("got %q, want %q", got, want) 2949 } 2950 2951 // Remove the default KMS key. 2952 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}) 2953 if attrs.Encryption != nil { 2954 t.Fatalf("got %#v, want nil", attrs.Encryption) 2955 } 2956} 2957 2958func TestIntegration_PredefinedACLs(t *testing.T) { 2959 check := func(msg string, rs []ACLRule, i int, wantEntity ACLEntity, wantRole ACLRole) { 2960 if i >= len(rs) { 2961 t.Errorf("%s: no rule at index %d", msg, i) 2962 return 2963 } 2964 got := rs[i] 2965 if got.Entity != wantEntity || got.Role != wantRole { 2966 t.Errorf("%s[%d]: got %+v, want Entity %s and Role %s", 2967 msg, i, got, wantEntity, wantRole) 2968 } 2969 } 2970 checkPrefix := func(msg string, rs []ACLRule, i int, wantPrefix string, wantRole ACLRole) { 2971 if i >= len(rs) { 2972 t.Errorf("%s: no rule at index %d", msg, i) 2973 return 2974 } 2975 got := rs[i] 2976 if !strings.HasPrefix(string(got.Entity), wantPrefix) || got.Role != wantRole { 2977 t.Errorf("%s[%d]: got %+v, want Entity %s... and Role %s", 2978 msg, i, got, wantPrefix, wantRole) 2979 } 2980 } 2981 2982 ctx := context.Background() 2983 client := testConfig(ctx, t) 2984 defer client.Close() 2985 h := testHelper{t} 2986 2987 bkt := client.Bucket(uidSpace.New()) 2988 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2989 PredefinedACL: "authenticatedRead", 2990 PredefinedDefaultObjectACL: "publicRead", 2991 }) 2992 defer h.mustDeleteBucket(bkt) 2993 attrs := h.mustBucketAttrs(bkt) 2994 checkPrefix("Bucket.ACL", attrs.ACL, 0, "project-owners", RoleOwner) 2995 check("Bucket.ACL", attrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 2996 check("DefaultObjectACL", attrs.DefaultObjectACL, 0, AllUsers, RoleReader) 2997 2998 // Bucket update 2999 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{ 3000 PredefinedACL: "private", 3001 PredefinedDefaultObjectACL: "authenticatedRead", 3002 }) 3003 checkPrefix("Bucket.ACL update", attrs.ACL, 0, "project-owners", RoleOwner) 3004 check("DefaultObjectACL update", attrs.DefaultObjectACL, 0, AllAuthenticatedUsers, RoleReader) 3005 3006 // Object creation 3007 obj := bkt.Object("private") 3008 w := obj.NewWriter(ctx) 3009 w.PredefinedACL = "authenticatedRead" 3010 h.mustWrite(w, []byte("hello")) 3011 defer h.mustDeleteObject(obj) 3012 checkPrefix("Object.ACL", w.Attrs().ACL, 0, "user", RoleOwner) 3013 check("Object.ACL", w.Attrs().ACL, 1, AllAuthenticatedUsers, RoleReader) 3014 3015 // Object update 3016 oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}) 3017 checkPrefix("Object.ACL update", oattrs.ACL, 0, "user", RoleOwner) 3018 if got := len(oattrs.ACL); got != 1 { 3019 t.Errorf("got %d ACLs, want 1", got) 3020 } 3021 3022 // Copy 3023 dst := bkt.Object("dst") 3024 copier := dst.CopierFrom(obj) 3025 copier.PredefinedACL = "publicRead" 3026 oattrs, err := copier.Run(ctx) 3027 if err != nil { 3028 t.Fatal(err) 3029 } 3030 defer h.mustDeleteObject(dst) 3031 // The copied object still retains the "private" ACL of the source object. 3032 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 3033 check("Copy dest", oattrs.ACL, 1, AllUsers, RoleReader) 3034 3035 // Compose 3036 comp := bkt.Object("comp") 3037 composer := comp.ComposerFrom(obj, dst) 3038 composer.PredefinedACL = "authenticatedRead" 3039 oattrs, err = composer.Run(ctx) 3040 if err != nil { 3041 t.Fatal(err) 3042 } 3043 defer h.mustDeleteObject(comp) 3044 // The composed object still retains the "private" ACL. 3045 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 3046 check("Copy dest", oattrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 3047} 3048 3049func TestIntegration_ServiceAccount(t *testing.T) { 3050 ctx := context.Background() 3051 client := testConfig(ctx, t) 3052 defer client.Close() 3053 3054 s, err := client.ServiceAccount(ctx, testutil.ProjID()) 3055 if err != nil { 3056 t.Fatal(err) 3057 } 3058 want := "@gs-project-accounts.iam.gserviceaccount.com" 3059 if !strings.Contains(s, want) { 3060 t.Fatalf("got %v, want to contain %v", s, want) 3061 } 3062} 3063 3064func TestIntegration_ReaderAttrs(t *testing.T) { 3065 ctx := context.Background() 3066 client := testConfig(ctx, t) 3067 defer client.Close() 3068 bkt := client.Bucket(bucketName) 3069 3070 const defaultType = "text/plain" 3071 obj := "some-object" 3072 c := randomContents() 3073 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 3074 t.Errorf("Write for %v failed with %v", obj, err) 3075 } 3076 oh := bkt.Object(obj) 3077 3078 rc, err := oh.NewReader(ctx) 3079 if err != nil { 3080 t.Fatal(err) 3081 } 3082 3083 attrs, err := oh.Attrs(ctx) 3084 if err != nil { 3085 t.Fatal(err) 3086 } 3087 3088 got := rc.Attrs 3089 want := ReaderObjectAttrs{ 3090 Size: attrs.Size, 3091 ContentType: attrs.ContentType, 3092 ContentEncoding: attrs.ContentEncoding, 3093 CacheControl: attrs.CacheControl, 3094 LastModified: got.LastModified, // ignored, tested separately 3095 Generation: attrs.Generation, 3096 Metageneration: attrs.Metageneration, 3097 } 3098 if got != want { 3099 t.Fatalf("got %v, wanted %v", got, want) 3100 } 3101 3102 if got.LastModified.IsZero() { 3103 t.Fatal("LastModified is 0, should be >0") 3104 } 3105} 3106 3107// Ensures that a file stored with a: 3108// * Content-Encoding of "gzip" 3109// * Content-Type of "text/plain" 3110// will be properly served back. 3111// See: 3112// * https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip 3113// * https://github.com/googleapis/google-cloud-go/issues/1800 3114func TestIntegration_NewReaderWithContentEncodingGzip(t *testing.T) { 3115 ctx := context.Background() 3116 client := testConfig(ctx, t) 3117 defer client.Close() 3118 3119 h := testHelper{t} 3120 3121 projectID := testutil.ProjID() 3122 bkt := client.Bucket(uidSpace.New()) 3123 h.mustCreate(bkt, projectID, nil) 3124 defer h.mustDeleteBucket(bkt) 3125 obj := bkt.Object("decompressive-transcoding") 3126 original := bytes.Repeat([]byte("a"), 4<<10) 3127 3128 // Wrap the file upload in a retry. 3129 // TODO: Investigate removing retry after resolving 3130 // https://github.com/googleapis/google-api-go-client/issues/392. 3131 err := retry(ctx, func() error { 3132 // Firstly upload the gzip compressed file. 3133 w := obj.NewWriter(ctx) 3134 // Compress and upload the content. 3135 gzw := gzip.NewWriter(w) 3136 if _, err := gzw.Write(original); err != nil { 3137 return fmt.Errorf("Failed to compress content: %v", err) 3138 } 3139 if err := gzw.Close(); err != nil { 3140 return fmt.Errorf("Failed to compress content: %v", err) 3141 } 3142 if err := w.Close(); err != nil { 3143 return fmt.Errorf("Failed to finish uploading the file: %v", err) 3144 } 3145 return nil 3146 }, 3147 nil) 3148 3149 defer h.mustDeleteObject(obj) 3150 3151 // Now update the Content-Encoding and Content-Type to enable 3152 // decompressive transcoding. 3153 updatedAttrs, err := obj.Update(ctx, ObjectAttrsToUpdate{ 3154 ContentEncoding: "gzip", 3155 ContentType: "text/plain", 3156 }) 3157 if err != nil { 3158 t.Fatalf("Attribute update failure: %v", err) 3159 } 3160 if g, w := updatedAttrs.ContentEncoding, "gzip"; g != w { 3161 t.Fatalf("ContentEncoding mismtach:\nGot: %q\nWant: %q", g, w) 3162 } 3163 if g, w := updatedAttrs.ContentType, "text/plain"; g != w { 3164 t.Fatalf("ContentType mismtach:\nGot: %q\nWant: %q", g, w) 3165 } 3166 3167 rWhole, err := obj.NewReader(ctx) 3168 if err != nil { 3169 t.Fatalf("Failed to create wholesome reader: %v", err) 3170 } 3171 blobWhole, err := ioutil.ReadAll(rWhole) 3172 rWhole.Close() 3173 if err != nil { 3174 t.Fatalf("Failed to read the whole body: %v", err) 3175 } 3176 if g, w := blobWhole, original; !bytes.Equal(g, w) { 3177 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 3178 } 3179 3180 // Now try a range read, which should return the whole body anyways since 3181 // for decompressive transcoding, range requests ARE IGNORED by Cloud Storage. 3182 r2kBTo3kB, err := obj.NewRangeReader(ctx, 2<<10, 3<<10) 3183 if err != nil { 3184 t.Fatalf("Failed to create range reader: %v", err) 3185 } 3186 blob2kBTo3kB, err := ioutil.ReadAll(r2kBTo3kB) 3187 r2kBTo3kB.Close() 3188 if err != nil { 3189 t.Fatalf("Failed to read with the 2kB to 3kB range request: %v", err) 3190 } 3191 // The ENTIRE body MUST be served back regardless of the requested range. 3192 if g, w := blob2kBTo3kB, original; !bytes.Equal(g, w) { 3193 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 3194 } 3195} 3196 3197func TestIntegration_HMACKey(t *testing.T) { 3198 ctx := context.Background() 3199 client := testConfig(ctx, t) 3200 defer client.Close() 3201 3202 projectID := testutil.ProjID() 3203 3204 // Use the service account email from the user's credentials. Requires that the 3205 // credentials are set via a JSON credentials file. 3206 // Note that a service account may only have up to 5 active HMAC keys at once; if 3207 // we see flakes because of this, we should consider switching to using a project 3208 // pool. 3209 credentials := testutil.CredentialsEnv(ctx, "GCLOUD_TESTS_GOLANG_KEY") 3210 if credentials == nil { 3211 t.Fatal("credentials could not be determined, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 3212 } 3213 if credentials.JSON == nil { 3214 t.Fatal("could not read the JSON key file, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 3215 } 3216 conf, err := google.JWTConfigFromJSON(credentials.JSON) 3217 if err != nil { 3218 t.Fatal(err) 3219 } 3220 serviceAccountEmail := conf.Email 3221 3222 hmacKey, err := client.CreateHMACKey(ctx, projectID, serviceAccountEmail) 3223 if err != nil { 3224 t.Fatalf("Failed to create HMACKey: %v", err) 3225 } 3226 if hmacKey == nil { 3227 t.Fatal("Unexpectedly got back a nil HMAC key") 3228 } 3229 3230 if hmacKey.State != Active { 3231 t.Fatalf("Unexpected state %q, expected %q", hmacKey.State, Active) 3232 } 3233 3234 hkh := client.HMACKeyHandle(projectID, hmacKey.AccessID) 3235 // 1. Ensure that we CANNOT delete an ACTIVE key. 3236 if err := hkh.Delete(ctx); err == nil { 3237 t.Fatal("Unexpectedly deleted key whose state is ACTIVE: No error from Delete.") 3238 } 3239 3240 invalidStates := []HMACState{"", Deleted, "active", "inactive", "foo_bar"} 3241 for _, invalidState := range invalidStates { 3242 t.Run("invalid-"+string(invalidState), func(t *testing.T) { 3243 _, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3244 State: invalidState, 3245 }) 3246 if err == nil { 3247 t.Fatal("Unexpectedly succeeded") 3248 } 3249 invalidStateMsg := fmt.Sprintf(`storage: invalid state %q for update, must be either "ACTIVE" or "INACTIVE"`, invalidState) 3250 if err.Error() != invalidStateMsg { 3251 t.Fatalf("Mismatched error: got: %q\nwant: %q", err, invalidStateMsg) 3252 } 3253 }) 3254 } 3255 3256 // 2.1. Setting the State to Inactive should succeed. 3257 hu, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3258 State: Inactive, 3259 }) 3260 if err != nil { 3261 t.Fatalf("Unexpected Update failure: %v", err) 3262 } 3263 if got, want := hu.State, Inactive; got != want { 3264 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 3265 } 3266 3267 // 2.2. Setting the State back to Active should succeed. 3268 hu, err = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3269 State: Active, 3270 }) 3271 if err != nil { 3272 t.Fatalf("Unexpected Update failure: %v", err) 3273 } 3274 if got, want := hu.State, Active; got != want { 3275 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 3276 } 3277 3278 // 3. Verify that keys are listed as expected. 3279 iter := client.ListHMACKeys(ctx, projectID) 3280 count := 0 3281 for ; ; count++ { 3282 _, err := iter.Next() 3283 if err == iterator.Done { 3284 break 3285 } 3286 if err != nil { 3287 t.Fatalf("Failed to ListHMACKeys: %v", err) 3288 } 3289 } 3290 if count == 0 { 3291 t.Fatal("Failed to list any HMACKeys") 3292 } 3293 3294 // 4. Finally set it to back to Inactive and 3295 // then retry the deletion which should now succeed. 3296 _, _ = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 3297 State: Inactive, 3298 }) 3299 if err := hkh.Delete(ctx); err != nil { 3300 t.Fatalf("Unexpected deletion failure: %v", err) 3301 } 3302 3303 _, err = hkh.Get(ctx) 3304 if err != nil && !strings.Contains(err.Error(), "404") { 3305 // If the deleted key has already been garbage collected, a 404 is expected. 3306 // Other errors should cause a failure and are not expected. 3307 t.Fatalf("Unexpected error: %v", err) 3308 } 3309} 3310 3311func TestIntegration_PostPolicyV4(t *testing.T) { 3312 jwtConf, err := testutil.JWTConfig() 3313 if err != nil { 3314 t.Fatal(err) 3315 } 3316 if jwtConf == nil { 3317 t.Skip("JSON key file is not present") 3318 } 3319 3320 ctx := context.Background() 3321 client := testConfig(ctx, t) 3322 defer client.Close() 3323 3324 projectID := testutil.ProjID() 3325 newBucketName := uidSpace.New() 3326 b := client.Bucket(newBucketName) 3327 h := testHelper{t} 3328 h.mustCreate(b, projectID, nil) 3329 defer h.mustDeleteBucket(b) 3330 3331 statusCodeToRespond := 200 3332 opts := &PostPolicyV4Options{ 3333 GoogleAccessID: jwtConf.Email, 3334 PrivateKey: jwtConf.PrivateKey, 3335 3336 Expires: time.Now().Add(30 * time.Minute), 3337 3338 Fields: &PolicyV4Fields{ 3339 StatusCodeOnSuccess: statusCodeToRespond, 3340 ContentType: "text/plain", 3341 ACL: "public-read", 3342 }, 3343 3344 // The conditions that the uploaded file will be expected to conform to. 3345 Conditions: []PostPolicyV4Condition{ 3346 // Make the file a maximum of 10mB. 3347 ConditionContentLengthRange(0, 10<<20), 3348 ConditionStartsWith("$acl", "public"), 3349 }, 3350 } 3351 3352 objectName := "my-object.txt" 3353 pv4, err := GenerateSignedPostPolicyV4(newBucketName, objectName, opts) 3354 if err != nil { 3355 t.Fatal(err) 3356 } 3357 3358 formBuf := new(bytes.Buffer) 3359 mw := multipart.NewWriter(formBuf) 3360 for fieldName, value := range pv4.Fields { 3361 if err := mw.WriteField(fieldName, value); err != nil { 3362 t.Fatalf("Failed to write form field: %q: %v", fieldName, err) 3363 } 3364 } 3365 3366 // Now let's perform the upload. 3367 fileBody := bytes.Repeat([]byte("a"), 25) 3368 mf, err := mw.CreateFormFile("file", "myfile.txt") 3369 if err != nil { 3370 t.Fatal(err) 3371 } 3372 if _, err := mf.Write(fileBody); err != nil { 3373 t.Fatal(err) 3374 } 3375 if err := mw.Close(); err != nil { 3376 t.Fatal(err) 3377 } 3378 3379 // Compose the HTTP request. 3380 req, err := http.NewRequest("POST", pv4.URL, formBuf) 3381 if err != nil { 3382 t.Fatalf("Failed to compose HTTP request: %v", err) 3383 } 3384 // Ensure the Content-Type is derived from the writer. 3385 req.Header.Set("Content-Type", mw.FormDataContentType()) 3386 res, err := http.DefaultClient.Do(req) 3387 if err != nil { 3388 t.Fatal(err) 3389 } 3390 if g, w := res.StatusCode, statusCodeToRespond; g != w { 3391 blob, _ := httputil.DumpResponse(res, true) 3392 t.Fatalf("Status code in response mismatch: got %d want %d\nBody: %s", g, w, blob) 3393 } 3394 io.Copy(ioutil.Discard, res.Body) 3395 3396 // Verify that the file was properly uploaded, by 3397 // reading back its attributes and content! 3398 obj := b.Object(objectName) 3399 defer h.mustDeleteObject(obj) 3400 3401 attrs, err := obj.Attrs(ctx) 3402 if err != nil { 3403 t.Fatalf("Failed to retrieve attributes: %v", err) 3404 } 3405 if g, w := attrs.Size, int64(len(fileBody)); g != w { 3406 t.Errorf("ContentLength mismatch: got %d want %d", g, w) 3407 } 3408 if g, w := attrs.MD5, md5.Sum(fileBody); !bytes.Equal(g, w[:]) { 3409 t.Errorf("MD5Checksum mismatch\nGot: %x\nWant: %x", g, w) 3410 } 3411 3412 // Compare the uploaded body with the expected. 3413 rd, err := obj.NewReader(ctx) 3414 if err != nil { 3415 t.Fatalf("Failed to create a reader: %v", err) 3416 } 3417 gotBody, err := ioutil.ReadAll(rd) 3418 if err != nil { 3419 t.Fatalf("Failed to read the body: %v", err) 3420 } 3421 if diff := testutil.Diff(string(gotBody), string(fileBody)); diff != "" { 3422 t.Fatalf("Body mismatch: got - want +\n%s", diff) 3423 } 3424} 3425 3426// Verify that custom scopes passed in by the user are applied correctly. 3427func TestIntegration_Scopes(t *testing.T) { 3428 // A default client should be able to write objects since it has scope of 3429 // FullControl 3430 ctx := context.Background() 3431 clientFullControl := testConfig(ctx, t) 3432 defer clientFullControl.Close() 3433 3434 bkt := clientFullControl.Bucket(bucketName) 3435 obj := "FakeObj1" 3436 contents := []byte("This object should be written successfully\n") 3437 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 3438 t.Fatalf("writing: %v", err) 3439 } 3440 3441 // A client with ReadOnly scope should not be able to write successfully. 3442 clientReadOnly, err := NewClient(ctx, option.WithScopes(ScopeReadOnly)) 3443 defer clientReadOnly.Close() 3444 if err != nil { 3445 t.Fatalf("error creating client: %v", err) 3446 } 3447 3448 bkt = clientReadOnly.Bucket(bucketName) 3449 obj = "FakeObj2" 3450 contents = []byte("This object should not be written.\n") 3451 3452 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err == nil { 3453 t.Fatal("client with ScopeReadOnly was able to write an object unexpectedly.") 3454 } 3455 3456} 3457 3458type testHelper struct { 3459 t *testing.T 3460} 3461 3462func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) { 3463 if err := b.Create(context.Background(), projID, attrs); err != nil { 3464 h.t.Fatalf("%s: bucket create: %v", loc(), err) 3465 } 3466} 3467 3468func (h testHelper) mustDeleteBucket(b *BucketHandle) { 3469 if err := b.Delete(context.Background()); err != nil { 3470 h.t.Fatalf("%s: bucket delete: %v", loc(), err) 3471 } 3472} 3473 3474func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs { 3475 attrs, err := b.Attrs(context.Background()) 3476 if err != nil { 3477 h.t.Fatalf("%s: bucket attrs: %v", loc(), err) 3478 } 3479 return attrs 3480} 3481 3482func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate) *BucketAttrs { 3483 attrs, err := b.Update(context.Background(), ua) 3484 if err != nil { 3485 h.t.Fatalf("%s: update: %v", loc(), err) 3486 } 3487 return attrs 3488} 3489 3490func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs { 3491 attrs, err := o.Attrs(context.Background()) 3492 if err != nil { 3493 h.t.Fatalf("%s: object attrs: %v", loc(), err) 3494 } 3495 return attrs 3496} 3497 3498func (h testHelper) mustDeleteObject(o *ObjectHandle) { 3499 if err := o.Delete(context.Background()); err != nil { 3500 h.t.Fatalf("%s: object delete: %v", loc(), err) 3501 } 3502} 3503 3504func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate) *ObjectAttrs { 3505 attrs, err := o.Update(context.Background(), ua) 3506 if err != nil { 3507 h.t.Fatalf("%s: update: %v", loc(), err) 3508 } 3509 return attrs 3510} 3511 3512func (h testHelper) mustWrite(w *Writer, data []byte) { 3513 if _, err := w.Write(data); err != nil { 3514 w.Close() 3515 h.t.Fatalf("%s: write: %v", loc(), err) 3516 } 3517 if err := w.Close(); err != nil { 3518 h.t.Fatalf("%s: close write: %v", loc(), err) 3519 } 3520} 3521 3522func (h testHelper) mustRead(obj *ObjectHandle) []byte { 3523 data, err := readObject(context.Background(), obj) 3524 if err != nil { 3525 h.t.Fatalf("%s: read: %v", loc(), err) 3526 } 3527 return data 3528} 3529 3530func (h testHelper) mustNewReader(obj *ObjectHandle) *Reader { 3531 r, err := obj.NewReader(context.Background()) 3532 if err != nil { 3533 h.t.Fatalf("%s: new reader: %v", loc(), err) 3534 } 3535 return r 3536} 3537 3538func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error { 3539 w := obj.NewWriter(ctx) 3540 w.ContentType = contentType 3541 w.CacheControl = "public, max-age=60" 3542 if contents != nil { 3543 if _, err := w.Write(contents); err != nil { 3544 _ = w.Close() 3545 return err 3546 } 3547 } 3548 return w.Close() 3549} 3550 3551// loc returns a string describing the file and line of its caller's call site. In 3552// other words, if a test function calls a helper, and the helper calls loc, then the 3553// string will refer to the line on which the test function called the helper. 3554// TODO(jba): use t.Helper once we drop go 1.6. 3555func loc() string { 3556 _, file, line, ok := runtime.Caller(2) 3557 if !ok { 3558 return "???" 3559 } 3560 return fmt.Sprintf("%s:%d", filepath.Base(file), line) 3561} 3562 3563func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) { 3564 r, err := obj.NewReader(ctx) 3565 if err != nil { 3566 return nil, err 3567 } 3568 defer r.Close() 3569 return ioutil.ReadAll(r) 3570} 3571 3572// cleanupBuckets deletes the bucket used for testing, as well as old 3573// testing buckets that weren't cleaned previously. 3574func cleanupBuckets() error { 3575 if testing.Short() { 3576 return nil // Don't clean up in short mode. 3577 } 3578 ctx := context.Background() 3579 client := config(ctx) 3580 if client == nil { 3581 return nil // Don't cleanup if we're not configured correctly. 3582 } 3583 defer client.Close() 3584 if err := killBucket(ctx, client, bucketName); err != nil { 3585 return err 3586 } 3587 3588 // Delete buckets whose name begins with our test prefix, and which were 3589 // created a while ago. (Unfortunately GCS doesn't provide last-modified 3590 // time, which would be a better way to check for staleness.) 3591 const expireAge = 24 * time.Hour 3592 projectID := testutil.ProjID() 3593 it := client.Buckets(ctx, projectID) 3594 it.Prefix = testPrefix 3595 for { 3596 bktAttrs, err := it.Next() 3597 if err == iterator.Done { 3598 break 3599 } 3600 if err != nil { 3601 return err 3602 } 3603 if time.Since(bktAttrs.Created) > expireAge { 3604 log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge) 3605 if err := killBucket(ctx, client, bktAttrs.Name); err != nil { 3606 return err 3607 } 3608 } 3609 } 3610 return nil 3611} 3612 3613// killBucket deletes a bucket and all its objects. 3614func killBucket(ctx context.Context, client *Client, bucketName string) error { 3615 bkt := client.Bucket(bucketName) 3616 // Bucket must be empty to delete. 3617 it := bkt.Objects(ctx, nil) 3618 for { 3619 objAttrs, err := it.Next() 3620 if err == iterator.Done { 3621 break 3622 } 3623 if err != nil { 3624 return err 3625 } 3626 // Objects with a hold must have the hold released. 3627 if objAttrs.EventBasedHold || objAttrs.TemporaryHold { 3628 obj := bkt.Object(objAttrs.Name) 3629 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{EventBasedHold: false, TemporaryHold: false}); err != nil { 3630 return fmt.Errorf("removing hold from %q: %v", bucketName+"/"+objAttrs.Name, err) 3631 } 3632 } 3633 if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil { 3634 return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err) 3635 } 3636 } 3637 // GCS is eventually consistent, so this delete may fail because the 3638 // replica still sees an object in the bucket. We log the error and expect 3639 // a later test run to delete the bucket. 3640 if err := bkt.Delete(ctx); err != nil { 3641 log.Printf("deleting %q: %v", bucketName, err) 3642 } 3643 return nil 3644} 3645 3646func randomContents() []byte { 3647 h := md5.New() 3648 io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000))) 3649 return h.Sum(nil) 3650} 3651 3652type zeros struct{} 3653 3654func (zeros) Read(p []byte) (int, error) { return len(p), nil } 3655 3656// Make a GET request to a URL using an unauthenticated client, and return its contents. 3657func getURL(url string, headers map[string][]string) ([]byte, error) { 3658 req, err := http.NewRequest("GET", url, nil) 3659 if err != nil { 3660 return nil, err 3661 } 3662 req.Header = headers 3663 res, err := http.DefaultClient.Do(req) 3664 if err != nil { 3665 return nil, err 3666 } 3667 defer res.Body.Close() 3668 bytes, err := ioutil.ReadAll(res.Body) 3669 if err != nil { 3670 return nil, err 3671 } 3672 if res.StatusCode != 200 { 3673 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 3674 } 3675 return bytes, nil 3676} 3677 3678// Make a PUT request to a URL using an unauthenticated client, and return its contents. 3679func putURL(url string, headers map[string][]string, payload io.Reader) ([]byte, error) { 3680 req, err := http.NewRequest("PUT", url, payload) 3681 if err != nil { 3682 return nil, err 3683 } 3684 req.Header = headers 3685 res, err := http.DefaultClient.Do(req) 3686 if err != nil { 3687 return nil, err 3688 } 3689 defer res.Body.Close() 3690 bytes, err := ioutil.ReadAll(res.Body) 3691 if err != nil { 3692 return nil, err 3693 } 3694 if res.StatusCode != 200 { 3695 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 3696 } 3697 return bytes, nil 3698} 3699 3700func namesEqual(obj *ObjectAttrs, bucketName, objectName string) bool { 3701 return obj.Bucket == bucketName && obj.Name == objectName 3702} 3703 3704func keyFileEmail(filename string) (string, error) { 3705 bytes, err := ioutil.ReadFile(filename) 3706 if err != nil { 3707 return "", err 3708 } 3709 var v struct { 3710 ClientEmail string `json:"client_email"` 3711 } 3712 if err := json.Unmarshal(bytes, &v); err != nil { 3713 return "", err 3714 } 3715 return v.ClientEmail, nil 3716} 3717 3718func containsACL(acls []ACLRule, e ACLEntity, r ACLRole) bool { 3719 for _, a := range acls { 3720 if a.Entity == e && a.Role == r { 3721 return true 3722 } 3723 } 3724 return false 3725} 3726 3727func hasRule(acl []ACLRule, rule ACLRule) bool { 3728 for _, r := range acl { 3729 if cmp.Equal(r, rule) { 3730 return true 3731 } 3732 } 3733 return false 3734} 3735 3736// retry retries a function call as well as an (optional) correctness check for up 3737// to 11 seconds. Both call and check must run without error in order to succeed. 3738// If the timeout is hit, the most recent error from call or check will be returned. 3739// This function should be used to wrap calls that might cause integration test 3740// flakes due to delays in propagation (for example, metadata updates). 3741func retry(ctx context.Context, call func() error, check func() error) error { 3742 timeout := time.After(11 * time.Second) 3743 var err error 3744 for { 3745 select { 3746 case <-timeout: 3747 return err 3748 default: 3749 } 3750 err = call() 3751 if err == nil { 3752 if check == nil || check() == nil { 3753 return nil 3754 } 3755 err = check() 3756 } 3757 time.Sleep(200 * time.Millisecond) 3758 } 3759} 3760