1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "bytes" 19 "compress/gzip" 20 "crypto/md5" 21 "crypto/sha256" 22 "encoding/base64" 23 "encoding/json" 24 "flag" 25 "fmt" 26 "hash/crc32" 27 "io" 28 "io/ioutil" 29 "log" 30 "math/rand" 31 "net/http" 32 "os" 33 "path/filepath" 34 "runtime" 35 "sort" 36 "strconv" 37 "strings" 38 "testing" 39 "time" 40 41 "github.com/google/go-cmp/cmp" 42 "github.com/google/go-cmp/cmp/cmpopts" 43 "golang.org/x/net/context" 44 45 "cloud.google.com/go/httpreplay" 46 "cloud.google.com/go/iam" 47 "cloud.google.com/go/internal/testutil" 48 "cloud.google.com/go/internal/uid" 49 "google.golang.org/api/googleapi" 50 "google.golang.org/api/iterator" 51 itesting "google.golang.org/api/iterator/testing" 52 "google.golang.org/api/option" 53) 54 55const ( 56 testPrefix = "go-integration-test" 57 replayFilename = "storage.replay" 58) 59 60var ( 61 record = flag.Bool("record", false, "record RPCs") 62 63 uidSpace *uid.Space 64 bucketName string 65 // Use our own random number generator to isolate the sequence of random numbers from 66 // other packages. This makes it possible to use HTTP replay and draw the same sequence 67 // of numbers as during recording. 68 rng *rand.Rand 69 newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error) 70 71 replaying bool 72) 73 74func TestMain(m *testing.M) { 75 cleanup := initIntegrationTest() 76 exit := m.Run() 77 if err := cleanup(); err != nil { 78 // Don't fail the test if cleanup fails. 79 log.Printf("Post-test cleanup failed: %v", err) 80 } 81 os.Exit(exit) 82} 83 84// If integration tests will be run, create a unique bucket for them. 85// Also, set newTestClient to handle record/replay. 86// Return a cleanup function. 87func initIntegrationTest() func() error { 88 flag.Parse() // needed for testing.Short() 89 switch { 90 case testing.Short() && *record: 91 log.Fatal("cannot combine -short and -record") 92 return nil 93 94 case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "": 95 // go test -short with a replay file will replay the integration tests, if 96 // the appropriate environment variables have been set. 97 replaying = true 98 httpreplay.DebugHeaders() 99 replayer, err := httpreplay.NewReplayer(replayFilename) 100 if err != nil { 101 log.Fatal(err) 102 } 103 var t time.Time 104 if err := json.Unmarshal(replayer.Initial(), &t); err != nil { 105 log.Fatal(err) 106 } 107 initUIDsAndRand(t) 108 newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) { 109 hc, err := replayer.Client(ctx) // no creds needed 110 if err != nil { 111 return nil, err 112 } 113 return NewClient(ctx, option.WithHTTPClient(hc)) 114 } 115 log.Printf("replaying from %s", replayFilename) 116 return func() error { return replayer.Close() } 117 118 case testing.Short(): 119 // go test -short without a replay file skips the integration tests. 120 if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" { 121 log.Print("replay not supported for Go versions before 1.8") 122 } 123 newTestClient = nil 124 return func() error { return nil } 125 126 default: // Run integration tests against a real backend. 127 now := time.Now().UTC() 128 initUIDsAndRand(now) 129 var cleanup func() error 130 if *record && httpreplay.Supported() { 131 // Remember the time for replay. 132 nowBytes, err := json.Marshal(now) 133 if err != nil { 134 log.Fatal(err) 135 } 136 recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes) 137 if err != nil { 138 log.Fatalf("could not record: %v", err) 139 } 140 newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 141 hc, err := recorder.Client(ctx, opts...) 142 if err != nil { 143 return nil, err 144 } 145 return NewClient(ctx, option.WithHTTPClient(hc)) 146 } 147 cleanup = func() error { 148 err1 := cleanupBuckets() 149 err2 := recorder.Close() 150 if err1 != nil { 151 return err1 152 } 153 return err2 154 } 155 log.Printf("recording to %s", replayFilename) 156 } else { 157 if *record { 158 log.Print("record not supported for Go versions before 1.8") 159 } 160 newTestClient = NewClient 161 cleanup = cleanupBuckets 162 } 163 ctx := context.Background() 164 client := config(ctx) 165 if client == nil { 166 return func() error { return nil } 167 } 168 defer client.Close() 169 if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil { 170 log.Fatalf("creating bucket %q: %v", bucketName, err) 171 } 172 return cleanup 173 } 174} 175 176func initUIDsAndRand(t time.Time) { 177 uidSpace = uid.NewSpace(testPrefix, &uid.Options{Time: t}) 178 bucketName = uidSpace.New() 179 // Use our own random source, to avoid other parts of the program taking 180 // random numbers from the global source and putting record and replay 181 // out of sync. 182 rng = testutil.NewRand(t) 183} 184 185// testConfig returns the Client used to access GCS. testConfig skips 186// the current test if credentials are not available or when being run 187// in Short mode. 188func testConfig(ctx context.Context, t *testing.T) *Client { 189 if testing.Short() && !replaying { 190 t.Skip("Integration tests skipped in short mode") 191 } 192 client := config(ctx) 193 if client == nil { 194 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 195 } 196 return client 197} 198 199// config is like testConfig, but it doesn't need a *testing.T. 200func config(ctx context.Context) *Client { 201 ts := testutil.TokenSource(ctx, ScopeFullControl) 202 if ts == nil { 203 return nil 204 } 205 client, err := newTestClient(ctx, option.WithTokenSource(ts)) 206 if err != nil { 207 log.Fatalf("NewClient: %v", err) 208 } 209 return client 210} 211 212func TestIntegration_BucketMethods(t *testing.T) { 213 ctx := context.Background() 214 client := testConfig(ctx, t) 215 defer client.Close() 216 h := testHelper{t} 217 218 projectID := testutil.ProjID() 219 newBucketName := uidSpace.New() 220 b := client.Bucket(newBucketName) 221 // Test Create and Delete. 222 h.mustCreate(b, projectID, nil) 223 attrs := h.mustBucketAttrs(b) 224 if got, want := attrs.MetaGeneration, int64(1); got != want { 225 t.Errorf("got metagen %d, want %d", got, want) 226 } 227 if got, want := attrs.StorageClass, "STANDARD"; got != want { 228 t.Errorf("got storage class %q, want %q", got, want) 229 } 230 if attrs.VersioningEnabled { 231 t.Error("got versioning enabled, wanted it disabled") 232 } 233 h.mustDeleteBucket(b) 234 235 // Test Create and Delete with attributes. 236 labels := map[string]string{ 237 "l1": "v1", 238 "empty": "", 239 } 240 attrs = &BucketAttrs{ 241 StorageClass: "NEARLINE", 242 VersioningEnabled: true, 243 Labels: labels, 244 Lifecycle: Lifecycle{ 245 Rules: []LifecycleRule{{ 246 Action: LifecycleAction{ 247 Type: SetStorageClassAction, 248 StorageClass: "NEARLINE", 249 }, 250 Condition: LifecycleCondition{ 251 AgeInDays: 10, 252 Liveness: Archived, 253 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 254 MatchesStorageClasses: []string{"MULTI_REGIONAL", "STANDARD"}, 255 NumNewerVersions: 3, 256 }, 257 }, { 258 Action: LifecycleAction{ 259 Type: DeleteAction, 260 }, 261 Condition: LifecycleCondition{ 262 AgeInDays: 30, 263 Liveness: Live, 264 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 265 MatchesStorageClasses: []string{"NEARLINE"}, 266 NumNewerVersions: 10, 267 }, 268 }}, 269 }, 270 } 271 h.mustCreate(b, projectID, attrs) 272 attrs = h.mustBucketAttrs(b) 273 if got, want := attrs.MetaGeneration, int64(1); got != want { 274 t.Errorf("got metagen %d, want %d", got, want) 275 } 276 if got, want := attrs.StorageClass, "NEARLINE"; got != want { 277 t.Errorf("got storage class %q, want %q", got, want) 278 } 279 if !attrs.VersioningEnabled { 280 t.Error("got versioning disabled, wanted it enabled") 281 } 282 if got, want := attrs.Labels, labels; !testutil.Equal(got, want) { 283 t.Errorf("labels: got %v, want %v", got, want) 284 } 285 h.mustDeleteBucket(b) 286} 287 288func TestIntegration_BucketUpdate(t *testing.T) { 289 ctx := context.Background() 290 client := testConfig(ctx, t) 291 defer client.Close() 292 h := testHelper{t} 293 294 b := client.Bucket(bucketName) 295 attrs := h.mustBucketAttrs(b) 296 if attrs.VersioningEnabled { 297 t.Fatal("bucket should not have versioning by default") 298 } 299 if len(attrs.Labels) > 0 { 300 t.Fatal("bucket should not have labels initially") 301 } 302 303 // Using empty BucketAttrsToUpdate should be a no-nop. 304 attrs = h.mustUpdateBucket(b, BucketAttrsToUpdate{}) 305 if attrs.VersioningEnabled { 306 t.Fatal("should not have versioning") 307 } 308 if len(attrs.Labels) > 0 { 309 t.Fatal("should not have labels") 310 } 311 312 // Turn on versioning, add some labels. 313 ua := BucketAttrsToUpdate{VersioningEnabled: true} 314 ua.SetLabel("l1", "v1") 315 ua.SetLabel("empty", "") 316 attrs = h.mustUpdateBucket(b, ua) 317 if !attrs.VersioningEnabled { 318 t.Fatal("should have versioning now") 319 } 320 wantLabels := map[string]string{ 321 "l1": "v1", 322 "empty": "", 323 } 324 if !testutil.Equal(attrs.Labels, wantLabels) { 325 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 326 } 327 328 // Turn off versioning again; add and remove some more labels. 329 ua = BucketAttrsToUpdate{VersioningEnabled: false} 330 ua.SetLabel("l1", "v2") // update 331 ua.SetLabel("new", "new") // create 332 ua.DeleteLabel("empty") // delete 333 ua.DeleteLabel("absent") // delete non-existent 334 attrs = h.mustUpdateBucket(b, ua) 335 if attrs.VersioningEnabled { 336 t.Fatal("should have versioning off") 337 } 338 wantLabels = map[string]string{ 339 "l1": "v2", 340 "new": "new", 341 } 342 if !testutil.Equal(attrs.Labels, wantLabels) { 343 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 344 } 345 346 // Configure a lifecycle 347 wantLifecycle := Lifecycle{ 348 Rules: []LifecycleRule{ 349 { 350 Action: LifecycleAction{Type: "Delete"}, 351 Condition: LifecycleCondition{AgeInDays: 30}, 352 }, 353 }, 354 } 355 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle} 356 attrs = h.mustUpdateBucket(b, ua) 357 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 358 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 359 } 360} 361 362func TestIntegration_ConditionalDelete(t *testing.T) { 363 ctx := context.Background() 364 client := testConfig(ctx, t) 365 defer client.Close() 366 h := testHelper{t} 367 368 o := client.Bucket(bucketName).Object("conddel") 369 370 wc := o.NewWriter(ctx) 371 wc.ContentType = "text/plain" 372 h.mustWrite(wc, []byte("foo")) 373 374 gen := wc.Attrs().Generation 375 metaGen := wc.Attrs().Metageneration 376 377 if err := o.Generation(gen - 1).Delete(ctx); err == nil { 378 t.Fatalf("Unexpected successful delete with Generation") 379 } 380 if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil { 381 t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch") 382 } 383 if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil { 384 t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch") 385 } 386 if err := o.Generation(gen).Delete(ctx); err != nil { 387 t.Fatalf("final delete failed: %v", err) 388 } 389} 390 391func TestIntegration_Objects(t *testing.T) { 392 // TODO(jba): Use subtests (Go 1.7). 393 ctx := context.Background() 394 client := testConfig(ctx, t) 395 defer client.Close() 396 h := testHelper{t} 397 bkt := client.Bucket(bucketName) 398 399 const defaultType = "text/plain" 400 401 // Populate object names and make a map for their contents. 402 objects := []string{ 403 "obj1", 404 "obj2", 405 "obj/with/slashes", 406 } 407 contents := make(map[string][]byte) 408 409 // Test Writer. 410 for _, obj := range objects { 411 c := randomContents() 412 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 413 t.Errorf("Write for %v failed with %v", obj, err) 414 } 415 contents[obj] = c 416 } 417 418 testObjectIterator(t, bkt, objects) 419 420 // Test Reader. 421 for _, obj := range objects { 422 rc, err := bkt.Object(obj).NewReader(ctx) 423 if err != nil { 424 t.Errorf("Can't create a reader for %v, errored with %v", obj, err) 425 continue 426 } 427 if !rc.checkCRC { 428 t.Errorf("%v: not checking CRC", obj) 429 } 430 slurp, err := ioutil.ReadAll(rc) 431 if err != nil { 432 t.Errorf("Can't ReadAll object %v, errored with %v", obj, err) 433 } 434 if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { 435 t.Errorf("Contents (%q) = %q; want %q", obj, got, want) 436 } 437 if got, want := rc.Size(), len(contents[obj]); got != int64(want) { 438 t.Errorf("Size (%q) = %d; want %d", obj, got, want) 439 } 440 if got, want := rc.ContentType(), "text/plain"; got != want { 441 t.Errorf("ContentType (%q) = %q; want %q", obj, got, want) 442 } 443 if got, want := rc.CacheControl(), "public, max-age=60"; got != want { 444 t.Errorf("CacheControl (%q) = %q; want %q", obj, got, want) 445 } 446 rc.Close() 447 448 // Check early close. 449 buf := make([]byte, 1) 450 rc, err = bkt.Object(obj).NewReader(ctx) 451 if err != nil { 452 t.Fatalf("%v: %v", obj, err) 453 } 454 _, err = rc.Read(buf) 455 if err != nil { 456 t.Fatalf("%v: %v", obj, err) 457 } 458 if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) { 459 t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want) 460 } 461 if err := rc.Close(); err != nil { 462 t.Errorf("%v Close: %v", obj, err) 463 } 464 } 465 466 obj := objects[0] 467 objlen := int64(len(contents[obj])) 468 // Test Range Reader. 469 for i, r := range []struct { 470 offset, length, want int64 471 }{ 472 {0, objlen, objlen}, 473 {0, objlen / 2, objlen / 2}, 474 {objlen / 2, objlen, objlen / 2}, 475 {0, 0, 0}, 476 {objlen / 2, 0, 0}, 477 {objlen / 2, -1, objlen / 2}, 478 {0, objlen * 2, objlen}, 479 } { 480 rc, err := bkt.Object(obj).NewRangeReader(ctx, r.offset, r.length) 481 if err != nil { 482 t.Errorf("%+v: Can't create a range reader for %v, errored with %v", i, obj, err) 483 continue 484 } 485 if rc.Size() != objlen { 486 t.Errorf("%+v: Reader has a content-size of %d, want %d", i, rc.Size(), objlen) 487 } 488 if rc.Remain() != r.want { 489 t.Errorf("%+v: Reader's available bytes reported as %d, want %d", i, rc.Remain(), r.want) 490 } 491 slurp, err := ioutil.ReadAll(rc) 492 if err != nil { 493 t.Errorf("%+v: can't ReadAll object %v, errored with %v", r, obj, err) 494 continue 495 } 496 if len(slurp) != int(r.want) { 497 t.Errorf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", i, r.offset, r.length, len(slurp), r.want) 498 continue 499 } 500 if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) { 501 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 502 } 503 rc.Close() 504 } 505 506 objName := objects[0] 507 508 // Test NewReader googleapi.Error. 509 // Since a 429 or 5xx is hard to cause, we trigger a 416. 510 realLen := len(contents[objName]) 511 _, err := bkt.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10) 512 if err, ok := err.(*googleapi.Error); !ok { 513 t.Error("NewRangeReader did not return a googleapi.Error") 514 } else { 515 if err.Code != 416 { 516 t.Errorf("Code = %d; want %d", err.Code, 416) 517 } 518 if len(err.Header) == 0 { 519 t.Error("Missing googleapi.Error.Header") 520 } 521 if len(err.Body) == 0 { 522 t.Error("Missing googleapi.Error.Body") 523 } 524 } 525 526 // Test StatObject. 527 o := h.mustObjectAttrs(bkt.Object(objName)) 528 if got, want := o.Name, objName; got != want { 529 t.Errorf("Name (%v) = %q; want %q", objName, got, want) 530 } 531 if got, want := o.ContentType, defaultType; got != want { 532 t.Errorf("ContentType (%v) = %q; want %q", objName, got, want) 533 } 534 created := o.Created 535 // Check that the object is newer than its containing bucket. 536 bAttrs := h.mustBucketAttrs(bkt) 537 if o.Created.Before(bAttrs.Created) { 538 t.Errorf("Object %v is older than its containing bucket, %v", o, bAttrs) 539 } 540 541 // Test object copy. 542 copyName := "copy-" + objName 543 copyObj, err := bkt.Object(copyName).CopierFrom(bkt.Object(objName)).Run(ctx) 544 if err != nil { 545 t.Errorf("Copier.Run failed with %v", err) 546 } else if !namesEqual(copyObj, bucketName, copyName) { 547 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 548 copyObj.Bucket, copyObj.Name, bucketName, copyName) 549 } 550 551 // Copying with attributes. 552 const contentEncoding = "identity" 553 copier := bkt.Object(copyName).CopierFrom(bkt.Object(objName)) 554 copier.ContentEncoding = contentEncoding 555 copyObj, err = copier.Run(ctx) 556 if err != nil { 557 t.Errorf("Copier.Run failed with %v", err) 558 } else { 559 if !namesEqual(copyObj, bucketName, copyName) { 560 t.Errorf("Copy object bucket, name: got %q.%q, want %q.%q", 561 copyObj.Bucket, copyObj.Name, bucketName, copyName) 562 } 563 if copyObj.ContentEncoding != contentEncoding { 564 t.Errorf("Copy ContentEncoding: got %q, want %q", copyObj.ContentEncoding, contentEncoding) 565 } 566 } 567 568 // Test UpdateAttrs. 569 metadata := map[string]string{"key": "value"} 570 updated := h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 571 ContentType: "text/html", 572 ContentLanguage: "en", 573 Metadata: metadata, 574 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, 575 }) 576 if got, want := updated.ContentType, "text/html"; got != want { 577 t.Errorf("updated.ContentType == %q; want %q", got, want) 578 } 579 if got, want := updated.ContentLanguage, "en"; got != want { 580 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 581 } 582 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) { 583 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want) 584 } 585 if got, want := updated.Created, created; got != want { 586 t.Errorf("updated.Created == %q; want %q", got, want) 587 } 588 if !updated.Created.Before(updated.Updated) { 589 t.Errorf("updated.Updated should be newer than update.Created") 590 } 591 592 // Delete ContentType and ContentLanguage. 593 updated = h.mustUpdateObject(bkt.Object(objName), ObjectAttrsToUpdate{ 594 ContentType: "", 595 ContentLanguage: "", 596 Metadata: map[string]string{}, 597 }) 598 if got, want := updated.ContentType, ""; got != want { 599 t.Errorf("updated.ContentType == %q; want %q", got, want) 600 } 601 if got, want := updated.ContentLanguage, ""; got != want { 602 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 603 } 604 if updated.Metadata != nil { 605 t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata) 606 } 607 if got, want := updated.Created, created; got != want { 608 t.Errorf("updated.Created == %q; want %q", got, want) 609 } 610 if !updated.Created.Before(updated.Updated) { 611 t.Errorf("updated.Updated should be newer than update.Created") 612 } 613 614 // Test checksums. 615 checksumCases := []struct { 616 name string 617 contents [][]byte 618 size int64 619 md5 string 620 crc32c uint32 621 }{ 622 { 623 name: "checksum-object", 624 contents: [][]byte{[]byte("hello"), []byte("world")}, 625 size: 10, 626 md5: "fc5e038d38a57032085441e7fe7010b0", 627 crc32c: 1456190592, 628 }, 629 { 630 name: "zero-object", 631 contents: [][]byte{}, 632 size: 0, 633 md5: "d41d8cd98f00b204e9800998ecf8427e", 634 crc32c: 0, 635 }, 636 } 637 for _, c := range checksumCases { 638 wc := bkt.Object(c.name).NewWriter(ctx) 639 for _, data := range c.contents { 640 if _, err := wc.Write(data); err != nil { 641 t.Errorf("Write(%q) failed with %q", data, err) 642 } 643 } 644 if err = wc.Close(); err != nil { 645 t.Errorf("%q: close failed with %q", c.name, err) 646 } 647 obj := wc.Attrs() 648 if got, want := obj.Size, c.size; got != want { 649 t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want) 650 } 651 if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want { 652 t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want) 653 } 654 if got, want := obj.CRC32C, c.crc32c; got != want { 655 t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want) 656 } 657 } 658 659 // Test public ACL. 660 publicObj := objects[0] 661 if err = bkt.Object(publicObj).ACL().Set(ctx, AllUsers, RoleReader); err != nil { 662 t.Errorf("PutACLEntry failed with %v", err) 663 } 664 publicClient, err := newTestClient(ctx, option.WithoutAuthentication()) 665 if err != nil { 666 t.Fatal(err) 667 } 668 669 slurp := h.mustRead(publicClient.Bucket(bucketName).Object(publicObj)) 670 if !bytes.Equal(slurp, contents[publicObj]) { 671 t.Errorf("Public object's content: got %q, want %q", slurp, contents[publicObj]) 672 } 673 674 // Test writer error handling. 675 wc := publicClient.Bucket(bucketName).Object(publicObj).NewWriter(ctx) 676 if _, err := wc.Write([]byte("hello")); err != nil { 677 t.Errorf("Write unexpectedly failed with %v", err) 678 } 679 if err = wc.Close(); err == nil { 680 t.Error("Close expected an error, found none") 681 } 682 683 // Test deleting the copy object. 684 h.mustDeleteObject(bkt.Object(copyName)) 685 // Deleting it a second time should return ErrObjectNotExist. 686 if err := bkt.Object(copyName).Delete(ctx); err != ErrObjectNotExist { 687 t.Errorf("second deletion of %v = %v; want ErrObjectNotExist", copyName, err) 688 } 689 _, err = bkt.Object(copyName).Attrs(ctx) 690 if err != ErrObjectNotExist { 691 t.Errorf("Copy is expected to be deleted, stat errored with %v", err) 692 } 693 694 // Test object composition. 695 var compSrcs []*ObjectHandle 696 var wantContents []byte 697 for _, obj := range objects { 698 compSrcs = append(compSrcs, bkt.Object(obj)) 699 wantContents = append(wantContents, contents[obj]...) 700 } 701 checkCompose := func(obj *ObjectHandle, wantContentType string) { 702 rc := h.mustNewReader(obj) 703 slurp, err = ioutil.ReadAll(rc) 704 if err != nil { 705 t.Fatalf("ioutil.ReadAll: %v", err) 706 } 707 defer rc.Close() 708 if !bytes.Equal(slurp, wantContents) { 709 t.Errorf("Composed object contents\ngot: %q\nwant: %q", slurp, wantContents) 710 } 711 if got := rc.ContentType(); got != wantContentType { 712 t.Errorf("Composed object content-type = %q, want %q", got, wantContentType) 713 } 714 } 715 716 // Compose should work even if the user sets no destination attributes. 717 compDst := bkt.Object("composed1") 718 c := compDst.ComposerFrom(compSrcs...) 719 if _, err := c.Run(ctx); err != nil { 720 t.Fatalf("ComposeFrom error: %v", err) 721 } 722 checkCompose(compDst, "application/octet-stream") 723 724 // It should also work if we do. 725 compDst = bkt.Object("composed2") 726 c = compDst.ComposerFrom(compSrcs...) 727 c.ContentType = "text/json" 728 if _, err := c.Run(ctx); err != nil { 729 t.Fatalf("ComposeFrom error: %v", err) 730 } 731 checkCompose(compDst, "text/json") 732} 733 734func TestIntegration_Encoding(t *testing.T) { 735 ctx := context.Background() 736 client := testConfig(ctx, t) 737 defer client.Close() 738 bkt := client.Bucket(bucketName) 739 740 // Test content encoding 741 const zeroCount = 20 << 1 // TODO: should be 20 << 20 742 obj := bkt.Object("gzip-test") 743 w := obj.NewWriter(ctx) 744 w.ContentEncoding = "gzip" 745 gw := gzip.NewWriter(w) 746 if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil { 747 t.Fatalf("io.Copy, upload: %v", err) 748 } 749 if err := gw.Close(); err != nil { 750 t.Errorf("gzip.Close(): %v", err) 751 } 752 if err := w.Close(); err != nil { 753 t.Errorf("w.Close(): %v", err) 754 } 755 r, err := obj.NewReader(ctx) 756 if err != nil { 757 t.Fatalf("NewReader(gzip-test): %v", err) 758 } 759 n, err := io.Copy(ioutil.Discard, r) 760 if err != nil { 761 t.Errorf("io.Copy, download: %v", err) 762 } 763 if n != zeroCount { 764 t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount) 765 } 766 767 // Test NotFound. 768 _, err = bkt.Object("obj-not-exists").NewReader(ctx) 769 if err != ErrObjectNotExist { 770 t.Errorf("Object should not exist, err found to be %v", err) 771 } 772} 773 774func namesEqual(obj *ObjectAttrs, bucketName, objectName string) bool { 775 return obj.Bucket == bucketName && obj.Name == objectName 776} 777 778func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) { 779 ctx := context.Background() 780 h := testHelper{t} 781 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 782 names := make([]string, len(objects)) 783 copy(names, objects) 784 sort.Strings(names) 785 var attrs []*ObjectAttrs 786 for _, name := range names { 787 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 788 } 789 msg, ok := itesting.TestIterator(attrs, 790 func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) }, 791 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 792 if !ok { 793 t.Errorf("ObjectIterator.Next: %s", msg) 794 } 795 // TODO(jba): test query.Delimiter != "" 796} 797 798func TestIntegration_SignedURL(t *testing.T) { 799 if testing.Short() { // do not test during replay 800 t.Skip("Integration tests skipped in short mode") 801 } 802 // To test SignedURL, we need a real user email and private key. Extract them 803 // from the JSON key file. 804 jwtConf, err := testutil.JWTConfig() 805 if err != nil { 806 t.Fatal(err) 807 } 808 if jwtConf == nil { 809 t.Skip("JSON key file is not present") 810 } 811 812 ctx := context.Background() 813 client := testConfig(ctx, t) 814 defer client.Close() 815 816 bkt := client.Bucket(bucketName) 817 obj := "signedURL" 818 contents := []byte("This is a test of SignedURL.\n") 819 md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents 820 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 821 t.Fatalf("writing: %v", err) 822 } 823 for _, test := range []struct { 824 desc string 825 opts SignedURLOptions 826 headers map[string][]string 827 fail bool 828 }{ 829 { 830 desc: "basic", 831 }, 832 { 833 desc: "MD5 sent and matches", 834 opts: SignedURLOptions{MD5: md5}, 835 headers: map[string][]string{"Content-MD5": {md5}}, 836 }, 837 { 838 desc: "MD5 not sent", 839 opts: SignedURLOptions{MD5: md5}, 840 fail: true, 841 }, 842 { 843 desc: "Content-Type sent and matches", 844 opts: SignedURLOptions{ContentType: "text/plain"}, 845 headers: map[string][]string{"Content-Type": {"text/plain"}}, 846 }, 847 { 848 desc: "Content-Type sent but does not match", 849 opts: SignedURLOptions{ContentType: "text/plain"}, 850 headers: map[string][]string{"Content-Type": {"application/json"}}, 851 fail: true, 852 }, 853 { 854 desc: "Canonical headers sent and match", 855 opts: SignedURLOptions{Headers: []string{ 856 " X-Goog-Foo: Bar baz ", 857 "X-Goog-Novalue", // ignored: no value 858 "X-Google-Foo", // ignored: wrong prefix 859 }}, 860 headers: map[string][]string{"X-Goog-foo": {"Bar baz "}}, 861 }, 862 { 863 desc: "Canonical headers sent but don't match", 864 opts: SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}}, 865 headers: map[string][]string{"X-Goog-Foo": {"bar baz"}}, 866 fail: true, 867 }, 868 } { 869 opts := test.opts 870 opts.GoogleAccessID = jwtConf.Email 871 opts.PrivateKey = jwtConf.PrivateKey 872 opts.Method = "GET" 873 opts.Expires = time.Now().Add(time.Hour) 874 u, err := SignedURL(bucketName, obj, &opts) 875 if err != nil { 876 t.Errorf("%s: SignedURL: %v", test.desc, err) 877 continue 878 } 879 got, err := getURL(u, test.headers) 880 if err != nil && !test.fail { 881 t.Errorf("%s: getURL %q: %v", test.desc, u, err) 882 } else if err == nil && !bytes.Equal(got, contents) { 883 t.Errorf("%s: got %q, want %q", test.desc, got, contents) 884 } 885 } 886} 887 888// Make a GET request to a URL using an unauthenticated client, and return its contents. 889func getURL(url string, headers map[string][]string) ([]byte, error) { 890 req, err := http.NewRequest("GET", url, nil) 891 if err != nil { 892 return nil, err 893 } 894 req.Header = headers 895 res, err := http.DefaultClient.Do(req) 896 if err != nil { 897 return nil, err 898 } 899 defer res.Body.Close() 900 bytes, err := ioutil.ReadAll(res.Body) 901 if err != nil { 902 return nil, err 903 } 904 if res.StatusCode != 200 { 905 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 906 } 907 return bytes, nil 908} 909 910func TestIntegration_ACL(t *testing.T) { 911 ctx := context.Background() 912 client := testConfig(ctx, t) 913 defer client.Close() 914 915 bkt := client.Bucket(bucketName) 916 917 entity := ACLEntity("domain-google.com") 918 rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"} 919 if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil { 920 t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err) 921 } 922 923 acl, err := bkt.DefaultObjectACL().List(ctx) 924 if err != nil { 925 t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucketName, err) 926 } else if !hasRule(acl, rule) { 927 t.Errorf("default ACL missing %#v", rule) 928 } 929 aclObjects := []string{"acl1", "acl2"} 930 for _, obj := range aclObjects { 931 c := randomContents() 932 if err := writeObject(ctx, bkt.Object(obj), "", c); err != nil { 933 t.Errorf("Write for %v failed with %v", obj, err) 934 } 935 } 936 name := aclObjects[0] 937 o := bkt.Object(name) 938 acl, err = o.ACL().List(ctx) 939 if err != nil { 940 t.Errorf("Can't retrieve ACL of %v", name) 941 } else if !hasRule(acl, rule) { 942 t.Errorf("object ACL missing %+v", rule) 943 } 944 if err := o.ACL().Delete(ctx, entity); err != nil { 945 t.Errorf("object ACL: could not delete entity %s", entity) 946 } 947 // Delete the default ACL rule. We can't move this code earlier in the 948 // test, because the test depends on the fact that the object ACL inherits 949 // it. 950 if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil { 951 t.Errorf("default ACL: could not delete entity %s", entity) 952 } 953 954 entity2 := ACLEntity("user-jbd@google.com") 955 rule2 := ACLRule{Entity: entity2, Role: RoleReader, Email: "jbd@google.com"} 956 if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil { 957 t.Errorf("Error while putting bucket ACL rule: %v", err) 958 } 959 bACL, err := bkt.ACL().List(ctx) 960 if err != nil { 961 t.Errorf("Error while getting the ACL of the bucket: %v", err) 962 } else if !hasRule(bACL, rule2) { 963 t.Errorf("bucket ACL missing %+v", rule2) 964 } 965 if err := bkt.ACL().Delete(ctx, entity2); err != nil { 966 t.Errorf("Error while deleting bucket ACL rule: %v", err) 967 } 968 969} 970 971func hasRule(acl []ACLRule, rule ACLRule) bool { 972 for _, r := range acl { 973 if cmp.Equal(r, rule) { 974 return true 975 } 976 } 977 return false 978} 979 980func TestIntegration_ValidObjectNames(t *testing.T) { 981 ctx := context.Background() 982 client := testConfig(ctx, t) 983 defer client.Close() 984 985 bkt := client.Bucket(bucketName) 986 987 validNames := []string{ 988 "gopher", 989 "Гоферови", 990 "a", 991 strings.Repeat("a", 1024), 992 } 993 for _, name := range validNames { 994 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 995 t.Errorf("Object %q write failed: %v. Want success", name, err) 996 continue 997 } 998 defer bkt.Object(name).Delete(ctx) 999 } 1000 1001 invalidNames := []string{ 1002 "", // Too short. 1003 strings.Repeat("a", 1025), // Too long. 1004 "new\nlines", 1005 "bad\xffunicode", 1006 } 1007 for _, name := range invalidNames { 1008 // Invalid object names will either cause failure during Write or Close. 1009 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 1010 continue 1011 } 1012 defer bkt.Object(name).Delete(ctx) 1013 t.Errorf("%q should have failed. Didn't", name) 1014 } 1015} 1016 1017func TestIntegration_WriterContentType(t *testing.T) { 1018 ctx := context.Background() 1019 client := testConfig(ctx, t) 1020 defer client.Close() 1021 1022 obj := client.Bucket(bucketName).Object("content") 1023 testCases := []struct { 1024 content string 1025 setType, wantType string 1026 }{ 1027 { 1028 content: "It was the best of times, it was the worst of times.", 1029 wantType: "text/plain; charset=utf-8", 1030 }, 1031 { 1032 content: "<html><head><title>My first page</title></head></html>", 1033 wantType: "text/html; charset=utf-8", 1034 }, 1035 { 1036 content: "<html><head><title>My first page</title></head></html>", 1037 setType: "text/html", 1038 wantType: "text/html", 1039 }, 1040 { 1041 content: "<html><head><title>My first page</title></head></html>", 1042 setType: "image/jpeg", 1043 wantType: "image/jpeg", 1044 }, 1045 } 1046 for i, tt := range testCases { 1047 if err := writeObject(ctx, obj, tt.setType, []byte(tt.content)); err != nil { 1048 t.Errorf("writing #%d: %v", i, err) 1049 } 1050 attrs, err := obj.Attrs(ctx) 1051 if err != nil { 1052 t.Errorf("obj.Attrs: %v", err) 1053 continue 1054 } 1055 if got := attrs.ContentType; got != tt.wantType { 1056 t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType) 1057 } 1058 } 1059} 1060 1061func TestIntegration_ZeroSizedObject(t *testing.T) { 1062 t.Parallel() 1063 ctx := context.Background() 1064 client := testConfig(ctx, t) 1065 defer client.Close() 1066 h := testHelper{t} 1067 1068 obj := client.Bucket(bucketName).Object("zero") 1069 1070 // Check writing it works as expected. 1071 w := obj.NewWriter(ctx) 1072 if err := w.Close(); err != nil { 1073 t.Fatalf("Writer.Close: %v", err) 1074 } 1075 defer obj.Delete(ctx) 1076 1077 // Check we can read it too. 1078 body := h.mustRead(obj) 1079 if len(body) != 0 { 1080 t.Errorf("Body is %v, want empty []byte{}", body) 1081 } 1082} 1083 1084func TestIntegration_Encryption(t *testing.T) { 1085 // This function tests customer-supplied encryption keys for all operations 1086 // involving objects. Bucket and ACL operations aren't tested because they 1087 // aren't affected by customer encryption. Neither is deletion. 1088 ctx := context.Background() 1089 client := testConfig(ctx, t) 1090 defer client.Close() 1091 h := testHelper{t} 1092 1093 obj := client.Bucket(bucketName).Object("customer-encryption") 1094 key := []byte("my-secret-AES-256-encryption-key") 1095 keyHash := sha256.Sum256(key) 1096 keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:]) 1097 key2 := []byte("My-Secret-AES-256-Encryption-Key") 1098 contents := "top secret." 1099 1100 checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) { 1101 // Performing a metadata operation without the key should succeed. 1102 attrs, err := f(obj) 1103 if err != nil { 1104 t.Fatalf("%s: %v", msg, err) 1105 } 1106 // The key hash should match... 1107 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1108 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1109 } 1110 // ...but CRC and MD5 should not be present. 1111 if attrs.CRC32C != 0 { 1112 t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C) 1113 } 1114 if len(attrs.MD5) > 0 { 1115 t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5) 1116 } 1117 1118 // Performing a metadata operation with the key should succeed. 1119 attrs, err = f(obj.Key(key)) 1120 if err != nil { 1121 t.Fatalf("%s: %v", msg, err) 1122 } 1123 // Check the key and content hashes. 1124 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 1125 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 1126 } 1127 if attrs.CRC32C == 0 { 1128 t.Errorf("%s: CRC: got 0, want non-zero", msg) 1129 } 1130 if len(attrs.MD5) == 0 { 1131 t.Errorf("%s: MD5: got len == 0, want len > 0", msg) 1132 } 1133 } 1134 1135 checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) { 1136 // Reading the object without the key should fail. 1137 if _, err := readObject(ctx, o); err == nil { 1138 t.Errorf("%s: reading without key: want error, got nil", msg) 1139 } 1140 // Reading the object with the key should succeed. 1141 got := h.mustRead(o.Key(k)) 1142 gotContents := string(got) 1143 // And the contents should match what we wrote. 1144 if gotContents != wantContents { 1145 t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents) 1146 } 1147 } 1148 1149 checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) { 1150 got := h.mustRead(obj) 1151 gotContents := string(got) 1152 if gotContents != wantContents { 1153 t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents) 1154 } 1155 } 1156 1157 // Write to obj using our own encryption key, which is a valid 32-byte 1158 // AES-256 key. 1159 h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents)) 1160 1161 checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) { 1162 return o.Attrs(ctx) 1163 }) 1164 1165 checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) { 1166 return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 1167 }) 1168 1169 checkRead("first object", obj, key, contents) 1170 1171 obj2 := client.Bucket(bucketName).Object("customer-encryption-2") 1172 // Copying an object without the key should fail. 1173 if _, err := obj2.CopierFrom(obj).Run(ctx); err == nil { 1174 t.Fatal("want error, got nil") 1175 } 1176 // Copying an object with the key should succeed. 1177 if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1178 t.Fatal(err) 1179 } 1180 // The destination object is not encrypted; we can read it without a key. 1181 checkReadUnencrypted("copy dest", obj2, contents) 1182 1183 // Providing a key on the destination but not the source should fail, 1184 // since the source is encrypted. 1185 if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil { 1186 t.Fatal("want error, got nil") 1187 } 1188 1189 // But copying with keys for both source and destination should succeed. 1190 if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil { 1191 t.Fatal(err) 1192 } 1193 // And the destination should be encrypted, meaning we can only read it 1194 // with a key. 1195 checkRead("copy destination", obj2, key2, contents) 1196 1197 // Change obj2's key to prepare for compose, where all objects must have 1198 // the same key. Also illustrates key rotation: copy an object to itself 1199 // with a different key. 1200 if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil { 1201 t.Fatal(err) 1202 } 1203 obj3 := client.Bucket(bucketName).Object("customer-encryption-3") 1204 // Composing without keys should fail. 1205 if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil { 1206 t.Fatal("want error, got nil") 1207 } 1208 // Keys on the source objects result in an error. 1209 if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil { 1210 t.Fatal("want error, got nil") 1211 } 1212 // A key on the destination object both decrypts the source objects 1213 // and encrypts the destination. 1214 if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil { 1215 t.Fatalf("got %v, want nil", err) 1216 } 1217 // Check that the destination in encrypted. 1218 checkRead("compose destination", obj3, key, contents+contents) 1219 1220 // You can't compose one or more unencrypted source objects into an 1221 // encrypted destination object. 1222 _, err := obj2.CopierFrom(obj2.Key(key)).Run(ctx) // unencrypt obj2 1223 if err != nil { 1224 t.Fatal(err) 1225 } 1226 if _, err := obj3.Key(key).ComposerFrom(obj2).Run(ctx); err == nil { 1227 t.Fatal("got nil, want error") 1228 } 1229} 1230 1231func TestIntegration_NonexistentBucket(t *testing.T) { 1232 t.Parallel() 1233 ctx := context.Background() 1234 client := testConfig(ctx, t) 1235 defer client.Close() 1236 1237 bkt := client.Bucket(uidSpace.New()) 1238 if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist { 1239 t.Errorf("Attrs: got %v, want ErrBucketNotExist", err) 1240 } 1241 it := bkt.Objects(ctx, nil) 1242 if _, err := it.Next(); err != ErrBucketNotExist { 1243 t.Errorf("Objects: got %v, want ErrBucketNotExist", err) 1244 } 1245} 1246 1247func TestIntegration_PerObjectStorageClass(t *testing.T) { 1248 const ( 1249 defaultStorageClass = "STANDARD" 1250 newStorageClass = "MULTI_REGIONAL" 1251 ) 1252 ctx := context.Background() 1253 client := testConfig(ctx, t) 1254 defer client.Close() 1255 h := testHelper{t} 1256 1257 bkt := client.Bucket(bucketName) 1258 1259 // The bucket should have the default storage class. 1260 battrs := h.mustBucketAttrs(bkt) 1261 if battrs.StorageClass != defaultStorageClass { 1262 t.Fatalf("bucket storage class: got %q, want %q", 1263 battrs.StorageClass, defaultStorageClass) 1264 } 1265 // Write an object; it should start with the bucket's storage class. 1266 obj := bkt.Object("posc") 1267 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 1268 oattrs, err := obj.Attrs(ctx) 1269 if err != nil { 1270 t.Fatal(err) 1271 } 1272 if oattrs.StorageClass != defaultStorageClass { 1273 t.Fatalf("object storage class: got %q, want %q", 1274 oattrs.StorageClass, defaultStorageClass) 1275 } 1276 // Now use Copy to change the storage class. 1277 copier := obj.CopierFrom(obj) 1278 copier.StorageClass = newStorageClass 1279 oattrs2, err := copier.Run(ctx) 1280 if err != nil { 1281 log.Fatal(err) 1282 } 1283 if oattrs2.StorageClass != newStorageClass { 1284 t.Fatalf("new object storage class: got %q, want %q", 1285 oattrs2.StorageClass, newStorageClass) 1286 } 1287 1288 // We can also write a new object using a non-default storage class. 1289 obj2 := bkt.Object("posc2") 1290 w := obj2.NewWriter(ctx) 1291 w.StorageClass = newStorageClass 1292 h.mustWrite(w, []byte("xxx")) 1293 if w.Attrs().StorageClass != newStorageClass { 1294 t.Fatalf("new object storage class: got %q, want %q", 1295 w.Attrs().StorageClass, newStorageClass) 1296 } 1297} 1298 1299func TestIntegration_BucketInCopyAttrs(t *testing.T) { 1300 // Confirm that if bucket is included in the object attributes of a rewrite 1301 // call, but object name and content-type aren't, then we get an error. See 1302 // the comment in Copier.Run. 1303 ctx := context.Background() 1304 client := testConfig(ctx, t) 1305 defer client.Close() 1306 h := testHelper{t} 1307 1308 bkt := client.Bucket(bucketName) 1309 obj := bkt.Object("bucketInCopyAttrs") 1310 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 1311 copier := obj.CopierFrom(obj) 1312 rawObject := copier.ObjectAttrs.toRawObject(bucketName) 1313 _, err := copier.callRewrite(ctx, rawObject) 1314 if err == nil { 1315 t.Errorf("got nil, want error") 1316 } 1317} 1318 1319func TestIntegration_NoUnicodeNormalization(t *testing.T) { 1320 t.Parallel() 1321 ctx := context.Background() 1322 client := testConfig(ctx, t) 1323 defer client.Close() 1324 bkt := client.Bucket("storage-library-test-bucket") 1325 h := testHelper{t} 1326 1327 for _, tst := range []struct { 1328 nameQuoted, content string 1329 }{ 1330 {`"Caf\u00e9"`, "Normalization Form C"}, 1331 {`"Cafe\u0301"`, "Normalization Form D"}, 1332 } { 1333 name, err := strconv.Unquote(tst.nameQuoted) 1334 if err != nil { 1335 t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err) 1336 } 1337 if got := string(h.mustRead(bkt.Object(name))); got != tst.content { 1338 t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content) 1339 } 1340 } 1341} 1342 1343func TestIntegration_HashesOnUpload(t *testing.T) { 1344 // Check that the user can provide hashes on upload, and that these are checked. 1345 ctx := context.Background() 1346 client := testConfig(ctx, t) 1347 defer client.Close() 1348 obj := client.Bucket(bucketName).Object("hashesOnUpload-1") 1349 data := []byte("I can't wait to be verified") 1350 1351 write := func(w *Writer) error { 1352 if _, err := w.Write(data); err != nil { 1353 _ = w.Close() 1354 return err 1355 } 1356 return w.Close() 1357 } 1358 1359 crc32c := crc32.Checksum(data, crc32cTable) 1360 // The correct CRC should succeed. 1361 w := obj.NewWriter(ctx) 1362 w.CRC32C = crc32c 1363 w.SendCRC32C = true 1364 if err := write(w); err != nil { 1365 t.Fatal(err) 1366 } 1367 1368 // If we change the CRC, validation should fail. 1369 w = obj.NewWriter(ctx) 1370 w.CRC32C = crc32c + 1 1371 w.SendCRC32C = true 1372 if err := write(w); err == nil { 1373 t.Fatal("write with bad CRC32c: want error, got nil") 1374 } 1375 1376 // If we have the wrong CRC but forget to send it, we succeed. 1377 w = obj.NewWriter(ctx) 1378 w.CRC32C = crc32c + 1 1379 if err := write(w); err != nil { 1380 t.Fatal(err) 1381 } 1382 1383 // MD5 1384 md5 := md5.Sum(data) 1385 // The correct MD5 should succeed. 1386 w = obj.NewWriter(ctx) 1387 w.MD5 = md5[:] 1388 if err := write(w); err != nil { 1389 t.Fatal(err) 1390 } 1391 1392 // If we change the MD5, validation should fail. 1393 w = obj.NewWriter(ctx) 1394 w.MD5 = append([]byte(nil), md5[:]...) 1395 w.MD5[0]++ 1396 if err := write(w); err == nil { 1397 t.Fatal("write with bad MD5: want error, got nil") 1398 } 1399} 1400 1401func TestIntegration_BucketIAM(t *testing.T) { 1402 ctx := context.Background() 1403 client := testConfig(ctx, t) 1404 defer client.Close() 1405 1406 bkt := client.Bucket(bucketName) 1407 1408 // This bucket is unique to this test run. So we don't have 1409 // to worry about other runs interfering with our IAM policy 1410 // changes. 1411 1412 member := "projectViewer:" + testutil.ProjID() 1413 role := iam.RoleName("roles/storage.objectViewer") 1414 // Get the bucket's IAM policy. 1415 policy, err := bkt.IAM().Policy(ctx) 1416 if err != nil { 1417 t.Fatalf("Getting policy: %v", err) 1418 } 1419 // The member should not have the role. 1420 if policy.HasRole(member, role) { 1421 t.Errorf("member %q has role %q", member, role) 1422 } 1423 // Change the policy. 1424 policy.Add(member, role) 1425 if err := bkt.IAM().SetPolicy(ctx, policy); err != nil { 1426 t.Fatalf("SetPolicy: %v", err) 1427 } 1428 // Confirm that the binding was added. 1429 policy, err = bkt.IAM().Policy(ctx) 1430 if err != nil { 1431 t.Fatalf("Getting policy: %v", err) 1432 } 1433 if !policy.HasRole(member, role) { 1434 t.Errorf("member %q does not have role %q", member, role) 1435 } 1436 1437 // Check TestPermissions. 1438 // This client should have all these permissions (and more). 1439 perms := []string{"storage.buckets.get", "storage.buckets.delete"} 1440 got, err := bkt.IAM().TestPermissions(ctx, perms) 1441 if err != nil { 1442 t.Fatalf("TestPermissions: %v", err) 1443 } 1444 sort.Strings(perms) 1445 sort.Strings(got) 1446 if !testutil.Equal(got, perms) { 1447 t.Errorf("got %v, want %v", got, perms) 1448 } 1449} 1450 1451func TestIntegration_RequesterPays(t *testing.T) { 1452 // This test needs a second project and user (token source) to test 1453 // all possibilities. Since we need these things for Firestore already, 1454 // we use them here. 1455 // 1456 // There are up to three entities involved in a requester-pays call: 1457 // 1458 // 1. The user making the request. Here, we use 1459 // a. The account used to create the token source used for all our 1460 // integration tests (see testutil.TokenSource). 1461 // b. The account used for the Firestore tests. 1462 // 2. The project that owns the requester-pays bucket. Here, that 1463 // is the test project ID (see testutil.ProjID). 1464 // 3. The project provided as the userProject parameter of the request; 1465 // the project to be billed. This test uses: 1466 // a. The project that owns the requester-pays bucket (same as (2)) 1467 // b. Another project (the Firestore project). 1468 // 1469 // The following must hold for this test to work: 1470 // - (1a) must have resourcemanager.projects.createBillingAssignment permission 1471 // (Owner role) on (2) (the project, not the bucket). 1472 // - (1b) must NOT have that permission on (2). 1473 // - (1b) must have serviceusage.services.use permission (Editor role) on (3b). 1474 // - (1b) must NOT have that permission on (3a). 1475 // - (1a) must NOT have that permission on (3b). 1476 const wantErrorCode = 400 1477 1478 ctx := context.Background() 1479 client := testConfig(ctx, t) 1480 defer client.Close() 1481 h := testHelper{t} 1482 1483 bucketName2 := uidSpace.New() 1484 b1 := client.Bucket(bucketName2) 1485 projID := testutil.ProjID() 1486 // Use Firestore project as a project that does not contain the bucket. 1487 otherProjID := os.Getenv(envFirestoreProjID) 1488 if otherProjID == "" { 1489 t.Fatalf("need a second project (env var %s)", envFirestoreProjID) 1490 } 1491 ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl) 1492 if ts == nil { 1493 t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey) 1494 } 1495 otherClient, err := newTestClient(ctx, option.WithTokenSource(ts)) 1496 if err != nil { 1497 t.Fatal(err) 1498 } 1499 defer otherClient.Close() 1500 b2 := otherClient.Bucket(bucketName2) 1501 user, err := keyFileEmail(os.Getenv("GCLOUD_TESTS_GOLANG_KEY")) 1502 if err != nil { 1503 t.Fatal(err) 1504 } 1505 otherUser, err := keyFileEmail(os.Getenv(envFirestorePrivateKey)) 1506 if err != nil { 1507 t.Fatal(err) 1508 } 1509 1510 // Create a requester-pays bucket. The bucket is contained in the project projID. 1511 h.mustCreate(b1, projID, &BucketAttrs{RequesterPays: true}) 1512 if err := b1.ACL().Set(ctx, ACLEntity("user-"+otherUser), RoleOwner); err != nil { 1513 t.Fatal(err) 1514 } 1515 1516 // Extract the error code from err if it's a googleapi.Error. 1517 errCode := func(err error) int { 1518 if err == nil { 1519 return 0 1520 } 1521 if err, ok := err.(*googleapi.Error); ok { 1522 return err.Code 1523 } 1524 return -1 1525 } 1526 1527 // Call f under various conditions. 1528 // Here b and ob refer to the same bucket, but b is bound to client, 1529 // while ob is bound to otherClient. The clients differ in their credentials, 1530 // i.e. the identity of the user making the RPC: b's user is an Owner on the 1531 // bucket's containing project, ob's is not. 1532 call := func(msg string, f func(*BucketHandle) error) { 1533 // user: an Owner on the containing project 1534 // userProject: absent 1535 // result: success, by the rule permitting access by owners of the containing bucket. 1536 if err := f(b1); err != nil { 1537 t.Errorf("%s: %v, want nil\n"+ 1538 "confirm that %s is an Owner on %s", 1539 msg, err, user, projID) 1540 } 1541 // user: an Owner on the containing project 1542 // userProject: containing project 1543 // result: success, by the same rule as above; userProject is unnecessary but allowed. 1544 if err := f(b1.UserProject(projID)); err != nil { 1545 t.Errorf("%s: got %v, want nil", msg, err) 1546 } 1547 // user: not an Owner on the containing project 1548 // userProject: absent 1549 // result: failure, by the standard requester-pays rule 1550 err := f(b2) 1551 if got, want := errCode(err), wantErrorCode; got != want { 1552 t.Errorf("%s: got error %v with code %d, want code %d\n"+ 1553 "confirm that %s is NOT an Owner on %s", 1554 msg, err, got, want, otherUser, projID) 1555 } 1556 // user: not an Owner on the containing project 1557 // userProject: not the containing one, but user has Editor role on it 1558 // result: success, by the standard requester-pays rule 1559 if err := f(b2.UserProject(otherProjID)); err != nil { 1560 t.Errorf("%s: got %v, want nil\n"+ 1561 "confirm that %s is an Editor on %s and that that project has billing enabled", 1562 msg, err, otherUser, otherProjID) 1563 } 1564 // user: not an Owner on the containing project 1565 // userProject: the containing one, on which the user does NOT have Editor permission. 1566 // result: failure 1567 err = f(b2.UserProject("veener-jba")) 1568 if got, want := errCode(err), 403; got != want { 1569 t.Errorf("%s: got error %v, want code %d\n"+ 1570 "confirm that %s is NOT an Editor on %s", 1571 msg, err, want, otherUser, "veener-jba") 1572 } 1573 } 1574 1575 // Getting its attributes requires a user project. 1576 var attrs *BucketAttrs 1577 call("Bucket attrs", func(b *BucketHandle) error { 1578 a, err := b.Attrs(ctx) 1579 if a != nil { 1580 attrs = a 1581 } 1582 return err 1583 }) 1584 if attrs != nil { 1585 if got, want := attrs.RequesterPays, true; got != want { 1586 t.Fatalf("attr.RequesterPays = %t, want %t", got, want) 1587 } 1588 } 1589 // Object operations. 1590 call("write object", func(b *BucketHandle) error { 1591 return writeObject(ctx, b.Object("foo"), "text/plain", []byte("hello")) 1592 }) 1593 call("read object", func(b *BucketHandle) error { 1594 _, err := readObject(ctx, b.Object("foo")) 1595 return err 1596 }) 1597 call("object attrs", func(b *BucketHandle) error { 1598 _, err := b.Object("foo").Attrs(ctx) 1599 return err 1600 }) 1601 call("update object", func(b *BucketHandle) error { 1602 _, err := b.Object("foo").Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 1603 return err 1604 }) 1605 1606 // ACL operations. 1607 entity := ACLEntity("domain-google.com") 1608 call("bucket acl set", func(b *BucketHandle) error { 1609 return b.ACL().Set(ctx, entity, RoleReader) 1610 }) 1611 call("bucket acl list", func(b *BucketHandle) error { 1612 _, err := b.ACL().List(ctx) 1613 return err 1614 }) 1615 call("bucket acl delete", func(b *BucketHandle) error { 1616 err := b.ACL().Delete(ctx, entity) 1617 if errCode(err) == 404 { 1618 // Since we call the function multiple times, it will 1619 // fail with NotFound for all but the first. 1620 return nil 1621 } 1622 return err 1623 }) 1624 call("default object acl set", func(b *BucketHandle) error { 1625 return b.DefaultObjectACL().Set(ctx, entity, RoleReader) 1626 }) 1627 call("default object acl list", func(b *BucketHandle) error { 1628 _, err := b.DefaultObjectACL().List(ctx) 1629 return err 1630 }) 1631 call("default object acl delete", func(b *BucketHandle) error { 1632 err := b.DefaultObjectACL().Delete(ctx, entity) 1633 if errCode(err) == 404 { 1634 return nil 1635 } 1636 return err 1637 }) 1638 call("object acl set", func(b *BucketHandle) error { 1639 return b.Object("foo").ACL().Set(ctx, entity, RoleReader) 1640 }) 1641 call("object acl list", func(b *BucketHandle) error { 1642 _, err := b.Object("foo").ACL().List(ctx) 1643 return err 1644 }) 1645 call("object acl delete", func(b *BucketHandle) error { 1646 err := b.Object("foo").ACL().Delete(ctx, entity) 1647 if errCode(err) == 404 { 1648 return nil 1649 } 1650 return err 1651 }) 1652 1653 // Copy and compose. 1654 call("copy", func(b *BucketHandle) error { 1655 _, err := b.Object("copy").CopierFrom(b.Object("foo")).Run(ctx) 1656 return err 1657 }) 1658 call("compose", func(b *BucketHandle) error { 1659 _, err := b.Object("compose").ComposerFrom(b.Object("foo"), b.Object("copy")).Run(ctx) 1660 return err 1661 }) 1662 call("delete object", func(b *BucketHandle) error { 1663 // Make sure the object exists, so we don't get confused by ErrObjectNotExist. 1664 // The storage service may perform validation in any order (perhaps in parallel), 1665 // so if we delete an object that doesn't exist and for which we lack permission, 1666 // we could see either of those two errors. (See Google-internal bug 78341001.) 1667 h.mustWrite(b1.Object("foo").NewWriter(ctx), []byte("hello")) // note: b1, not b. 1668 return b.Object("foo").Delete(ctx) 1669 }) 1670 b1.Object("foo").Delete(ctx) // Make sure object is deleted. 1671 for _, obj := range []string{"copy", "compose"} { 1672 if err := b1.UserProject(projID).Object(obj).Delete(ctx); err != nil { 1673 t.Fatalf("could not delete %q: %v", obj, err) 1674 } 1675 } 1676 1677 h.mustDeleteBucket(b1) 1678} 1679 1680// TODO(jba): move to testutil, factor out from firestore/integration_test.go. 1681const ( 1682 envFirestoreProjID = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID" 1683 envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY" 1684) 1685 1686func keyFileEmail(filename string) (string, error) { 1687 bytes, err := ioutil.ReadFile(filename) 1688 if err != nil { 1689 return "", err 1690 } 1691 var v struct { 1692 ClientEmail string `json:"client_email"` 1693 } 1694 if err := json.Unmarshal(bytes, &v); err != nil { 1695 return "", err 1696 } 1697 return v.ClientEmail, nil 1698} 1699 1700func TestIntegration_Notifications(t *testing.T) { 1701 ctx := context.Background() 1702 client := testConfig(ctx, t) 1703 defer client.Close() 1704 bkt := client.Bucket(bucketName) 1705 1706 checkNotifications := func(msg string, want map[string]*Notification) { 1707 got, err := bkt.Notifications(ctx) 1708 if err != nil { 1709 t.Fatal(err) 1710 } 1711 if diff := testutil.Diff(got, want); diff != "" { 1712 t.Errorf("%s: got=-, want=+:\n%s", msg, diff) 1713 } 1714 } 1715 checkNotifications("initial", map[string]*Notification{}) 1716 1717 nArg := &Notification{ 1718 TopicProjectID: testutil.ProjID(), 1719 TopicID: "go-storage-notification-test", 1720 PayloadFormat: NoPayload, 1721 } 1722 n, err := bkt.AddNotification(ctx, nArg) 1723 if err != nil { 1724 t.Fatal(err) 1725 } 1726 nArg.ID = n.ID 1727 if !testutil.Equal(n, nArg) { 1728 t.Errorf("got %+v, want %+v", n, nArg) 1729 } 1730 checkNotifications("after add", map[string]*Notification{n.ID: n}) 1731 1732 if err := bkt.DeleteNotification(ctx, n.ID); err != nil { 1733 t.Fatal(err) 1734 } 1735 checkNotifications("after delete", map[string]*Notification{}) 1736} 1737 1738func TestIntegration_PublicBucket(t *testing.T) { 1739 // Confirm that an unauthenticated client can access a public bucket. 1740 // See https://cloud.google.com/storage/docs/public-datasets/landsat 1741 if testing.Short() && !replaying { 1742 t.Skip("Integration tests skipped in short mode") 1743 } 1744 1745 const landsatBucket = "gcp-public-data-landsat" 1746 const landsatPrefix = "LC08/PRE/044/034/LC80440342016259LGN00/" 1747 const landsatObject = landsatPrefix + "LC80440342016259LGN00_MTL.txt" 1748 1749 // Create an unauthenticated client. 1750 ctx := context.Background() 1751 client, err := newTestClient(ctx, option.WithoutAuthentication()) 1752 if err != nil { 1753 t.Fatal(err) 1754 } 1755 defer client.Close() 1756 h := testHelper{t} 1757 bkt := client.Bucket(landsatBucket) 1758 obj := bkt.Object(landsatObject) 1759 1760 // Read a public object. 1761 bytes := h.mustRead(obj) 1762 if got, want := len(bytes), 7903; got != want { 1763 t.Errorf("len(bytes) = %d, want %d", got, want) 1764 } 1765 1766 // List objects in a public bucket. 1767 iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix}) 1768 gotCount := 0 1769 for { 1770 _, err := iter.Next() 1771 if err == iterator.Done { 1772 break 1773 } 1774 if err != nil { 1775 t.Fatal(err) 1776 } 1777 gotCount++ 1778 } 1779 if wantCount := 13; gotCount != wantCount { 1780 t.Errorf("object count: got %d, want %d", gotCount, wantCount) 1781 } 1782 1783 errCode := func(err error) int { 1784 if err, ok := err.(*googleapi.Error); !ok { 1785 return -1 1786 } else { 1787 return err.Code 1788 } 1789 } 1790 1791 // Reading from or writing to a non-public bucket fails. 1792 c := testConfig(ctx, t) 1793 defer c.Close() 1794 nonPublicObj := client.Bucket(bucketName).Object("noauth") 1795 // Oddly, reading returns 403 but writing returns 401. 1796 _, err = readObject(ctx, nonPublicObj) 1797 if got, want := errCode(err), 403; got != want { 1798 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 1799 } 1800 err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b")) 1801 if got, want := errCode(err), 401; got != want { 1802 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 1803 } 1804} 1805 1806func TestIntegration_ReadCRC(t *testing.T) { 1807 // Test that the checksum is handled correctly when reading files. 1808 // For gzipped files, see https://github.com/GoogleCloudPlatform/google-cloud-dotnet/issues/1641. 1809 if testing.Short() && !replaying { 1810 t.Skip("Integration tests skipped in short mode") 1811 } 1812 1813 const ( 1814 // This is an uncompressed file. 1815 // See https://cloud.google.com/storage/docs/public-datasets/landsat 1816 uncompressedBucket = "gcp-public-data-landsat" 1817 uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt" 1818 1819 gzippedBucket = "storage-library-test-bucket" 1820 gzippedObject = "gzipped-text.txt" 1821 gzippedContents = "hello world" // uncompressed contents of the file 1822 ) 1823 ctx := context.Background() 1824 client, err := newTestClient(ctx, option.WithoutAuthentication()) 1825 if err != nil { 1826 t.Fatal(err) 1827 } 1828 defer client.Close() 1829 1830 for _, test := range []struct { 1831 desc string 1832 obj *ObjectHandle 1833 offset, length int64 1834 readCompressed bool // don't decompress a gzipped file 1835 1836 wantErr bool 1837 wantCheck bool // Should Reader try to check the CRC? 1838 }{ 1839 { 1840 desc: "uncompressed, entire file", 1841 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 1842 offset: 0, 1843 length: -1, 1844 readCompressed: false, 1845 wantCheck: true, 1846 }, 1847 { 1848 desc: "uncompressed, entire file, don't decompress", 1849 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 1850 offset: 0, 1851 length: -1, 1852 readCompressed: true, 1853 wantCheck: true, 1854 }, 1855 { 1856 desc: "uncompressed, suffix", 1857 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 1858 offset: 1, 1859 length: -1, 1860 readCompressed: false, 1861 wantCheck: false, 1862 }, 1863 { 1864 desc: "uncompressed, prefix", 1865 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 1866 offset: 0, 1867 length: 18, 1868 readCompressed: false, 1869 wantCheck: false, 1870 }, 1871 { 1872 // When a gzipped file is unzipped on read, we can't verify the checksum 1873 // because it was computed against the zipped contents. We can detect 1874 // this case using http.Response.Uncompressed. 1875 desc: "compressed, entire file, unzipped", 1876 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 1877 offset: 0, 1878 length: -1, 1879 readCompressed: false, 1880 wantCheck: false, 1881 }, 1882 { 1883 // When we read a gzipped file uncompressed, it's like reading a regular file: 1884 // the served content and the CRC match. 1885 desc: "compressed, entire file, read compressed", 1886 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 1887 offset: 0, 1888 length: -1, 1889 readCompressed: true, 1890 wantCheck: true, 1891 }, 1892 { 1893 desc: "compressed, partial, server unzips", 1894 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 1895 offset: 1, 1896 length: 8, 1897 readCompressed: false, 1898 wantErr: true, // GCS can't serve part of a gzipped object 1899 wantCheck: false, 1900 }, 1901 { 1902 desc: "compressed, partial, read compressed", 1903 obj: client.Bucket(gzippedBucket).Object(gzippedObject), 1904 offset: 1, 1905 length: 8, 1906 readCompressed: true, 1907 wantCheck: false, 1908 }, 1909 } { 1910 obj := test.obj.ReadCompressed(test.readCompressed) 1911 r, err := obj.NewRangeReader(ctx, test.offset, test.length) 1912 if err != nil { 1913 if test.wantErr { 1914 continue 1915 } 1916 t.Fatalf("%s: %v", test.desc, err) 1917 } 1918 if got, want := r.checkCRC, test.wantCheck; got != want { 1919 t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want) 1920 } 1921 _, err = ioutil.ReadAll(r) 1922 _ = r.Close() 1923 if err != nil { 1924 t.Fatalf("%s: %v", test.desc, err) 1925 } 1926 } 1927} 1928 1929func TestIntegration_CancelWrite(t *testing.T) { 1930 // Verify that canceling the writer's context immediately stops uploading an object. 1931 ctx := context.Background() 1932 client := testConfig(ctx, t) 1933 defer client.Close() 1934 bkt := client.Bucket(bucketName) 1935 1936 cctx, cancel := context.WithCancel(ctx) 1937 defer cancel() 1938 obj := bkt.Object("cancel-write") 1939 w := obj.NewWriter(cctx) 1940 w.ChunkSize = googleapi.MinUploadChunkSize 1941 buf := make([]byte, w.ChunkSize) 1942 // Write the first chunk. This is read in its entirety before sending the request 1943 // (see google.golang.org/api/gensupport.PrepareUpload), so we expect it to return 1944 // without error. 1945 _, err := w.Write(buf) 1946 if err != nil { 1947 t.Fatal(err) 1948 } 1949 // Now cancel the context. 1950 cancel() 1951 // The next Write should return context.Canceled. 1952 _, err = w.Write(buf) 1953 if err != context.Canceled { 1954 t.Fatalf("got %v, wanted context.Canceled", err) 1955 } 1956 // The Close should too. 1957 err = w.Close() 1958 if err != context.Canceled { 1959 t.Fatalf("got %v, wanted context.Canceled", err) 1960 } 1961} 1962 1963func TestIntegration_UpdateCORS(t *testing.T) { 1964 ctx := context.Background() 1965 client := testConfig(ctx, t) 1966 defer client.Close() 1967 h := testHelper{t} 1968 1969 initialSettings := []CORS{ 1970 { 1971 MaxAge: time.Hour, 1972 Methods: []string{"POST"}, 1973 Origins: []string{"some-origin.com"}, 1974 ResponseHeaders: []string{"foo-bar"}, 1975 }, 1976 } 1977 1978 for _, test := range []struct { 1979 input []CORS 1980 want []CORS 1981 }{ 1982 { 1983 input: []CORS{ 1984 { 1985 MaxAge: time.Hour, 1986 Methods: []string{"GET"}, 1987 Origins: []string{"*"}, 1988 ResponseHeaders: []string{"some-header"}, 1989 }, 1990 }, 1991 want: []CORS{ 1992 { 1993 MaxAge: time.Hour, 1994 Methods: []string{"GET"}, 1995 Origins: []string{"*"}, 1996 ResponseHeaders: []string{"some-header"}, 1997 }, 1998 }, 1999 }, 2000 { 2001 input: []CORS{}, 2002 want: nil, 2003 }, 2004 { 2005 input: nil, 2006 want: []CORS{ 2007 { 2008 MaxAge: time.Hour, 2009 Methods: []string{"POST"}, 2010 Origins: []string{"some-origin.com"}, 2011 ResponseHeaders: []string{"foo-bar"}, 2012 }, 2013 }, 2014 }, 2015 } { 2016 bkt := client.Bucket(uidSpace.New()) 2017 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings}) 2018 defer h.mustDeleteBucket(bkt) 2019 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input}) 2020 attrs := h.mustBucketAttrs(bkt) 2021 if diff := testutil.Diff(attrs.CORS, test.want); diff != "" { 2022 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2023 } 2024 } 2025} 2026 2027func TestIntegration_UpdateRetentionPolicy(t *testing.T) { 2028 ctx := context.Background() 2029 client := testConfig(ctx, t) 2030 defer client.Close() 2031 h := testHelper{t} 2032 2033 initial := &RetentionPolicy{RetentionPeriod: time.Minute} 2034 2035 for _, test := range []struct { 2036 input *RetentionPolicy 2037 want *RetentionPolicy 2038 }{ 2039 { // Update 2040 input: &RetentionPolicy{RetentionPeriod: time.Hour}, 2041 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2042 }, 2043 { // Update even with timestamp (EffectiveTime should be ignored) 2044 input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()}, 2045 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 2046 }, 2047 { // Remove 2048 input: &RetentionPolicy{}, 2049 want: nil, 2050 }, 2051 { // Remove even with timestamp (EffectiveTime should be ignored) 2052 input: &RetentionPolicy{EffectiveTime: time.Now()}, 2053 want: nil, 2054 }, 2055 { // Ignore 2056 input: nil, 2057 want: initial, 2058 }, 2059 } { 2060 bkt := client.Bucket(uidSpace.New()) 2061 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial}) 2062 defer h.mustDeleteBucket(bkt) 2063 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input}) 2064 attrs := h.mustBucketAttrs(bkt) 2065 if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 { 2066 // Should be set by the server and parsed by the client 2067 t.Fatal("EffectiveTime should be set, but it was not") 2068 } 2069 if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" { 2070 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 2071 } 2072 } 2073} 2074 2075func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) { 2076 ctx := context.Background() 2077 client := testConfig(ctx, t) 2078 defer client.Close() 2079 h := testHelper{t} 2080 2081 bkt := client.Bucket(uidSpace.New()) 2082 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}}) 2083 2084 oh := bkt.Object("some-object") 2085 if err := writeObject(ctx, oh, "text/plain", []byte("hello world")); err != nil { 2086 t.Fatal(err) 2087 } 2088 2089 if err := oh.Delete(ctx); err == nil { 2090 t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil") 2091 } 2092 2093 // Remove the retention period 2094 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}) 2095 h.mustDeleteObject(oh) 2096 h.mustDeleteBucket(bkt) 2097} 2098 2099func TestIntegration_LockBucket(t *testing.T) { 2100 ctx := context.Background() 2101 client := testConfig(ctx, t) 2102 defer client.Close() 2103 h := testHelper{t} 2104 2105 bkt := client.Bucket(uidSpace.New()) 2106 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}}) 2107 attrs := h.mustBucketAttrs(bkt) 2108 err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx) 2109 if err != nil { 2110 t.Fatal("could not lock", err) 2111 } 2112 2113 _, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 2114 if err == nil { 2115 t.Fatal("Expected error updating locked bucket, got nil") 2116 } 2117} 2118 2119func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) { 2120 ctx := context.Background() 2121 client := testConfig(ctx, t) 2122 defer client.Close() 2123 h := testHelper{t} 2124 2125 bkt := client.Bucket(uidSpace.New()) 2126 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2127 RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}, 2128 }) 2129 err := bkt.LockRetentionPolicy(ctx) 2130 if err == nil { 2131 t.Fatal("expected error locking bucket without metageneration condition, got nil") 2132 } 2133} 2134 2135func TestIntegration_KMS(t *testing.T) { 2136 ctx := context.Background() 2137 client := testConfig(ctx, t) 2138 defer client.Close() 2139 h := testHelper{t} 2140 2141 keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING") 2142 if keyRingName == "" { 2143 t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details") 2144 } 2145 keyName1 := keyRingName + "/cryptoKeys/key1" 2146 keyName2 := keyRingName + "/cryptoKeys/key2" 2147 contents := []byte("my secret") 2148 2149 write := func(obj *ObjectHandle, setKey bool) { 2150 w := obj.NewWriter(ctx) 2151 if setKey { 2152 w.KMSKeyName = keyName1 2153 } 2154 h.mustWrite(w, contents) 2155 } 2156 2157 checkRead := func(obj *ObjectHandle) { 2158 got := h.mustRead(obj) 2159 if !bytes.Equal(got, contents) { 2160 t.Errorf("got %v, want %v", got, contents) 2161 } 2162 attrs := h.mustObjectAttrs(obj) 2163 if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 { 2164 t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1) 2165 } 2166 } 2167 2168 // Write an object with a key, then read it to verify its contents and the presence of the key name. 2169 bkt := client.Bucket(bucketName) 2170 obj := bkt.Object("kms") 2171 write(obj, true) 2172 checkRead(obj) 2173 h.mustDeleteObject(obj) 2174 2175 // Encrypt an object with a CSEK, then copy it using a CMEK. 2176 src := bkt.Object("csek").Key(testEncryptionKey) 2177 if err := writeObject(ctx, src, "text/plain", contents); err != nil { 2178 t.Fatal(err) 2179 } 2180 dest := bkt.Object("cmek") 2181 c := dest.CopierFrom(src) 2182 c.DestinationKMSKeyName = keyName1 2183 if _, err := c.Run(ctx); err != nil { 2184 t.Fatal(err) 2185 } 2186 checkRead(dest) 2187 src.Delete(ctx) 2188 dest.Delete(ctx) 2189 2190 // Create a bucket with a default key, then write and read an object. 2191 bkt = client.Bucket(uidSpace.New()) 2192 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2193 Location: "US", 2194 Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1}, 2195 }) 2196 defer h.mustDeleteBucket(bkt) 2197 2198 attrs := h.mustBucketAttrs(bkt) 2199 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want { 2200 t.Fatalf("got %q, want %q", got, want) 2201 } 2202 obj = bkt.Object("kms") 2203 write(obj, false) 2204 checkRead(obj) 2205 h.mustDeleteObject(obj) 2206 2207 // Update the bucket's default key to a different name. 2208 // (This key doesn't have to exist.) 2209 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}) 2210 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 2211 t.Fatalf("got %q, want %q", got, want) 2212 } 2213 attrs = h.mustBucketAttrs(bkt) 2214 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 2215 t.Fatalf("got %q, want %q", got, want) 2216 } 2217 2218 // Remove the default KMS key. 2219 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}) 2220 if attrs.Encryption != nil { 2221 t.Fatalf("got %#v, want nil", attrs.Encryption) 2222 } 2223} 2224 2225func TestIntegration_PredefinedACLs(t *testing.T) { 2226 check := func(msg string, rs []ACLRule, i int, wantEntity ACLEntity, wantRole ACLRole) { 2227 if i >= len(rs) { 2228 t.Errorf("%s: no rule at index %d", msg, i) 2229 return 2230 } 2231 got := rs[i] 2232 if got.Entity != wantEntity || got.Role != wantRole { 2233 t.Errorf("%s[%d]: got %+v, want Entity %s and Role %s", 2234 msg, i, got, wantEntity, wantRole) 2235 } 2236 } 2237 checkPrefix := func(msg string, rs []ACLRule, i int, wantPrefix string, wantRole ACLRole) { 2238 if i >= len(rs) { 2239 t.Errorf("%s: no rule at index %d", msg, i) 2240 return 2241 } 2242 got := rs[i] 2243 if !strings.HasPrefix(string(got.Entity), wantPrefix) || got.Role != wantRole { 2244 t.Errorf("%s[%d]: got %+v, want Entity %s... and Role %s", 2245 msg, i, got, wantPrefix, wantRole) 2246 } 2247 } 2248 2249 ctx := context.Background() 2250 client := testConfig(ctx, t) 2251 defer client.Close() 2252 h := testHelper{t} 2253 2254 bkt := client.Bucket(uidSpace.New()) 2255 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 2256 PredefinedACL: "authenticatedRead", 2257 PredefinedDefaultObjectACL: "publicRead", 2258 }) 2259 defer h.mustDeleteBucket(bkt) 2260 attrs := h.mustBucketAttrs(bkt) 2261 checkPrefix("Bucket.ACL", attrs.ACL, 0, "project-owners", RoleOwner) 2262 check("Bucket.ACL", attrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 2263 check("DefaultObjectACL", attrs.DefaultObjectACL, 0, AllUsers, RoleReader) 2264 2265 // Bucket update 2266 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{ 2267 PredefinedACL: "private", 2268 PredefinedDefaultObjectACL: "authenticatedRead", 2269 }) 2270 checkPrefix("Bucket.ACL update", attrs.ACL, 0, "project-owners", RoleOwner) 2271 check("DefaultObjectACL update", attrs.DefaultObjectACL, 0, AllAuthenticatedUsers, RoleReader) 2272 2273 // Object creation 2274 obj := bkt.Object("private") 2275 w := obj.NewWriter(ctx) 2276 w.PredefinedACL = "authenticatedRead" 2277 h.mustWrite(w, []byte("hello")) 2278 defer h.mustDeleteObject(obj) 2279 checkPrefix("Object.ACL", w.Attrs().ACL, 0, "user", RoleOwner) 2280 check("Object.ACL", w.Attrs().ACL, 1, AllAuthenticatedUsers, RoleReader) 2281 2282 // Object update 2283 oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}) 2284 checkPrefix("Object.ACL update", oattrs.ACL, 0, "user", RoleOwner) 2285 if got := len(oattrs.ACL); got != 1 { 2286 t.Errorf("got %d ACLs, want 1", got) 2287 } 2288 2289 // Copy 2290 dst := bkt.Object("dst") 2291 copier := dst.CopierFrom(obj) 2292 copier.PredefinedACL = "publicRead" 2293 oattrs, err := copier.Run(ctx) 2294 if err != nil { 2295 t.Fatal(err) 2296 } 2297 defer h.mustDeleteObject(dst) 2298 // The copied object still retains the "private" ACL of the source object. 2299 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 2300 check("Copy dest", oattrs.ACL, 1, AllUsers, RoleReader) 2301 2302 // Compose 2303 comp := bkt.Object("comp") 2304 composer := comp.ComposerFrom(obj, dst) 2305 composer.PredefinedACL = "authenticatedRead" 2306 oattrs, err = composer.Run(ctx) 2307 if err != nil { 2308 t.Fatal(err) 2309 } 2310 defer h.mustDeleteObject(comp) 2311 // The composed object still retains the "private" ACL. 2312 checkPrefix("Copy dest", oattrs.ACL, 0, "user", RoleOwner) 2313 check("Copy dest", oattrs.ACL, 1, AllAuthenticatedUsers, RoleReader) 2314} 2315 2316type testHelper struct { 2317 t *testing.T 2318} 2319 2320func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) { 2321 if err := b.Create(context.Background(), projID, attrs); err != nil { 2322 h.t.Fatalf("%s: bucket create: %v", loc(), err) 2323 } 2324} 2325 2326func (h testHelper) mustDeleteBucket(b *BucketHandle) { 2327 if err := b.Delete(context.Background()); err != nil { 2328 h.t.Fatalf("%s: bucket delete: %v", loc(), err) 2329 } 2330} 2331 2332func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs { 2333 attrs, err := b.Attrs(context.Background()) 2334 if err != nil { 2335 h.t.Fatalf("%s: bucket attrs: %v", loc(), err) 2336 } 2337 return attrs 2338} 2339 2340func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate) *BucketAttrs { 2341 attrs, err := b.Update(context.Background(), ua) 2342 if err != nil { 2343 h.t.Fatalf("%s: update: %v", loc(), err) 2344 } 2345 return attrs 2346} 2347 2348func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs { 2349 attrs, err := o.Attrs(context.Background()) 2350 if err != nil { 2351 h.t.Fatalf("%s: object attrs: %v", loc(), err) 2352 } 2353 return attrs 2354} 2355 2356func (h testHelper) mustDeleteObject(o *ObjectHandle) { 2357 if err := o.Delete(context.Background()); err != nil { 2358 h.t.Fatalf("%s: object delete: %v", loc(), err) 2359 } 2360} 2361 2362func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate) *ObjectAttrs { 2363 attrs, err := o.Update(context.Background(), ua) 2364 if err != nil { 2365 h.t.Fatalf("%s: update: %v", loc(), err) 2366 } 2367 return attrs 2368} 2369 2370func (h testHelper) mustWrite(w *Writer, data []byte) { 2371 if _, err := w.Write(data); err != nil { 2372 w.Close() 2373 h.t.Fatalf("%s: write: %v", loc(), err) 2374 } 2375 if err := w.Close(); err != nil { 2376 h.t.Fatalf("%s: close write: %v", loc(), err) 2377 } 2378} 2379 2380func (h testHelper) mustRead(obj *ObjectHandle) []byte { 2381 data, err := readObject(context.Background(), obj) 2382 if err != nil { 2383 h.t.Fatalf("%s: read: %v", loc(), err) 2384 } 2385 return data 2386} 2387 2388func (h testHelper) mustNewReader(obj *ObjectHandle) *Reader { 2389 r, err := obj.NewReader(context.Background()) 2390 if err != nil { 2391 h.t.Fatalf("%s: new reader: %v", loc(), err) 2392 } 2393 return r 2394} 2395 2396func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error { 2397 w := obj.NewWriter(ctx) 2398 w.ContentType = contentType 2399 w.CacheControl = "public, max-age=60" 2400 if contents != nil { 2401 if _, err := w.Write(contents); err != nil { 2402 _ = w.Close() 2403 return err 2404 } 2405 } 2406 return w.Close() 2407} 2408 2409// loc returns a string describing the file and line of its caller's call site. In 2410// other words, if a test function calls a helper, and the helper calls loc, then the 2411// string will refer to the line on which the test function called the helper. 2412// TODO(jba): use t.Helper once we drop go 1.6. 2413func loc() string { 2414 _, file, line, ok := runtime.Caller(2) 2415 if !ok { 2416 return "???" 2417 } 2418 return fmt.Sprintf("%s:%d", filepath.Base(file), line) 2419} 2420 2421func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) { 2422 r, err := obj.NewReader(ctx) 2423 if err != nil { 2424 return nil, err 2425 } 2426 defer r.Close() 2427 return ioutil.ReadAll(r) 2428} 2429 2430// cleanupBuckets deletes the bucket used for testing, as well as old 2431// testing buckets that weren't cleaned previously. 2432func cleanupBuckets() error { 2433 if testing.Short() { 2434 return nil // Don't clean up in short mode. 2435 } 2436 ctx := context.Background() 2437 client := config(ctx) 2438 if client == nil { 2439 return nil // Don't cleanup if we're not configured correctly. 2440 } 2441 defer client.Close() 2442 if err := killBucket(ctx, client, bucketName); err != nil { 2443 return err 2444 } 2445 2446 // Delete buckets whose name begins with our test prefix, and which were 2447 // created a while ago. (Unfortunately GCS doesn't provide last-modified 2448 // time, which would be a better way to check for staleness.) 2449 const expireAge = 24 * time.Hour 2450 projectID := testutil.ProjID() 2451 it := client.Buckets(ctx, projectID) 2452 it.Prefix = testPrefix 2453 for { 2454 bktAttrs, err := it.Next() 2455 if err == iterator.Done { 2456 break 2457 } 2458 if err != nil { 2459 return err 2460 } 2461 if time.Since(bktAttrs.Created) > expireAge { 2462 log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge) 2463 if err := killBucket(ctx, client, bktAttrs.Name); err != nil { 2464 return err 2465 } 2466 } 2467 } 2468 return nil 2469} 2470 2471// killBucket deletes a bucket and all its objects. 2472func killBucket(ctx context.Context, client *Client, bucketName string) error { 2473 bkt := client.Bucket(bucketName) 2474 // Bucket must be empty to delete. 2475 it := bkt.Objects(ctx, nil) 2476 for { 2477 objAttrs, err := it.Next() 2478 if err == iterator.Done { 2479 break 2480 } 2481 if err != nil { 2482 return err 2483 } 2484 if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil { 2485 return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err) 2486 } 2487 } 2488 // GCS is eventually consistent, so this delete may fail because the 2489 // replica still sees an object in the bucket. We log the error and expect 2490 // a later test run to delete the bucket. 2491 if err := bkt.Delete(ctx); err != nil { 2492 log.Printf("deleting %q: %v", bucketName, err) 2493 } 2494 return nil 2495} 2496 2497func randomContents() []byte { 2498 h := md5.New() 2499 io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000))) 2500 return h.Sum(nil) 2501} 2502 2503type zeros struct{} 2504 2505func (zeros) Read(p []byte) (int, error) { return len(p), nil } 2506