1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package datastore 16 17import ( 18 "context" 19 "encoding/json" 20 "errors" 21 "flag" 22 "fmt" 23 "log" 24 "os" 25 "reflect" 26 "sort" 27 "strings" 28 "sync" 29 "testing" 30 "time" 31 32 "cloud.google.com/go/internal/testutil" 33 "cloud.google.com/go/internal/uid" 34 "cloud.google.com/go/rpcreplay" 35 "google.golang.org/api/iterator" 36 "google.golang.org/api/option" 37 "google.golang.org/grpc" 38 "google.golang.org/grpc/codes" 39 "google.golang.org/grpc/status" 40) 41 42// TODO(djd): Make test entity clean up more robust: some test entities may 43// be left behind if tests are aborted, the transport fails, etc. 44 45var timeNow = time.Now() 46 47// suffix is a timestamp-based suffix which is appended to key names, 48// particularly for the root keys of entity groups. This reduces flakiness 49// when the tests are run in parallel. 50var suffix string 51 52const replayFilename = "datastore.replay" 53 54type replayInfo struct { 55 ProjectID string 56 Time time.Time 57} 58 59var ( 60 record = flag.Bool("record", false, "record RPCs") 61 62 newTestClient = func(ctx context.Context, t *testing.T) *Client { 63 return newClient(ctx, t, nil) 64 } 65) 66 67func TestMain(m *testing.M) { 68 os.Exit(testMain(m)) 69} 70 71func testMain(m *testing.M) int { 72 flag.Parse() 73 if testing.Short() { 74 if *record { 75 log.Fatal("cannot combine -short and -record") 76 } 77 if testutil.CanReplay(replayFilename) { 78 initReplay() 79 } 80 } else if *record { 81 if testutil.ProjID() == "" { 82 log.Fatal("must record with a project ID") 83 } 84 b, err := json.Marshal(replayInfo{ 85 ProjectID: testutil.ProjID(), 86 Time: timeNow, 87 }) 88 if err != nil { 89 log.Fatal(err) 90 } 91 rec, err := rpcreplay.NewRecorder(replayFilename, b) 92 if err != nil { 93 log.Fatal(err) 94 } 95 defer func() { 96 if err := rec.Close(); err != nil { 97 log.Fatalf("closing recorder: %v", err) 98 } 99 }() 100 newTestClient = func(ctx context.Context, t *testing.T) *Client { 101 return newClient(ctx, t, rec.DialOptions()) 102 } 103 log.Printf("recording to %s", replayFilename) 104 } 105 suffix = fmt.Sprintf("-t%d", timeNow.UnixNano()) 106 return m.Run() 107} 108 109func initReplay() { 110 rep, err := rpcreplay.NewReplayer(replayFilename) 111 if err != nil { 112 log.Fatal(err) 113 } 114 defer rep.Close() 115 116 var ri replayInfo 117 if err := json.Unmarshal(rep.Initial(), &ri); err != nil { 118 log.Fatalf("unmarshaling initial replay info: %v", err) 119 } 120 timeNow = ri.Time.In(time.Local) 121 122 conn, err := rep.Connection() 123 if err != nil { 124 log.Fatal(err) 125 } 126 newTestClient = func(ctx context.Context, t *testing.T) *Client { 127 client, err := NewClient(ctx, ri.ProjectID, option.WithGRPCConn(conn)) 128 if err != nil { 129 t.Fatalf("NewClient: %v", err) 130 } 131 return client 132 } 133 log.Printf("replaying from %s", replayFilename) 134} 135 136func newClient(ctx context.Context, t *testing.T, dialOpts []grpc.DialOption) *Client { 137 if testing.Short() { 138 t.Skip("Integration tests skipped in short mode") 139 } 140 ts := testutil.TokenSource(ctx, ScopeDatastore) 141 if ts == nil { 142 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 143 } 144 opts := []option.ClientOption{option.WithTokenSource(ts)} 145 for _, opt := range dialOpts { 146 opts = append(opts, option.WithGRPCDialOption(opt)) 147 } 148 client, err := NewClient(ctx, testutil.ProjID(), opts...) 149 if err != nil { 150 t.Fatalf("NewClient: %v", err) 151 } 152 return client 153} 154 155func TestIntegration_Basics(t *testing.T) { 156 ctx, _ := context.WithTimeout(context.Background(), time.Second*20) 157 client := newTestClient(ctx, t) 158 defer client.Close() 159 160 type X struct { 161 I int 162 S string 163 T time.Time 164 } 165 166 x0 := X{66, "99", timeNow.Truncate(time.Millisecond)} 167 k, err := client.Put(ctx, IncompleteKey("BasicsX", nil), &x0) 168 if err != nil { 169 t.Fatalf("client.Put: %v", err) 170 } 171 x1 := X{} 172 err = client.Get(ctx, k, &x1) 173 if err != nil { 174 t.Errorf("client.Get: %v", err) 175 } 176 err = client.Delete(ctx, k) 177 if err != nil { 178 t.Errorf("client.Delete: %v", err) 179 } 180 if !testutil.Equal(x0, x1) { 181 t.Errorf("compare: x0=%v, x1=%v", x0, x1) 182 } 183} 184 185func TestIntegration_TopLevelKeyLoaded(t *testing.T) { 186 ctx, _ := context.WithTimeout(context.Background(), time.Second*20) 187 client := newTestClient(ctx, t) 188 defer client.Close() 189 190 completeKey := NameKey("EntityWithKey", "myent", nil) 191 192 type EntityWithKey struct { 193 I int 194 S string 195 K *Key `datastore:"__key__"` 196 } 197 198 in := &EntityWithKey{ 199 I: 12, 200 S: "abcd", 201 } 202 203 k, err := client.Put(ctx, completeKey, in) 204 if err != nil { 205 t.Fatalf("client.Put: %v", err) 206 } 207 208 var e EntityWithKey 209 err = client.Get(ctx, k, &e) 210 if err != nil { 211 t.Fatalf("client.Get: %v", err) 212 } 213 214 // The two keys should be absolutely identical. 215 if !testutil.Equal(e.K, k) { 216 t.Fatalf("e.K not equal to k; got %#v, want %#v", e.K, k) 217 } 218 219} 220 221func TestIntegration_ListValues(t *testing.T) { 222 ctx := context.Background() 223 client := newTestClient(ctx, t) 224 defer client.Close() 225 226 p0 := PropertyList{ 227 {Name: "L", Value: []interface{}{int64(12), "string", true}}, 228 } 229 k, err := client.Put(ctx, IncompleteKey("ListValue", nil), &p0) 230 if err != nil { 231 t.Fatalf("client.Put: %v", err) 232 } 233 var p1 PropertyList 234 if err := client.Get(ctx, k, &p1); err != nil { 235 t.Errorf("client.Get: %v", err) 236 } 237 if !testutil.Equal(p0, p1) { 238 t.Errorf("compare:\np0=%v\np1=%#v", p0, p1) 239 } 240 if err = client.Delete(ctx, k); err != nil { 241 t.Errorf("client.Delete: %v", err) 242 } 243} 244 245func TestIntegration_GetMulti(t *testing.T) { 246 ctx := context.Background() 247 client := newTestClient(ctx, t) 248 defer client.Close() 249 250 type X struct { 251 I int 252 } 253 p := NameKey("X", "x"+suffix, nil) 254 255 cases := []struct { 256 key *Key 257 put bool 258 }{ 259 {key: NameKey("X", "item1", p), put: true}, 260 {key: NameKey("X", "item2", p), put: false}, 261 {key: NameKey("X", "item3", p), put: false}, 262 {key: NameKey("X", "item3", p), put: false}, 263 {key: NameKey("X", "item4", p), put: true}, 264 } 265 266 var src, dst []*X 267 var srcKeys, dstKeys []*Key 268 for _, c := range cases { 269 dst = append(dst, &X{}) 270 dstKeys = append(dstKeys, c.key) 271 if c.put { 272 src = append(src, &X{}) 273 srcKeys = append(srcKeys, c.key) 274 } 275 } 276 if _, err := client.PutMulti(ctx, srcKeys, src); err != nil { 277 t.Error(err) 278 } 279 err := client.GetMulti(ctx, dstKeys, dst) 280 if err == nil { 281 t.Errorf("client.GetMulti got %v, expected error", err) 282 } 283 e, ok := err.(MultiError) 284 if !ok { 285 t.Errorf("client.GetMulti got %T, expected MultiError", err) 286 } 287 for i, err := range e { 288 got, want := err, (error)(nil) 289 if !cases[i].put { 290 got, want = err, ErrNoSuchEntity 291 } 292 if got != want { 293 t.Errorf("MultiError[%d] == %v, want %v", i, got, want) 294 } 295 } 296} 297 298type Z struct { 299 S string 300 T string `datastore:",noindex"` 301 P []byte 302 K []byte `datastore:",noindex"` 303} 304 305func (z Z) String() string { 306 var lens []string 307 v := reflect.ValueOf(z) 308 for i := 0; i < v.NumField(); i++ { 309 if l := v.Field(i).Len(); l > 0 { 310 lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l)) 311 } 312 } 313 return fmt.Sprintf("Z{ %s }", strings.Join(lens, ",")) 314} 315 316func TestIntegration_UnindexableValues(t *testing.T) { 317 ctx := context.Background() 318 client := newTestClient(ctx, t) 319 defer client.Close() 320 321 x1500 := strings.Repeat("x", 1500) 322 x1501 := strings.Repeat("x", 1501) 323 testCases := []struct { 324 in Z 325 wantErr bool 326 }{ 327 {in: Z{S: x1500}, wantErr: false}, 328 {in: Z{S: x1501}, wantErr: true}, 329 {in: Z{T: x1500}, wantErr: false}, 330 {in: Z{T: x1501}, wantErr: false}, 331 {in: Z{P: []byte(x1500)}, wantErr: false}, 332 {in: Z{P: []byte(x1501)}, wantErr: true}, 333 {in: Z{K: []byte(x1500)}, wantErr: false}, 334 {in: Z{K: []byte(x1501)}, wantErr: false}, 335 } 336 for _, tt := range testCases { 337 _, err := client.Put(ctx, IncompleteKey("BasicsZ", nil), &tt.in) 338 if (err != nil) != tt.wantErr { 339 t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) 340 } 341 } 342} 343 344func TestIntegration_NilKey(t *testing.T) { 345 ctx := context.Background() 346 client := newTestClient(ctx, t) 347 defer client.Close() 348 349 testCases := []struct { 350 in K0 351 wantErr bool 352 }{ 353 {in: K0{K: testKey0}, wantErr: false}, 354 {in: K0{}, wantErr: false}, 355 } 356 for _, tt := range testCases { 357 _, err := client.Put(ctx, IncompleteKey("NilKey", nil), &tt.in) 358 if (err != nil) != tt.wantErr { 359 t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) 360 } 361 } 362} 363 364type SQChild struct { 365 I, J int 366 T, U int64 367} 368 369type SQTestCase struct { 370 desc string 371 q *Query 372 wantCount int 373 wantSum int 374} 375 376func testSmallQueries(ctx context.Context, t *testing.T, client *Client, parent *Key, children []*SQChild, 377 testCases []SQTestCase, extraTests ...func()) { 378 keys := make([]*Key, len(children)) 379 for i := range keys { 380 keys[i] = IncompleteKey("SQChild", parent) 381 } 382 keys, err := client.PutMulti(ctx, keys, children) 383 if err != nil { 384 t.Fatalf("client.PutMulti: %v", err) 385 } 386 defer func() { 387 err := client.DeleteMulti(ctx, keys) 388 if err != nil { 389 t.Errorf("client.DeleteMulti: %v", err) 390 } 391 }() 392 393 for _, tc := range testCases { 394 count, err := client.Count(ctx, tc.q) 395 if err != nil { 396 t.Errorf("Count %q: %v", tc.desc, err) 397 continue 398 } 399 if count != tc.wantCount { 400 t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount) 401 continue 402 } 403 } 404 405 for _, tc := range testCases { 406 var got []SQChild 407 _, err := client.GetAll(ctx, tc.q, &got) 408 if err != nil { 409 t.Errorf("client.GetAll %q: %v", tc.desc, err) 410 continue 411 } 412 sum := 0 413 for _, c := range got { 414 sum += c.I + c.J 415 } 416 if sum != tc.wantSum { 417 t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum) 418 continue 419 } 420 } 421 for _, x := range extraTests { 422 x() 423 } 424} 425 426func TestIntegration_Filters(t *testing.T) { 427 ctx := context.Background() 428 client := newTestClient(ctx, t) 429 defer client.Close() 430 431 parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil) 432 now := timeNow.Truncate(time.Millisecond).Unix() 433 children := []*SQChild{ 434 {I: 0, T: now, U: now}, 435 {I: 1, T: now, U: now}, 436 {I: 2, T: now, U: now}, 437 {I: 3, T: now, U: now}, 438 {I: 4, T: now, U: now}, 439 {I: 5, T: now, U: now}, 440 {I: 6, T: now, U: now}, 441 {I: 7, T: now, U: now}, 442 } 443 baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now) 444 testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ 445 { 446 "I>1", 447 baseQuery.Filter("I>", 1), 448 6, 449 2 + 3 + 4 + 5 + 6 + 7, 450 }, 451 { 452 "I>2 AND I<=5", 453 baseQuery.Filter("I>", 2).Filter("I<=", 5), 454 3, 455 3 + 4 + 5, 456 }, 457 { 458 "I>=3 AND I<3", 459 baseQuery.Filter("I>=", 3).Filter("I<", 3), 460 0, 461 0, 462 }, 463 { 464 "I=4", 465 baseQuery.Filter("I=", 4), 466 1, 467 4, 468 }, 469 }, func() { 470 got := []*SQChild{} 471 want := []*SQChild{ 472 {I: 0, T: now, U: now}, 473 {I: 1, T: now, U: now}, 474 {I: 2, T: now, U: now}, 475 {I: 3, T: now, U: now}, 476 {I: 4, T: now, U: now}, 477 {I: 5, T: now, U: now}, 478 {I: 6, T: now, U: now}, 479 {I: 7, T: now, U: now}, 480 } 481 _, err := client.GetAll(ctx, baseQuery.Order("I"), &got) 482 if err != nil { 483 t.Errorf("client.GetAll: %v", err) 484 } 485 if !testutil.Equal(got, want) { 486 t.Errorf("compare: got=%v, want=%v", got, want) 487 } 488 }, func() { 489 got := []*SQChild{} 490 want := []*SQChild{ 491 {I: 7, T: now, U: now}, 492 {I: 6, T: now, U: now}, 493 {I: 5, T: now, U: now}, 494 {I: 4, T: now, U: now}, 495 {I: 3, T: now, U: now}, 496 {I: 2, T: now, U: now}, 497 {I: 1, T: now, U: now}, 498 {I: 0, T: now, U: now}, 499 } 500 _, err := client.GetAll(ctx, baseQuery.Order("-I"), &got) 501 if err != nil { 502 t.Errorf("client.GetAll: %v", err) 503 } 504 if !testutil.Equal(got, want) { 505 t.Errorf("compare: got=%v, want=%v", got, want) 506 } 507 }) 508} 509 510type ckey struct{} 511 512func TestIntegration_LargeQuery(t *testing.T) { 513 ctx := context.Background() 514 client := newTestClient(ctx, t) 515 defer client.Close() 516 517 parent := NameKey("LQParent", "TestIntegration_LargeQuery"+suffix, nil) 518 now := timeNow.Truncate(time.Millisecond).Unix() 519 520 // Make a large number of children entities. 521 const n = 800 522 children := make([]*SQChild, 0, n) 523 keys := make([]*Key, 0, n) 524 for i := 0; i < n; i++ { 525 children = append(children, &SQChild{I: i, T: now, U: now}) 526 keys = append(keys, IncompleteKey("SQChild", parent)) 527 } 528 529 // Store using PutMulti in batches. 530 const batchSize = 500 531 for i := 0; i < n; i = i + 500 { 532 j := i + batchSize 533 if j > n { 534 j = n 535 } 536 fullKeys, err := client.PutMulti(ctx, keys[i:j], children[i:j]) 537 if err != nil { 538 t.Fatalf("PutMulti(%d, %d): %v", i, j, err) 539 } 540 defer func() { 541 err := client.DeleteMulti(ctx, fullKeys) 542 if err != nil { 543 t.Errorf("client.DeleteMulti: %v", err) 544 } 545 }() 546 } 547 548 q := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Order("I") 549 550 // Wait group to allow us to run query tests in parallel below. 551 var wg sync.WaitGroup 552 553 // Check we get the expected count and results for various limits/offsets. 554 queryTests := []struct { 555 limit, offset, want int 556 }{ 557 // Just limit. 558 {limit: 0, want: 0}, 559 {limit: 100, want: 100}, 560 {limit: 501, want: 501}, 561 {limit: n, want: n}, 562 {limit: n * 2, want: n}, 563 {limit: -1, want: n}, 564 // Just offset. 565 {limit: -1, offset: 100, want: n - 100}, 566 {limit: -1, offset: 500, want: n - 500}, 567 {limit: -1, offset: n, want: 0}, 568 // Limit and offset. 569 {limit: 100, offset: 100, want: 100}, 570 {limit: 1000, offset: 100, want: n - 100}, 571 {limit: 500, offset: 500, want: n - 500}, 572 } 573 for _, tt := range queryTests { 574 q := q.Limit(tt.limit).Offset(tt.offset) 575 wg.Add(1) 576 577 go func(limit, offset, want int) { 578 defer wg.Done() 579 // Check Count returns the expected number of results. 580 count, err := client.Count(ctx, q) 581 if err != nil { 582 t.Errorf("client.Count(limit=%d offset=%d): %v", limit, offset, err) 583 return 584 } 585 if count != want { 586 t.Errorf("Count(limit=%d offset=%d) returned %d, want %d", limit, offset, count, want) 587 } 588 589 var got []SQChild 590 _, err = client.GetAll(ctx, q, &got) 591 if err != nil { 592 t.Errorf("client.GetAll(limit=%d offset=%d): %v", limit, offset, err) 593 return 594 } 595 if len(got) != want { 596 t.Errorf("GetAll(limit=%d offset=%d) returned %d, want %d", limit, offset, len(got), want) 597 } 598 for i, child := range got { 599 if got, want := child.I, i+offset; got != want { 600 t.Errorf("GetAll(limit=%d offset=%d) got[%d].I == %d; want %d", limit, offset, i, got, want) 601 break 602 } 603 } 604 }(tt.limit, tt.offset, tt.want) 605 } 606 607 // Also check iterator cursor behaviour. 608 cursorTests := []struct { 609 limit, offset int // Query limit and offset. 610 count int // The number of times to call "next" 611 want int // The I value of the desired element, -1 for "Done". 612 }{ 613 // No limits. 614 {count: 0, limit: -1, want: 0}, 615 {count: 5, limit: -1, want: 5}, 616 {count: 500, limit: -1, want: 500}, 617 {count: 1000, limit: -1, want: -1}, // No more results. 618 // Limits. 619 {count: 5, limit: 5, want: 5}, 620 {count: 500, limit: 5, want: 5}, 621 {count: 1000, limit: 1000, want: -1}, // No more results. 622 // Offsets. 623 {count: 0, offset: 5, limit: -1, want: 5}, 624 {count: 5, offset: 5, limit: -1, want: 10}, 625 {count: 200, offset: 500, limit: -1, want: 700}, 626 {count: 200, offset: 1000, limit: -1, want: -1}, // No more results. 627 } 628 for _, tt := range cursorTests { 629 wg.Add(1) 630 631 go func(count, limit, offset, want int) { 632 defer wg.Done() 633 634 ctx := context.WithValue(ctx, ckey{}, fmt.Sprintf("c=%d,l=%d,o=%d", count, limit, offset)) 635 // Run iterator through count calls to Next. 636 it := client.Run(ctx, q.Limit(limit).Offset(offset).KeysOnly()) 637 for i := 0; i < count; i++ { 638 _, err := it.Next(nil) 639 if err == iterator.Done { 640 break 641 } 642 if err != nil { 643 t.Errorf("count=%d, limit=%d, offset=%d: it.Next failed at i=%d", count, limit, offset, i) 644 return 645 } 646 } 647 648 // Grab the cursor. 649 cursor, err := it.Cursor() 650 if err != nil { 651 t.Errorf("count=%d, limit=%d, offset=%d: it.Cursor: %v", count, limit, offset, err) 652 return 653 } 654 655 // Make a request for the next element. 656 it = client.Run(ctx, q.Limit(1).Start(cursor)) 657 var entity SQChild 658 _, err = it.Next(&entity) 659 switch { 660 case want == -1: 661 if err != iterator.Done { 662 t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor %v, want Done", count, limit, offset, err) 663 } 664 case err != nil: 665 t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor: %v, want nil", count, limit, offset, err) 666 case entity.I != want: 667 t.Errorf("count=%d, limit=%d, offset=%d: got.I = %d, want %d", count, limit, offset, entity.I, want) 668 } 669 }(tt.count, tt.limit, tt.offset, tt.want) 670 } 671 wg.Wait() 672} 673 674func TestIntegration_EventualConsistency(t *testing.T) { 675 // TODO(jba): either make this actually test eventual consistency, or 676 // delete it. Currently it behaves the same with or without the 677 // EventualConsistency call. 678 ctx := context.Background() 679 client := newTestClient(ctx, t) 680 defer client.Close() 681 682 parent := NameKey("SQParent", "TestIntegration_EventualConsistency"+suffix, nil) 683 now := timeNow.Truncate(time.Millisecond).Unix() 684 children := []*SQChild{ 685 {I: 0, T: now, U: now}, 686 {I: 1, T: now, U: now}, 687 {I: 2, T: now, U: now}, 688 } 689 query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency() 690 testSmallQueries(ctx, t, client, parent, children, nil, func() { 691 got, err := client.Count(ctx, query) 692 if err != nil { 693 t.Fatalf("Count: %v", err) 694 } 695 if got < 0 || 3 < got { 696 t.Errorf("Count: got %d, want [0,3]", got) 697 } 698 }) 699} 700 701func TestIntegration_Projection(t *testing.T) { 702 ctx := context.Background() 703 client := newTestClient(ctx, t) 704 defer client.Close() 705 706 parent := NameKey("SQParent", "TestIntegration_Projection"+suffix, nil) 707 now := timeNow.Truncate(time.Millisecond).Unix() 708 children := []*SQChild{ 709 {I: 1 << 0, J: 100, T: now, U: now}, 710 {I: 1 << 1, J: 100, T: now, U: now}, 711 {I: 1 << 2, J: 200, T: now, U: now}, 712 {I: 1 << 3, J: 300, T: now, U: now}, 713 {I: 1 << 4, J: 300, T: now, U: now}, 714 } 715 baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150) 716 testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ 717 { 718 "project", 719 baseQuery.Project("J"), 720 3, 721 200 + 300 + 300, 722 }, 723 { 724 "distinct", 725 baseQuery.Project("J").Distinct(), 726 2, 727 200 + 300, 728 }, 729 { 730 "distinct on", 731 baseQuery.Project("J").DistinctOn("J"), 732 2, 733 200 + 300, 734 }, 735 { 736 "project on meaningful (GD_WHEN) field", 737 baseQuery.Project("U"), 738 3, 739 0, 740 }, 741 }) 742} 743 744func TestIntegration_AllocateIDs(t *testing.T) { 745 ctx := context.Background() 746 client := newTestClient(ctx, t) 747 defer client.Close() 748 749 keys := make([]*Key, 5) 750 for i := range keys { 751 keys[i] = IncompleteKey("AllocID", nil) 752 } 753 keys, err := client.AllocateIDs(ctx, keys) 754 if err != nil { 755 t.Errorf("AllocID #0 failed: %v", err) 756 } 757 if want := len(keys); want != 5 { 758 t.Errorf("Expected to allocate 5 keys, %d keys are found", want) 759 } 760 for _, k := range keys { 761 if k.Incomplete() { 762 t.Errorf("Unexpeceted incomplete key found: %v", k) 763 } 764 } 765} 766 767func TestIntegration_GetAllWithFieldMismatch(t *testing.T) { 768 ctx := context.Background() 769 client := newTestClient(ctx, t) 770 defer client.Close() 771 772 type Fat struct { 773 X, Y int 774 } 775 type Thin struct { 776 X int 777 } 778 779 // Ancestor queries (those within an entity group) are strongly consistent 780 // by default, which prevents a test from being flaky. 781 // See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency 782 // for more information. 783 parent := NameKey("SQParent", "TestIntegration_GetAllWithFieldMismatch"+suffix, nil) 784 putKeys := make([]*Key, 3) 785 for i := range putKeys { 786 putKeys[i] = IDKey("GetAllThing", int64(10+i), parent) 787 _, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i}) 788 if err != nil { 789 t.Fatalf("client.Put: %v", err) 790 } 791 } 792 793 var got []Thin 794 want := []Thin{ 795 {X: 20}, 796 {X: 21}, 797 {X: 22}, 798 } 799 getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got) 800 if len(getKeys) != 3 && !testutil.Equal(getKeys, putKeys) { 801 t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys) 802 } 803 if !testutil.Equal(got, want) { 804 t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want) 805 } 806 if _, ok := err.(*ErrFieldMismatch); !ok { 807 t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err) 808 } 809} 810 811func TestIntegration_KindlessQueries(t *testing.T) { 812 ctx := context.Background() 813 client := newTestClient(ctx, t) 814 defer client.Close() 815 816 type Dee struct { 817 I int 818 Why string 819 } 820 type Dum struct { 821 I int 822 Pling string 823 } 824 825 parent := NameKey("Tweedle", "tweedle"+suffix, nil) 826 827 keys := []*Key{ 828 NameKey("Dee", "dee0", parent), 829 NameKey("Dum", "dum1", parent), 830 NameKey("Dum", "dum2", parent), 831 NameKey("Dum", "dum3", parent), 832 } 833 src := []interface{}{ 834 &Dee{1, "binary0001"}, 835 &Dum{2, "binary0010"}, 836 &Dum{4, "binary0100"}, 837 &Dum{8, "binary1000"}, 838 } 839 keys, err := client.PutMulti(ctx, keys, src) 840 if err != nil { 841 t.Fatalf("put: %v", err) 842 } 843 844 testCases := []struct { 845 desc string 846 query *Query 847 want []int 848 wantErr string 849 }{ 850 { 851 desc: "Dee", 852 query: NewQuery("Dee"), 853 want: []int{1}, 854 }, 855 { 856 desc: "Doh", 857 query: NewQuery("Doh"), 858 want: nil}, 859 { 860 desc: "Dum", 861 query: NewQuery("Dum"), 862 want: []int{2, 4, 8}, 863 }, 864 { 865 desc: "", 866 query: NewQuery(""), 867 want: []int{1, 2, 4, 8}, 868 }, 869 { 870 desc: "Kindless filter", 871 query: NewQuery("").Filter("__key__ =", keys[2]), 872 want: []int{4}, 873 }, 874 { 875 desc: "Kindless order", 876 query: NewQuery("").Order("__key__"), 877 want: []int{1, 2, 4, 8}, 878 }, 879 { 880 desc: "Kindless bad filter", 881 query: NewQuery("").Filter("I =", 4), 882 wantErr: "kind is required", 883 }, 884 { 885 desc: "Kindless bad order", 886 query: NewQuery("").Order("-__key__"), 887 wantErr: "kind is required for all orders except __key__ ascending", 888 }, 889 } 890 for _, test := range testCases { 891 t.Run(test.desc, func(t *testing.T) { 892 q := test.query.Ancestor(parent) 893 gotCount, err := client.Count(ctx, q) 894 if err != nil { 895 if test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr) { 896 t.Fatalf("count %q: err %v, want err %q", test.desc, err, test.wantErr) 897 } 898 return 899 } 900 if test.wantErr != "" { 901 t.Fatalf("count %q: want err %q", test.desc, test.wantErr) 902 } 903 if gotCount != len(test.want) { 904 t.Fatalf("count %q: got %d want %d", test.desc, gotCount, len(test.want)) 905 } 906 var got []int 907 for iter := client.Run(ctx, q); ; { 908 var dst struct { 909 I int 910 Why, Pling string 911 } 912 _, err := iter.Next(&dst) 913 if err == iterator.Done { 914 break 915 } 916 if err != nil { 917 t.Fatalf("iter.Next %q: %v", test.desc, err) 918 } 919 got = append(got, dst.I) 920 } 921 sort.Ints(got) 922 if !testutil.Equal(got, test.want) { 923 t.Fatalf("elems %q: got %+v want %+v", test.desc, got, test.want) 924 } 925 }) 926 } 927} 928 929func TestIntegration_Transaction(t *testing.T) { 930 ctx := context.Background() 931 client := newTestClient(ctx, t) 932 defer client.Close() 933 934 type Counter struct { 935 N int 936 T time.Time 937 } 938 939 bangErr := errors.New("bang") 940 tests := []struct { 941 desc string 942 causeConflict []bool 943 retErr []error 944 want int 945 wantErr error 946 }{ 947 { 948 desc: "3 attempts, no conflicts", 949 causeConflict: []bool{false}, 950 retErr: []error{nil}, 951 want: 11, 952 }, 953 { 954 desc: "1 attempt, user error", 955 causeConflict: []bool{false}, 956 retErr: []error{bangErr}, 957 wantErr: bangErr, 958 }, 959 { 960 desc: "2 attempts, 1 conflict", 961 causeConflict: []bool{true, false}, 962 retErr: []error{nil, nil}, 963 want: 13, // Each conflict increments by 2. 964 }, 965 { 966 desc: "3 attempts, 3 conflicts", 967 causeConflict: []bool{true, true, true}, 968 retErr: []error{nil, nil, nil}, 969 wantErr: ErrConcurrentTransaction, 970 }, 971 } 972 for i, test := range tests { 973 t.Run(test.desc, func(t *testing.T) { 974 // Put a new counter. 975 c := &Counter{N: 10, T: timeNow} 976 key, err := client.Put(ctx, IncompleteKey("TransCounter", nil), c) 977 if err != nil { 978 t.Fatal(err) 979 } 980 defer client.Delete(ctx, key) 981 982 // Increment the counter in a transaction. 983 // The test case can manually cause a conflict or return an 984 // error at each attempt. 985 var attempts int 986 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 987 attempts++ 988 if attempts > len(test.causeConflict) { 989 return fmt.Errorf("too many attempts. Got %d, max %d", attempts, len(test.causeConflict)) 990 } 991 992 var c Counter 993 if err := tx.Get(key, &c); err != nil { 994 return err 995 } 996 c.N++ 997 if _, err := tx.Put(key, &c); err != nil { 998 return err 999 } 1000 1001 if test.causeConflict[attempts-1] { 1002 c.N++ 1003 if _, err := client.Put(ctx, key, &c); err != nil { 1004 return err 1005 } 1006 } 1007 1008 return test.retErr[attempts-1] 1009 }, MaxAttempts(i)) 1010 1011 // Check the error returned by RunInTransaction. 1012 if err != test.wantErr { 1013 t.Fatalf("got err %v, want %v", err, test.wantErr) 1014 } 1015 if test.wantErr != nil { 1016 // If we were expecting an error, this is where the test ends. 1017 return 1018 } 1019 1020 // Check the final value of the counter. 1021 if err := client.Get(ctx, key, c); err != nil { 1022 t.Fatal(err) 1023 } 1024 if c.N != test.want { 1025 t.Fatalf("counter N=%d, want N=%d", c.N, test.want) 1026 } 1027 }) 1028 } 1029} 1030 1031func TestIntegration_ReadOnlyTransaction(t *testing.T) { 1032 if testing.Short() { 1033 t.Skip("Integration tests skipped in short mode") 1034 } 1035 ctx := context.Background() 1036 client := newClient(ctx, t, nil) 1037 defer client.Close() 1038 1039 type value struct{ N int } 1040 1041 // Put a value. 1042 const n = 5 1043 v := &value{N: n} 1044 key, err := client.Put(ctx, IncompleteKey("roTxn", nil), v) 1045 if err != nil { 1046 t.Fatal(err) 1047 } 1048 defer client.Delete(ctx, key) 1049 1050 // Read it from a read-only transaction. 1051 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 1052 if err := tx.Get(key, v); err != nil { 1053 return err 1054 } 1055 return nil 1056 }, ReadOnly) 1057 if err != nil { 1058 t.Fatal(err) 1059 } 1060 if v.N != n { 1061 t.Fatalf("got %d, want %d", v.N, n) 1062 } 1063 1064 // Attempting to write from a read-only transaction is an error. 1065 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 1066 if _, err := tx.Put(key, v); err != nil { 1067 return err 1068 } 1069 return nil 1070 }, ReadOnly) 1071 if err == nil { 1072 t.Fatal("got nil, want error") 1073 } 1074} 1075 1076func TestIntegration_NilPointers(t *testing.T) { 1077 ctx := context.Background() 1078 client := newTestClient(ctx, t) 1079 defer client.Close() 1080 1081 type X struct { 1082 S string 1083 } 1084 1085 src := []*X{{"zero"}, {"one"}} 1086 keys := []*Key{IncompleteKey("NilX", nil), IncompleteKey("NilX", nil)} 1087 keys, err := client.PutMulti(ctx, keys, src) 1088 if err != nil { 1089 t.Fatalf("PutMulti: %v", err) 1090 } 1091 1092 // It's okay to store into a slice of nil *X. 1093 xs := make([]*X, 2) 1094 if err := client.GetMulti(ctx, keys, xs); err != nil { 1095 t.Errorf("GetMulti: %v", err) 1096 } else if !testutil.Equal(xs, src) { 1097 t.Errorf("GetMulti fetched %v, want %v", xs, src) 1098 } 1099 1100 // It isn't okay to store into a single nil *X. 1101 var x0 *X 1102 if err, want := client.Get(ctx, keys[0], x0), ErrInvalidEntityType; err != want { 1103 t.Errorf("Get: err %v; want %v", err, want) 1104 } 1105 1106 // Test that deleting with duplicate keys work. 1107 keys = append(keys, keys...) 1108 if err := client.DeleteMulti(ctx, keys); err != nil { 1109 t.Errorf("Delete: %v", err) 1110 } 1111} 1112 1113func TestIntegration_NestedRepeatedElementNoIndex(t *testing.T) { 1114 ctx := context.Background() 1115 client := newTestClient(ctx, t) 1116 defer client.Close() 1117 1118 type Inner struct { 1119 Name string 1120 Value string `datastore:",noindex"` 1121 } 1122 type Outer struct { 1123 Config []Inner 1124 } 1125 m := &Outer{ 1126 Config: []Inner{ 1127 {Name: "short", Value: "a"}, 1128 {Name: "long", Value: strings.Repeat("a", 2000)}, 1129 }, 1130 } 1131 1132 key := NameKey("Nested", "Nested"+suffix, nil) 1133 if _, err := client.Put(ctx, key, m); err != nil { 1134 t.Fatalf("client.Put: %v", err) 1135 } 1136 if err := client.Delete(ctx, key); err != nil { 1137 t.Fatalf("client.Delete: %v", err) 1138 } 1139} 1140 1141func TestIntegration_PointerFields(t *testing.T) { 1142 ctx := context.Background() 1143 client := newTestClient(ctx, t) 1144 defer client.Close() 1145 1146 want := populatedPointers() 1147 key, err := client.Put(ctx, IncompleteKey("pointers", nil), want) 1148 if err != nil { 1149 t.Fatal(err) 1150 } 1151 var got Pointers 1152 if err := client.Get(ctx, key, &got); err != nil { 1153 t.Fatal(err) 1154 } 1155 if got.Pi == nil || *got.Pi != *want.Pi { 1156 t.Errorf("Pi: got %v, want %v", got.Pi, *want.Pi) 1157 } 1158 if got.Ps == nil || *got.Ps != *want.Ps { 1159 t.Errorf("Ps: got %v, want %v", got.Ps, *want.Ps) 1160 } 1161 if got.Pb == nil || *got.Pb != *want.Pb { 1162 t.Errorf("Pb: got %v, want %v", got.Pb, *want.Pb) 1163 } 1164 if got.Pf == nil || *got.Pf != *want.Pf { 1165 t.Errorf("Pf: got %v, want %v", got.Pf, *want.Pf) 1166 } 1167 if got.Pg == nil || *got.Pg != *want.Pg { 1168 t.Errorf("Pg: got %v, want %v", got.Pg, *want.Pg) 1169 } 1170 if got.Pt == nil || !got.Pt.Equal(*want.Pt) { 1171 t.Errorf("Pt: got %v, want %v", got.Pt, *want.Pt) 1172 } 1173} 1174 1175func TestIntegration_Mutate(t *testing.T) { 1176 // test Client.Mutate 1177 testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { 1178 return client.Mutate(ctx, muts...) 1179 }) 1180 // test Transaction.Mutate 1181 testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { 1182 var pkeys []*PendingKey 1183 commit, err := client.RunInTransaction(ctx, func(tx *Transaction) error { 1184 var err error 1185 pkeys, err = tx.Mutate(muts...) 1186 return err 1187 }) 1188 if err != nil { 1189 return nil, err 1190 } 1191 var keys []*Key 1192 for _, pk := range pkeys { 1193 keys = append(keys, commit.Key(pk)) 1194 } 1195 return keys, nil 1196 }) 1197} 1198 1199func testMutate(t *testing.T, mutate func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error)) { 1200 ctx := context.Background() 1201 client := newTestClient(ctx, t) 1202 defer client.Close() 1203 1204 type T struct{ I int } 1205 1206 check := func(k *Key, want interface{}) { 1207 var x T 1208 err := client.Get(ctx, k, &x) 1209 switch want := want.(type) { 1210 case error: 1211 if err != want { 1212 t.Errorf("key %s: got error %v, want %v", k, err, want) 1213 } 1214 case int: 1215 if err != nil { 1216 t.Fatalf("key %s: %v", k, err) 1217 } 1218 if x.I != want { 1219 t.Errorf("key %s: got %d, want %d", k, x.I, want) 1220 } 1221 default: 1222 panic("check: bad arg") 1223 } 1224 } 1225 1226 keys, err := mutate(ctx, client, 1227 NewInsert(IncompleteKey("t", nil), &T{1}), 1228 NewUpsert(IncompleteKey("t", nil), &T{2}), 1229 ) 1230 if err != nil { 1231 t.Fatal(err) 1232 } 1233 check(keys[0], 1) 1234 check(keys[1], 2) 1235 1236 _, err = mutate(ctx, client, 1237 NewUpdate(keys[0], &T{3}), 1238 NewDelete(keys[1]), 1239 ) 1240 if err != nil { 1241 t.Fatal(err) 1242 } 1243 check(keys[0], 3) 1244 check(keys[1], ErrNoSuchEntity) 1245 1246 _, err = mutate(ctx, client, NewInsert(keys[0], &T{4})) 1247 if got, want := status.Code(err), codes.AlreadyExists; got != want { 1248 t.Errorf("Insert existing key: got %s, want %s", got, want) 1249 } 1250 1251 _, err = mutate(ctx, client, NewUpdate(keys[1], &T{4})) 1252 if got, want := status.Code(err), codes.NotFound; got != want { 1253 t.Errorf("Update non-existing key: got %s, want %s", got, want) 1254 } 1255} 1256 1257func TestIntegration_DetectProjectID(t *testing.T) { 1258 if testing.Short() { 1259 t.Skip("Integration tests skipped in short mode") 1260 } 1261 ctx := context.Background() 1262 1263 creds := testutil.Credentials(ctx, ScopeDatastore) 1264 if creds == nil { 1265 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 1266 } 1267 1268 // Use creds with project ID. 1269 if _, err := NewClient(ctx, DetectProjectID, option.WithCredentials(creds)); err != nil { 1270 t.Errorf("NewClient: %v", err) 1271 } 1272 1273 ts := testutil.ErroringTokenSource{} 1274 // Try to use creds without project ID. 1275 _, err := NewClient(ctx, DetectProjectID, option.WithTokenSource(ts)) 1276 if err == nil || err.Error() != "datastore: see the docs on DetectProjectID" { 1277 t.Errorf("expected an error while using TokenSource that does not have a project ID") 1278 } 1279} 1280 1281var genKeyName = uid.NewSpace("datastore-integration", nil) 1282 1283func TestIntegration_Project_TimestampStoreAndRetrieve(t *testing.T) { 1284 t.Skip("https://github.com/googleapis/google-cloud-go/issues/1479") 1285 1286 ctx := context.Background() 1287 client := newTestClient(ctx, t) 1288 defer client.Close() 1289 1290 type T struct{ Created time.Time } 1291 1292 // We need to generate a new key to prevent any clashes with concurrent test runs, 1293 // as per: https://github.com/googleapis/google-cloud-go/issues/1479 1294 keyName := genKeyName.New() 1295 1296 now := time.Now() 1297 k, err := client.Put(ctx, IncompleteKey(keyName, nil), &T{Created: now}) 1298 if err != nil { 1299 t.Fatal(err) 1300 } 1301 defer func() { 1302 if err := client.Delete(ctx, k); err != nil { 1303 log.Println(err) 1304 } 1305 }() 1306 1307 q := NewQuery(keyName).Order("Created").Project("Created") 1308 res := []T{} 1309 if _, err := client.GetAll(ctx, q, &res); err != nil { 1310 t.Fatal(err) 1311 } 1312 if len(res) != 1 { 1313 t.Fatalf("expected 1 result, got %d", len(res)) 1314 } 1315 if got, want := res[0].Created.Unix(), now.Unix(); got != want { 1316 t.Fatalf("got %v, want %v", got, want) 1317 } 1318} 1319