1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package datastore 16 17import ( 18 "context" 19 "encoding/json" 20 "errors" 21 "flag" 22 "fmt" 23 "log" 24 "os" 25 "reflect" 26 "sort" 27 "strings" 28 "sync" 29 "testing" 30 "time" 31 32 "cloud.google.com/go/internal/testutil" 33 "cloud.google.com/go/internal/uid" 34 "cloud.google.com/go/rpcreplay" 35 "google.golang.org/api/iterator" 36 "google.golang.org/api/option" 37 "google.golang.org/grpc" 38 "google.golang.org/grpc/codes" 39 "google.golang.org/grpc/status" 40) 41 42// TODO(djd): Make test entity clean up more robust: some test entities may 43// be left behind if tests are aborted, the transport fails, etc. 44 45var timeNow = time.Now() 46 47// suffix is a timestamp-based suffix which is appended to key names, 48// particularly for the root keys of entity groups. This reduces flakiness 49// when the tests are run in parallel. 50var suffix string 51 52const replayFilename = "datastore.replay" 53 54type replayInfo struct { 55 ProjectID string 56 Time time.Time 57} 58 59var ( 60 record = flag.Bool("record", false, "record RPCs") 61 62 newTestClient = func(ctx context.Context, t *testing.T) *Client { 63 return newClient(ctx, t, nil) 64 } 65) 66 67func TestMain(m *testing.M) { 68 os.Exit(testMain(m)) 69} 70 71func testMain(m *testing.M) int { 72 flag.Parse() 73 if testing.Short() { 74 if *record { 75 log.Fatal("cannot combine -short and -record") 76 } 77 if testutil.CanReplay(replayFilename) { 78 initReplay() 79 } 80 } else if *record { 81 if testutil.ProjID() == "" { 82 log.Fatal("must record with a project ID") 83 } 84 b, err := json.Marshal(replayInfo{ 85 ProjectID: testutil.ProjID(), 86 Time: timeNow, 87 }) 88 if err != nil { 89 log.Fatal(err) 90 } 91 rec, err := rpcreplay.NewRecorder(replayFilename, b) 92 if err != nil { 93 log.Fatal(err) 94 } 95 defer func() { 96 if err := rec.Close(); err != nil { 97 log.Fatalf("closing recorder: %v", err) 98 } 99 }() 100 newTestClient = func(ctx context.Context, t *testing.T) *Client { 101 return newClient(ctx, t, rec.DialOptions()) 102 } 103 log.Printf("recording to %s", replayFilename) 104 } 105 suffix = fmt.Sprintf("-t%d", timeNow.UnixNano()) 106 return m.Run() 107} 108 109func initReplay() { 110 rep, err := rpcreplay.NewReplayer(replayFilename) 111 if err != nil { 112 log.Fatal(err) 113 } 114 defer rep.Close() 115 116 var ri replayInfo 117 if err := json.Unmarshal(rep.Initial(), &ri); err != nil { 118 log.Fatalf("unmarshaling initial replay info: %v", err) 119 } 120 timeNow = ri.Time.In(time.Local) 121 122 conn, err := rep.Connection() 123 if err != nil { 124 log.Fatal(err) 125 } 126 127 newTestClient = func(ctx context.Context, t *testing.T) *Client { 128 grpcHeadersEnforcer := &testutil.HeadersEnforcer{ 129 OnFailure: t.Fatalf, 130 Checkers: []*testutil.HeaderChecker{ 131 testutil.XGoogClientHeaderChecker, 132 }, 133 } 134 135 opts := append(grpcHeadersEnforcer.CallOptions(), option.WithGRPCConn(conn)) 136 client, err := NewClient(ctx, ri.ProjectID, opts...) 137 if err != nil { 138 t.Fatalf("NewClient: %v", err) 139 } 140 return client 141 } 142 log.Printf("replaying from %s", replayFilename) 143} 144 145func newClient(ctx context.Context, t *testing.T, dialOpts []grpc.DialOption) *Client { 146 if testing.Short() { 147 t.Skip("Integration tests skipped in short mode") 148 } 149 ts := testutil.TokenSource(ctx, ScopeDatastore) 150 if ts == nil { 151 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 152 } 153 154 grpcHeadersEnforcer := &testutil.HeadersEnforcer{ 155 OnFailure: t.Fatalf, 156 Checkers: []*testutil.HeaderChecker{ 157 testutil.XGoogClientHeaderChecker, 158 }, 159 } 160 opts := append(grpcHeadersEnforcer.CallOptions(), option.WithTokenSource(ts)) 161 for _, opt := range dialOpts { 162 opts = append(opts, option.WithGRPCDialOption(opt)) 163 } 164 client, err := NewClient(ctx, testutil.ProjID(), opts...) 165 if err != nil { 166 t.Fatalf("NewClient: %v", err) 167 } 168 return client 169} 170 171func TestIntegration_Basics(t *testing.T) { 172 ctx, _ := context.WithTimeout(context.Background(), time.Second*20) 173 client := newTestClient(ctx, t) 174 defer client.Close() 175 176 type X struct { 177 I int 178 S string 179 T time.Time 180 U interface{} 181 } 182 183 x0 := X{66, "99", timeNow.Truncate(time.Millisecond), "X"} 184 k, err := client.Put(ctx, IncompleteKey("BasicsX", nil), &x0) 185 if err != nil { 186 t.Fatalf("client.Put: %v", err) 187 } 188 x1 := X{} 189 err = client.Get(ctx, k, &x1) 190 if err != nil { 191 t.Errorf("client.Get: %v", err) 192 } 193 err = client.Delete(ctx, k) 194 if err != nil { 195 t.Errorf("client.Delete: %v", err) 196 } 197 if !testutil.Equal(x0, x1) { 198 t.Errorf("compare: x0=%v, x1=%v", x0, x1) 199 } 200} 201 202func TestIntegration_TopLevelKeyLoaded(t *testing.T) { 203 ctx, _ := context.WithTimeout(context.Background(), time.Second*20) 204 client := newTestClient(ctx, t) 205 defer client.Close() 206 207 completeKey := NameKey("EntityWithKey", "myent", nil) 208 209 type EntityWithKey struct { 210 I int 211 S string 212 K *Key `datastore:"__key__"` 213 } 214 215 in := &EntityWithKey{ 216 I: 12, 217 S: "abcd", 218 } 219 220 k, err := client.Put(ctx, completeKey, in) 221 if err != nil { 222 t.Fatalf("client.Put: %v", err) 223 } 224 225 var e EntityWithKey 226 err = client.Get(ctx, k, &e) 227 if err != nil { 228 t.Fatalf("client.Get: %v", err) 229 } 230 231 // The two keys should be absolutely identical. 232 if !testutil.Equal(e.K, k) { 233 t.Fatalf("e.K not equal to k; got %#v, want %#v", e.K, k) 234 } 235 236} 237 238func TestIntegration_ListValues(t *testing.T) { 239 ctx := context.Background() 240 client := newTestClient(ctx, t) 241 defer client.Close() 242 243 p0 := PropertyList{ 244 {Name: "L", Value: []interface{}{int64(12), "string", true}}, 245 } 246 k, err := client.Put(ctx, IncompleteKey("ListValue", nil), &p0) 247 if err != nil { 248 t.Fatalf("client.Put: %v", err) 249 } 250 var p1 PropertyList 251 if err := client.Get(ctx, k, &p1); err != nil { 252 t.Errorf("client.Get: %v", err) 253 } 254 if !testutil.Equal(p0, p1) { 255 t.Errorf("compare:\np0=%v\np1=%#v", p0, p1) 256 } 257 if err = client.Delete(ctx, k); err != nil { 258 t.Errorf("client.Delete: %v", err) 259 } 260} 261 262func TestIntegration_GetMulti(t *testing.T) { 263 ctx := context.Background() 264 client := newTestClient(ctx, t) 265 defer client.Close() 266 267 type X struct { 268 I int 269 } 270 p := NameKey("X", "x"+suffix, nil) 271 272 cases := []struct { 273 key *Key 274 put bool 275 }{ 276 {key: NameKey("X", "item1", p), put: true}, 277 {key: NameKey("X", "item2", p), put: false}, 278 {key: NameKey("X", "item3", p), put: false}, 279 {key: NameKey("X", "item3", p), put: false}, 280 {key: NameKey("X", "item4", p), put: true}, 281 } 282 283 var src, dst []*X 284 var srcKeys, dstKeys []*Key 285 for _, c := range cases { 286 dst = append(dst, &X{}) 287 dstKeys = append(dstKeys, c.key) 288 if c.put { 289 src = append(src, &X{}) 290 srcKeys = append(srcKeys, c.key) 291 } 292 } 293 if _, err := client.PutMulti(ctx, srcKeys, src); err != nil { 294 t.Error(err) 295 } 296 err := client.GetMulti(ctx, dstKeys, dst) 297 if err == nil { 298 t.Errorf("client.GetMulti got %v, expected error", err) 299 } 300 e, ok := err.(MultiError) 301 if !ok { 302 t.Errorf("client.GetMulti got %T, expected MultiError", err) 303 } 304 for i, err := range e { 305 got, want := err, (error)(nil) 306 if !cases[i].put { 307 got, want = err, ErrNoSuchEntity 308 } 309 if got != want { 310 t.Errorf("MultiError[%d] == %v, want %v", i, got, want) 311 } 312 } 313} 314 315type Z struct { 316 S string 317 T string `datastore:",noindex"` 318 P []byte 319 K []byte `datastore:",noindex"` 320} 321 322func (z Z) String() string { 323 var lens []string 324 v := reflect.ValueOf(z) 325 for i := 0; i < v.NumField(); i++ { 326 if l := v.Field(i).Len(); l > 0 { 327 lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l)) 328 } 329 } 330 return fmt.Sprintf("Z{ %s }", strings.Join(lens, ",")) 331} 332 333func TestIntegration_UnindexableValues(t *testing.T) { 334 ctx := context.Background() 335 client := newTestClient(ctx, t) 336 defer client.Close() 337 338 x1500 := strings.Repeat("x", 1500) 339 x1501 := strings.Repeat("x", 1501) 340 testCases := []struct { 341 in Z 342 wantErr bool 343 }{ 344 {in: Z{S: x1500}, wantErr: false}, 345 {in: Z{S: x1501}, wantErr: true}, 346 {in: Z{T: x1500}, wantErr: false}, 347 {in: Z{T: x1501}, wantErr: false}, 348 {in: Z{P: []byte(x1500)}, wantErr: false}, 349 {in: Z{P: []byte(x1501)}, wantErr: true}, 350 {in: Z{K: []byte(x1500)}, wantErr: false}, 351 {in: Z{K: []byte(x1501)}, wantErr: false}, 352 } 353 for _, tt := range testCases { 354 _, err := client.Put(ctx, IncompleteKey("BasicsZ", nil), &tt.in) 355 if (err != nil) != tt.wantErr { 356 t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) 357 } 358 } 359} 360 361func TestIntegration_NilKey(t *testing.T) { 362 ctx := context.Background() 363 client := newTestClient(ctx, t) 364 defer client.Close() 365 366 testCases := []struct { 367 in K0 368 wantErr bool 369 }{ 370 {in: K0{K: testKey0}, wantErr: false}, 371 {in: K0{}, wantErr: false}, 372 } 373 for _, tt := range testCases { 374 _, err := client.Put(ctx, IncompleteKey("NilKey", nil), &tt.in) 375 if (err != nil) != tt.wantErr { 376 t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) 377 } 378 } 379} 380 381type SQChild struct { 382 I, J int 383 T, U int64 384} 385 386type SQTestCase struct { 387 desc string 388 q *Query 389 wantCount int 390 wantSum int 391} 392 393func testSmallQueries(ctx context.Context, t *testing.T, client *Client, parent *Key, children []*SQChild, 394 testCases []SQTestCase, extraTests ...func()) { 395 keys := make([]*Key, len(children)) 396 for i := range keys { 397 keys[i] = IncompleteKey("SQChild", parent) 398 } 399 keys, err := client.PutMulti(ctx, keys, children) 400 if err != nil { 401 t.Fatalf("client.PutMulti: %v", err) 402 } 403 defer func() { 404 err := client.DeleteMulti(ctx, keys) 405 if err != nil { 406 t.Errorf("client.DeleteMulti: %v", err) 407 } 408 }() 409 410 for _, tc := range testCases { 411 count, err := client.Count(ctx, tc.q) 412 if err != nil { 413 t.Errorf("Count %q: %v", tc.desc, err) 414 continue 415 } 416 if count != tc.wantCount { 417 t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount) 418 continue 419 } 420 } 421 422 for _, tc := range testCases { 423 var got []SQChild 424 _, err := client.GetAll(ctx, tc.q, &got) 425 if err != nil { 426 t.Errorf("client.GetAll %q: %v", tc.desc, err) 427 continue 428 } 429 sum := 0 430 for _, c := range got { 431 sum += c.I + c.J 432 } 433 if sum != tc.wantSum { 434 t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum) 435 continue 436 } 437 } 438 for _, x := range extraTests { 439 x() 440 } 441} 442 443func TestIntegration_Filters(t *testing.T) { 444 ctx := context.Background() 445 client := newTestClient(ctx, t) 446 defer client.Close() 447 448 parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil) 449 now := timeNow.Truncate(time.Millisecond).Unix() 450 children := []*SQChild{ 451 {I: 0, T: now, U: now}, 452 {I: 1, T: now, U: now}, 453 {I: 2, T: now, U: now}, 454 {I: 3, T: now, U: now}, 455 {I: 4, T: now, U: now}, 456 {I: 5, T: now, U: now}, 457 {I: 6, T: now, U: now}, 458 {I: 7, T: now, U: now}, 459 } 460 baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now) 461 testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ 462 { 463 "I>1", 464 baseQuery.Filter("I>", 1), 465 6, 466 2 + 3 + 4 + 5 + 6 + 7, 467 }, 468 { 469 "I>2 AND I<=5", 470 baseQuery.Filter("I>", 2).Filter("I<=", 5), 471 3, 472 3 + 4 + 5, 473 }, 474 { 475 "I>=3 AND I<3", 476 baseQuery.Filter("I>=", 3).Filter("I<", 3), 477 0, 478 0, 479 }, 480 { 481 "I=4", 482 baseQuery.Filter("I=", 4), 483 1, 484 4, 485 }, 486 }, func() { 487 got := []*SQChild{} 488 want := []*SQChild{ 489 {I: 0, T: now, U: now}, 490 {I: 1, T: now, U: now}, 491 {I: 2, T: now, U: now}, 492 {I: 3, T: now, U: now}, 493 {I: 4, T: now, U: now}, 494 {I: 5, T: now, U: now}, 495 {I: 6, T: now, U: now}, 496 {I: 7, T: now, U: now}, 497 } 498 _, err := client.GetAll(ctx, baseQuery.Order("I"), &got) 499 if err != nil { 500 t.Errorf("client.GetAll: %v", err) 501 } 502 if !testutil.Equal(got, want) { 503 t.Errorf("compare: got=%v, want=%v", got, want) 504 } 505 }, func() { 506 got := []*SQChild{} 507 want := []*SQChild{ 508 {I: 7, T: now, U: now}, 509 {I: 6, T: now, U: now}, 510 {I: 5, T: now, U: now}, 511 {I: 4, T: now, U: now}, 512 {I: 3, T: now, U: now}, 513 {I: 2, T: now, U: now}, 514 {I: 1, T: now, U: now}, 515 {I: 0, T: now, U: now}, 516 } 517 _, err := client.GetAll(ctx, baseQuery.Order("-I"), &got) 518 if err != nil { 519 t.Errorf("client.GetAll: %v", err) 520 } 521 if !testutil.Equal(got, want) { 522 t.Errorf("compare: got=%v, want=%v", got, want) 523 } 524 }) 525} 526 527type ckey struct{} 528 529func TestIntegration_LargeQuery(t *testing.T) { 530 ctx := context.Background() 531 client := newTestClient(ctx, t) 532 defer client.Close() 533 534 parent := NameKey("LQParent", "TestIntegration_LargeQuery"+suffix, nil) 535 now := timeNow.Truncate(time.Millisecond).Unix() 536 537 // Make a large number of children entities. 538 const n = 800 539 children := make([]*SQChild, 0, n) 540 keys := make([]*Key, 0, n) 541 for i := 0; i < n; i++ { 542 children = append(children, &SQChild{I: i, T: now, U: now}) 543 keys = append(keys, IncompleteKey("SQChild", parent)) 544 } 545 546 // Store using PutMulti in batches. 547 const batchSize = 500 548 for i := 0; i < n; i = i + 500 { 549 j := i + batchSize 550 if j > n { 551 j = n 552 } 553 fullKeys, err := client.PutMulti(ctx, keys[i:j], children[i:j]) 554 if err != nil { 555 t.Fatalf("PutMulti(%d, %d): %v", i, j, err) 556 } 557 defer func() { 558 err := client.DeleteMulti(ctx, fullKeys) 559 if err != nil { 560 t.Errorf("client.DeleteMulti: %v", err) 561 } 562 }() 563 } 564 565 q := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Order("I") 566 567 // Wait group to allow us to run query tests in parallel below. 568 var wg sync.WaitGroup 569 570 // Check we get the expected count and results for various limits/offsets. 571 queryTests := []struct { 572 limit, offset, want int 573 }{ 574 // Just limit. 575 {limit: 0, want: 0}, 576 {limit: 100, want: 100}, 577 {limit: 501, want: 501}, 578 {limit: n, want: n}, 579 {limit: n * 2, want: n}, 580 {limit: -1, want: n}, 581 // Just offset. 582 {limit: -1, offset: 100, want: n - 100}, 583 {limit: -1, offset: 500, want: n - 500}, 584 {limit: -1, offset: n, want: 0}, 585 // Limit and offset. 586 {limit: 100, offset: 100, want: 100}, 587 {limit: 1000, offset: 100, want: n - 100}, 588 {limit: 500, offset: 500, want: n - 500}, 589 } 590 for _, tt := range queryTests { 591 q := q.Limit(tt.limit).Offset(tt.offset) 592 wg.Add(1) 593 594 go func(limit, offset, want int) { 595 defer wg.Done() 596 // Check Count returns the expected number of results. 597 count, err := client.Count(ctx, q) 598 if err != nil { 599 t.Errorf("client.Count(limit=%d offset=%d): %v", limit, offset, err) 600 return 601 } 602 if count != want { 603 t.Errorf("Count(limit=%d offset=%d) returned %d, want %d", limit, offset, count, want) 604 } 605 606 var got []SQChild 607 _, err = client.GetAll(ctx, q, &got) 608 if err != nil { 609 t.Errorf("client.GetAll(limit=%d offset=%d): %v", limit, offset, err) 610 return 611 } 612 if len(got) != want { 613 t.Errorf("GetAll(limit=%d offset=%d) returned %d, want %d", limit, offset, len(got), want) 614 } 615 for i, child := range got { 616 if got, want := child.I, i+offset; got != want { 617 t.Errorf("GetAll(limit=%d offset=%d) got[%d].I == %d; want %d", limit, offset, i, got, want) 618 break 619 } 620 } 621 }(tt.limit, tt.offset, tt.want) 622 } 623 624 // Also check iterator cursor behaviour. 625 cursorTests := []struct { 626 limit, offset int // Query limit and offset. 627 count int // The number of times to call "next" 628 want int // The I value of the desired element, -1 for "Done". 629 }{ 630 // No limits. 631 {count: 0, limit: -1, want: 0}, 632 {count: 5, limit: -1, want: 5}, 633 {count: 500, limit: -1, want: 500}, 634 {count: 1000, limit: -1, want: -1}, // No more results. 635 // Limits. 636 {count: 5, limit: 5, want: 5}, 637 {count: 500, limit: 5, want: 5}, 638 {count: 1000, limit: 1000, want: -1}, // No more results. 639 // Offsets. 640 {count: 0, offset: 5, limit: -1, want: 5}, 641 {count: 5, offset: 5, limit: -1, want: 10}, 642 {count: 200, offset: 500, limit: -1, want: 700}, 643 {count: 200, offset: 1000, limit: -1, want: -1}, // No more results. 644 } 645 for _, tt := range cursorTests { 646 wg.Add(1) 647 648 go func(count, limit, offset, want int) { 649 defer wg.Done() 650 651 ctx := context.WithValue(ctx, ckey{}, fmt.Sprintf("c=%d,l=%d,o=%d", count, limit, offset)) 652 // Run iterator through count calls to Next. 653 it := client.Run(ctx, q.Limit(limit).Offset(offset).KeysOnly()) 654 for i := 0; i < count; i++ { 655 _, err := it.Next(nil) 656 if err == iterator.Done { 657 break 658 } 659 if err != nil { 660 t.Errorf("count=%d, limit=%d, offset=%d: it.Next failed at i=%d", count, limit, offset, i) 661 return 662 } 663 } 664 665 // Grab the cursor. 666 cursor, err := it.Cursor() 667 if err != nil { 668 t.Errorf("count=%d, limit=%d, offset=%d: it.Cursor: %v", count, limit, offset, err) 669 return 670 } 671 672 // Make a request for the next element. 673 it = client.Run(ctx, q.Limit(1).Start(cursor)) 674 var entity SQChild 675 _, err = it.Next(&entity) 676 switch { 677 case want == -1: 678 if err != iterator.Done { 679 t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor %v, want Done", count, limit, offset, err) 680 } 681 case err != nil: 682 t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor: %v, want nil", count, limit, offset, err) 683 case entity.I != want: 684 t.Errorf("count=%d, limit=%d, offset=%d: got.I = %d, want %d", count, limit, offset, entity.I, want) 685 } 686 }(tt.count, tt.limit, tt.offset, tt.want) 687 } 688 wg.Wait() 689} 690 691func TestIntegration_EventualConsistency(t *testing.T) { 692 // TODO(jba): either make this actually test eventual consistency, or 693 // delete it. Currently it behaves the same with or without the 694 // EventualConsistency call. 695 ctx := context.Background() 696 client := newTestClient(ctx, t) 697 defer client.Close() 698 699 parent := NameKey("SQParent", "TestIntegration_EventualConsistency"+suffix, nil) 700 now := timeNow.Truncate(time.Millisecond).Unix() 701 children := []*SQChild{ 702 {I: 0, T: now, U: now}, 703 {I: 1, T: now, U: now}, 704 {I: 2, T: now, U: now}, 705 } 706 query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency() 707 testSmallQueries(ctx, t, client, parent, children, nil, func() { 708 got, err := client.Count(ctx, query) 709 if err != nil { 710 t.Fatalf("Count: %v", err) 711 } 712 if got < 0 || 3 < got { 713 t.Errorf("Count: got %d, want [0,3]", got) 714 } 715 }) 716} 717 718func TestIntegration_Projection(t *testing.T) { 719 ctx := context.Background() 720 client := newTestClient(ctx, t) 721 defer client.Close() 722 723 parent := NameKey("SQParent", "TestIntegration_Projection"+suffix, nil) 724 now := timeNow.Truncate(time.Millisecond).Unix() 725 children := []*SQChild{ 726 {I: 1 << 0, J: 100, T: now, U: now}, 727 {I: 1 << 1, J: 100, T: now, U: now}, 728 {I: 1 << 2, J: 200, T: now, U: now}, 729 {I: 1 << 3, J: 300, T: now, U: now}, 730 {I: 1 << 4, J: 300, T: now, U: now}, 731 } 732 baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150) 733 testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ 734 { 735 "project", 736 baseQuery.Project("J"), 737 3, 738 200 + 300 + 300, 739 }, 740 { 741 "distinct", 742 baseQuery.Project("J").Distinct(), 743 2, 744 200 + 300, 745 }, 746 { 747 "distinct on", 748 baseQuery.Project("J").DistinctOn("J"), 749 2, 750 200 + 300, 751 }, 752 { 753 "project on meaningful (GD_WHEN) field", 754 baseQuery.Project("U"), 755 3, 756 0, 757 }, 758 }) 759} 760 761func TestIntegration_AllocateIDs(t *testing.T) { 762 ctx := context.Background() 763 client := newTestClient(ctx, t) 764 defer client.Close() 765 766 keys := make([]*Key, 5) 767 for i := range keys { 768 keys[i] = IncompleteKey("AllocID", nil) 769 } 770 keys, err := client.AllocateIDs(ctx, keys) 771 if err != nil { 772 t.Errorf("AllocID #0 failed: %v", err) 773 } 774 if want := len(keys); want != 5 { 775 t.Errorf("Expected to allocate 5 keys, %d keys are found", want) 776 } 777 for _, k := range keys { 778 if k.Incomplete() { 779 t.Errorf("Unexpeceted incomplete key found: %v", k) 780 } 781 } 782} 783 784func TestIntegration_GetAllWithFieldMismatch(t *testing.T) { 785 ctx := context.Background() 786 client := newTestClient(ctx, t) 787 defer client.Close() 788 789 type Fat struct { 790 X, Y int 791 } 792 type Thin struct { 793 X int 794 } 795 796 // Ancestor queries (those within an entity group) are strongly consistent 797 // by default, which prevents a test from being flaky. 798 // See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency 799 // for more information. 800 parent := NameKey("SQParent", "TestIntegration_GetAllWithFieldMismatch"+suffix, nil) 801 putKeys := make([]*Key, 3) 802 for i := range putKeys { 803 putKeys[i] = IDKey("GetAllThing", int64(10+i), parent) 804 _, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i}) 805 if err != nil { 806 t.Fatalf("client.Put: %v", err) 807 } 808 } 809 810 var got []Thin 811 want := []Thin{ 812 {X: 20}, 813 {X: 21}, 814 {X: 22}, 815 } 816 getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got) 817 if len(getKeys) != 3 && !testutil.Equal(getKeys, putKeys) { 818 t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys) 819 } 820 if !testutil.Equal(got, want) { 821 t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want) 822 } 823 if _, ok := err.(*ErrFieldMismatch); !ok { 824 t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err) 825 } 826} 827 828func TestIntegration_KindlessQueries(t *testing.T) { 829 ctx := context.Background() 830 client := newTestClient(ctx, t) 831 defer client.Close() 832 833 type Dee struct { 834 I int 835 Why string 836 } 837 type Dum struct { 838 I int 839 Pling string 840 } 841 842 parent := NameKey("Tweedle", "tweedle"+suffix, nil) 843 844 keys := []*Key{ 845 NameKey("Dee", "dee0", parent), 846 NameKey("Dum", "dum1", parent), 847 NameKey("Dum", "dum2", parent), 848 NameKey("Dum", "dum3", parent), 849 } 850 src := []interface{}{ 851 &Dee{1, "binary0001"}, 852 &Dum{2, "binary0010"}, 853 &Dum{4, "binary0100"}, 854 &Dum{8, "binary1000"}, 855 } 856 keys, err := client.PutMulti(ctx, keys, src) 857 if err != nil { 858 t.Fatalf("put: %v", err) 859 } 860 861 testCases := []struct { 862 desc string 863 query *Query 864 want []int 865 wantErr string 866 }{ 867 { 868 desc: "Dee", 869 query: NewQuery("Dee"), 870 want: []int{1}, 871 }, 872 { 873 desc: "Doh", 874 query: NewQuery("Doh"), 875 want: nil}, 876 { 877 desc: "Dum", 878 query: NewQuery("Dum"), 879 want: []int{2, 4, 8}, 880 }, 881 { 882 desc: "", 883 query: NewQuery(""), 884 want: []int{1, 2, 4, 8}, 885 }, 886 { 887 desc: "Kindless filter", 888 query: NewQuery("").Filter("__key__ =", keys[2]), 889 want: []int{4}, 890 }, 891 { 892 desc: "Kindless order", 893 query: NewQuery("").Order("__key__"), 894 want: []int{1, 2, 4, 8}, 895 }, 896 { 897 desc: "Kindless bad filter", 898 query: NewQuery("").Filter("I =", 4), 899 wantErr: "kind is required", 900 }, 901 { 902 desc: "Kindless bad order", 903 query: NewQuery("").Order("-__key__"), 904 wantErr: "kind is required for all orders except __key__ ascending", 905 }, 906 } 907 for _, test := range testCases { 908 t.Run(test.desc, func(t *testing.T) { 909 q := test.query.Ancestor(parent) 910 gotCount, err := client.Count(ctx, q) 911 if err != nil { 912 if test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr) { 913 t.Fatalf("count %q: err %v, want err %q", test.desc, err, test.wantErr) 914 } 915 return 916 } 917 if test.wantErr != "" { 918 t.Fatalf("count %q: want err %q", test.desc, test.wantErr) 919 } 920 if gotCount != len(test.want) { 921 t.Fatalf("count %q: got %d want %d", test.desc, gotCount, len(test.want)) 922 } 923 var got []int 924 for iter := client.Run(ctx, q); ; { 925 var dst struct { 926 I int 927 Why, Pling string 928 } 929 _, err := iter.Next(&dst) 930 if err == iterator.Done { 931 break 932 } 933 if err != nil { 934 t.Fatalf("iter.Next %q: %v", test.desc, err) 935 } 936 got = append(got, dst.I) 937 } 938 sort.Ints(got) 939 if !testutil.Equal(got, test.want) { 940 t.Fatalf("elems %q: got %+v want %+v", test.desc, got, test.want) 941 } 942 }) 943 } 944} 945 946func TestIntegration_Transaction(t *testing.T) { 947 ctx := context.Background() 948 client := newTestClient(ctx, t) 949 defer client.Close() 950 951 type Counter struct { 952 N int 953 T time.Time 954 } 955 956 bangErr := errors.New("bang") 957 tests := []struct { 958 desc string 959 causeConflict []bool 960 retErr []error 961 want int 962 wantErr error 963 }{ 964 { 965 desc: "3 attempts, no conflicts", 966 causeConflict: []bool{false}, 967 retErr: []error{nil}, 968 want: 11, 969 }, 970 { 971 desc: "1 attempt, user error", 972 causeConflict: []bool{false}, 973 retErr: []error{bangErr}, 974 wantErr: bangErr, 975 }, 976 { 977 desc: "2 attempts, 1 conflict", 978 causeConflict: []bool{true, false}, 979 retErr: []error{nil, nil}, 980 want: 13, // Each conflict increments by 2. 981 }, 982 { 983 desc: "3 attempts, 3 conflicts", 984 causeConflict: []bool{true, true, true}, 985 retErr: []error{nil, nil, nil}, 986 wantErr: ErrConcurrentTransaction, 987 }, 988 } 989 for i, test := range tests { 990 t.Run(test.desc, func(t *testing.T) { 991 // Put a new counter. 992 c := &Counter{N: 10, T: timeNow} 993 key, err := client.Put(ctx, IncompleteKey("TransCounter", nil), c) 994 if err != nil { 995 t.Fatal(err) 996 } 997 defer client.Delete(ctx, key) 998 999 // Increment the counter in a transaction. 1000 // The test case can manually cause a conflict or return an 1001 // error at each attempt. 1002 var attempts int 1003 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 1004 attempts++ 1005 if attempts > len(test.causeConflict) { 1006 return fmt.Errorf("too many attempts. Got %d, max %d", attempts, len(test.causeConflict)) 1007 } 1008 1009 var c Counter 1010 if err := tx.Get(key, &c); err != nil { 1011 return err 1012 } 1013 c.N++ 1014 if _, err := tx.Put(key, &c); err != nil { 1015 return err 1016 } 1017 1018 if test.causeConflict[attempts-1] { 1019 c.N++ 1020 if _, err := client.Put(ctx, key, &c); err != nil { 1021 return err 1022 } 1023 } 1024 1025 return test.retErr[attempts-1] 1026 }, MaxAttempts(i)) 1027 1028 // Check the error returned by RunInTransaction. 1029 if err != test.wantErr { 1030 t.Fatalf("got err %v, want %v", err, test.wantErr) 1031 } 1032 if test.wantErr != nil { 1033 // If we were expecting an error, this is where the test ends. 1034 return 1035 } 1036 1037 // Check the final value of the counter. 1038 if err := client.Get(ctx, key, c); err != nil { 1039 t.Fatal(err) 1040 } 1041 if c.N != test.want { 1042 t.Fatalf("counter N=%d, want N=%d", c.N, test.want) 1043 } 1044 }) 1045 } 1046} 1047 1048func TestIntegration_ReadOnlyTransaction(t *testing.T) { 1049 if testing.Short() { 1050 t.Skip("Integration tests skipped in short mode") 1051 } 1052 ctx := context.Background() 1053 client := newClient(ctx, t, nil) 1054 defer client.Close() 1055 1056 type value struct{ N int } 1057 1058 // Put a value. 1059 const n = 5 1060 v := &value{N: n} 1061 key, err := client.Put(ctx, IncompleteKey("roTxn", nil), v) 1062 if err != nil { 1063 t.Fatal(err) 1064 } 1065 defer client.Delete(ctx, key) 1066 1067 // Read it from a read-only transaction. 1068 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 1069 if err := tx.Get(key, v); err != nil { 1070 return err 1071 } 1072 return nil 1073 }, ReadOnly) 1074 if err != nil { 1075 t.Fatal(err) 1076 } 1077 if v.N != n { 1078 t.Fatalf("got %d, want %d", v.N, n) 1079 } 1080 1081 // Attempting to write from a read-only transaction is an error. 1082 _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { 1083 if _, err := tx.Put(key, v); err != nil { 1084 return err 1085 } 1086 return nil 1087 }, ReadOnly) 1088 if err == nil { 1089 t.Fatal("got nil, want error") 1090 } 1091} 1092 1093func TestIntegration_NilPointers(t *testing.T) { 1094 ctx := context.Background() 1095 client := newTestClient(ctx, t) 1096 defer client.Close() 1097 1098 type X struct { 1099 S string 1100 } 1101 1102 src := []*X{{"zero"}, {"one"}} 1103 keys := []*Key{IncompleteKey("NilX", nil), IncompleteKey("NilX", nil)} 1104 keys, err := client.PutMulti(ctx, keys, src) 1105 if err != nil { 1106 t.Fatalf("PutMulti: %v", err) 1107 } 1108 1109 // It's okay to store into a slice of nil *X. 1110 xs := make([]*X, 2) 1111 if err := client.GetMulti(ctx, keys, xs); err != nil { 1112 t.Errorf("GetMulti: %v", err) 1113 } else if !testutil.Equal(xs, src) { 1114 t.Errorf("GetMulti fetched %v, want %v", xs, src) 1115 } 1116 1117 // It isn't okay to store into a single nil *X. 1118 var x0 *X 1119 if err, want := client.Get(ctx, keys[0], x0), ErrInvalidEntityType; err != want { 1120 t.Errorf("Get: err %v; want %v", err, want) 1121 } 1122 1123 // Test that deleting with duplicate keys work. 1124 keys = append(keys, keys...) 1125 if err := client.DeleteMulti(ctx, keys); err != nil { 1126 t.Errorf("Delete: %v", err) 1127 } 1128} 1129 1130func TestIntegration_NestedRepeatedElementNoIndex(t *testing.T) { 1131 ctx := context.Background() 1132 client := newTestClient(ctx, t) 1133 defer client.Close() 1134 1135 type Inner struct { 1136 Name string 1137 Value string `datastore:",noindex"` 1138 } 1139 type Outer struct { 1140 Config []Inner 1141 } 1142 m := &Outer{ 1143 Config: []Inner{ 1144 {Name: "short", Value: "a"}, 1145 {Name: "long", Value: strings.Repeat("a", 2000)}, 1146 }, 1147 } 1148 1149 key := NameKey("Nested", "Nested"+suffix, nil) 1150 if _, err := client.Put(ctx, key, m); err != nil { 1151 t.Fatalf("client.Put: %v", err) 1152 } 1153 if err := client.Delete(ctx, key); err != nil { 1154 t.Fatalf("client.Delete: %v", err) 1155 } 1156} 1157 1158func TestIntegration_PointerFields(t *testing.T) { 1159 ctx := context.Background() 1160 client := newTestClient(ctx, t) 1161 defer client.Close() 1162 1163 want := populatedPointers() 1164 key, err := client.Put(ctx, IncompleteKey("pointers", nil), want) 1165 if err != nil { 1166 t.Fatal(err) 1167 } 1168 var got Pointers 1169 if err := client.Get(ctx, key, &got); err != nil { 1170 t.Fatal(err) 1171 } 1172 if got.Pi == nil || *got.Pi != *want.Pi { 1173 t.Errorf("Pi: got %v, want %v", got.Pi, *want.Pi) 1174 } 1175 if got.Ps == nil || *got.Ps != *want.Ps { 1176 t.Errorf("Ps: got %v, want %v", got.Ps, *want.Ps) 1177 } 1178 if got.Pb == nil || *got.Pb != *want.Pb { 1179 t.Errorf("Pb: got %v, want %v", got.Pb, *want.Pb) 1180 } 1181 if got.Pf == nil || *got.Pf != *want.Pf { 1182 t.Errorf("Pf: got %v, want %v", got.Pf, *want.Pf) 1183 } 1184 if got.Pg == nil || *got.Pg != *want.Pg { 1185 t.Errorf("Pg: got %v, want %v", got.Pg, *want.Pg) 1186 } 1187 if got.Pt == nil || !got.Pt.Equal(*want.Pt) { 1188 t.Errorf("Pt: got %v, want %v", got.Pt, *want.Pt) 1189 } 1190} 1191 1192func TestIntegration_Mutate(t *testing.T) { 1193 // test Client.Mutate 1194 testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { 1195 return client.Mutate(ctx, muts...) 1196 }) 1197 // test Transaction.Mutate 1198 testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { 1199 var pkeys []*PendingKey 1200 commit, err := client.RunInTransaction(ctx, func(tx *Transaction) error { 1201 var err error 1202 pkeys, err = tx.Mutate(muts...) 1203 return err 1204 }) 1205 if err != nil { 1206 return nil, err 1207 } 1208 var keys []*Key 1209 for _, pk := range pkeys { 1210 keys = append(keys, commit.Key(pk)) 1211 } 1212 return keys, nil 1213 }) 1214} 1215 1216func testMutate(t *testing.T, mutate func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error)) { 1217 ctx := context.Background() 1218 client := newTestClient(ctx, t) 1219 defer client.Close() 1220 1221 type T struct{ I int } 1222 1223 check := func(k *Key, want interface{}) { 1224 var x T 1225 err := client.Get(ctx, k, &x) 1226 switch want := want.(type) { 1227 case error: 1228 if err != want { 1229 t.Errorf("key %s: got error %v, want %v", k, err, want) 1230 } 1231 case int: 1232 if err != nil { 1233 t.Fatalf("key %s: %v", k, err) 1234 } 1235 if x.I != want { 1236 t.Errorf("key %s: got %d, want %d", k, x.I, want) 1237 } 1238 default: 1239 panic("check: bad arg") 1240 } 1241 } 1242 1243 keys, err := mutate(ctx, client, 1244 NewInsert(IncompleteKey("t", nil), &T{1}), 1245 NewUpsert(IncompleteKey("t", nil), &T{2}), 1246 ) 1247 if err != nil { 1248 t.Fatal(err) 1249 } 1250 check(keys[0], 1) 1251 check(keys[1], 2) 1252 1253 _, err = mutate(ctx, client, 1254 NewUpdate(keys[0], &T{3}), 1255 NewDelete(keys[1]), 1256 ) 1257 if err != nil { 1258 t.Fatal(err) 1259 } 1260 check(keys[0], 3) 1261 check(keys[1], ErrNoSuchEntity) 1262 1263 _, err = mutate(ctx, client, NewInsert(keys[0], &T{4})) 1264 if got, want := status.Code(err), codes.AlreadyExists; got != want { 1265 t.Errorf("Insert existing key: got %s, want %s", got, want) 1266 } 1267 1268 _, err = mutate(ctx, client, NewUpdate(keys[1], &T{4})) 1269 if got, want := status.Code(err), codes.NotFound; got != want { 1270 t.Errorf("Update non-existing key: got %s, want %s", got, want) 1271 } 1272} 1273 1274func TestIntegration_DetectProjectID(t *testing.T) { 1275 if testing.Short() { 1276 t.Skip("Integration tests skipped in short mode") 1277 } 1278 ctx := context.Background() 1279 1280 creds := testutil.Credentials(ctx, ScopeDatastore) 1281 if creds == nil { 1282 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 1283 } 1284 1285 // Use creds with project ID. 1286 if _, err := NewClient(ctx, DetectProjectID, option.WithCredentials(creds)); err != nil { 1287 t.Errorf("NewClient: %v", err) 1288 } 1289 1290 ts := testutil.ErroringTokenSource{} 1291 // Try to use creds without project ID. 1292 _, err := NewClient(ctx, DetectProjectID, option.WithTokenSource(ts)) 1293 if err == nil || err.Error() != "datastore: see the docs on DetectProjectID" { 1294 t.Errorf("expected an error while using TokenSource that does not have a project ID") 1295 } 1296} 1297 1298var genKeyName = uid.NewSpace("datastore-integration", nil) 1299 1300func TestIntegration_Project_TimestampStoreAndRetrieve(t *testing.T) { 1301 ctx := context.Background() 1302 client := newTestClient(ctx, t) 1303 defer client.Close() 1304 1305 type T struct{ Created time.Time } 1306 1307 keyName := genKeyName.New() 1308 1309 now := time.Now() 1310 k, err := client.Put(ctx, IncompleteKey(keyName, nil), &T{Created: now}) 1311 if err != nil { 1312 t.Fatal(err) 1313 } 1314 defer func() { 1315 if err := client.Delete(ctx, k); err != nil { 1316 log.Println(err) 1317 } 1318 }() 1319 1320 // Without .Ancestor, this query is eventually consistent (so this test 1321 // would be flakey). Ancestor queries, however, are strongly consistent. 1322 // See more at https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/. 1323 q := NewQuery(k.Kind).Ancestor(k) 1324 res := []T{} 1325 if _, err := client.GetAll(ctx, q, &res); err != nil { 1326 t.Fatal(err) 1327 } 1328 if len(res) != 1 { 1329 t.Fatalf("expected 1 result, got %d", len(res)) 1330 } 1331 if got, want := res[0].Created.Unix(), now.Unix(); got != want { 1332 t.Fatalf("got %v, want %v", got, want) 1333 } 1334} 1335