1// Copyright 2014 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package datastore
16
17import (
18	"context"
19	"encoding/json"
20	"errors"
21	"flag"
22	"fmt"
23	"log"
24	"os"
25	"reflect"
26	"sort"
27	"strings"
28	"sync"
29	"testing"
30	"time"
31
32	"cloud.google.com/go/internal/testutil"
33	"cloud.google.com/go/internal/uid"
34	"cloud.google.com/go/rpcreplay"
35	"google.golang.org/api/iterator"
36	"google.golang.org/api/option"
37	"google.golang.org/grpc"
38	"google.golang.org/grpc/codes"
39	"google.golang.org/grpc/status"
40)
41
42// TODO(djd): Make test entity clean up more robust: some test entities may
43// be left behind if tests are aborted, the transport fails, etc.
44
45var timeNow = time.Now()
46
47// suffix is a timestamp-based suffix which is appended to key names,
48// particularly for the root keys of entity groups. This reduces flakiness
49// when the tests are run in parallel.
50var suffix string
51
52const replayFilename = "datastore.replay"
53
54type replayInfo struct {
55	ProjectID string
56	Time      time.Time
57}
58
59var (
60	record = flag.Bool("record", false, "record RPCs")
61
62	newTestClient = func(ctx context.Context, t *testing.T) *Client {
63		return newClient(ctx, t, nil)
64	}
65)
66
67func TestMain(m *testing.M) {
68	os.Exit(testMain(m))
69}
70
71func testMain(m *testing.M) int {
72	flag.Parse()
73	if testing.Short() {
74		if *record {
75			log.Fatal("cannot combine -short and -record")
76		}
77		if testutil.CanReplay(replayFilename) {
78			initReplay()
79		}
80	} else if *record {
81		if testutil.ProjID() == "" {
82			log.Fatal("must record with a project ID")
83		}
84		b, err := json.Marshal(replayInfo{
85			ProjectID: testutil.ProjID(),
86			Time:      timeNow,
87		})
88		if err != nil {
89			log.Fatal(err)
90		}
91		rec, err := rpcreplay.NewRecorder(replayFilename, b)
92		if err != nil {
93			log.Fatal(err)
94		}
95		defer func() {
96			if err := rec.Close(); err != nil {
97				log.Fatalf("closing recorder: %v", err)
98			}
99		}()
100		newTestClient = func(ctx context.Context, t *testing.T) *Client {
101			return newClient(ctx, t, rec.DialOptions())
102		}
103		log.Printf("recording to %s", replayFilename)
104	}
105	suffix = fmt.Sprintf("-t%d", timeNow.UnixNano())
106	return m.Run()
107}
108
109func initReplay() {
110	rep, err := rpcreplay.NewReplayer(replayFilename)
111	if err != nil {
112		log.Fatal(err)
113	}
114	defer rep.Close()
115
116	var ri replayInfo
117	if err := json.Unmarshal(rep.Initial(), &ri); err != nil {
118		log.Fatalf("unmarshaling initial replay info: %v", err)
119	}
120	timeNow = ri.Time.In(time.Local)
121
122	conn, err := rep.Connection()
123	if err != nil {
124		log.Fatal(err)
125	}
126	newTestClient = func(ctx context.Context, t *testing.T) *Client {
127		client, err := NewClient(ctx, ri.ProjectID, option.WithGRPCConn(conn))
128		if err != nil {
129			t.Fatalf("NewClient: %v", err)
130		}
131		return client
132	}
133	log.Printf("replaying from %s", replayFilename)
134}
135
136func newClient(ctx context.Context, t *testing.T, dialOpts []grpc.DialOption) *Client {
137	if testing.Short() {
138		t.Skip("Integration tests skipped in short mode")
139	}
140	ts := testutil.TokenSource(ctx, ScopeDatastore)
141	if ts == nil {
142		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
143	}
144	opts := []option.ClientOption{option.WithTokenSource(ts)}
145	for _, opt := range dialOpts {
146		opts = append(opts, option.WithGRPCDialOption(opt))
147	}
148	client, err := NewClient(ctx, testutil.ProjID(), opts...)
149	if err != nil {
150		t.Fatalf("NewClient: %v", err)
151	}
152	return client
153}
154
155func TestIntegration_Basics(t *testing.T) {
156	ctx, _ := context.WithTimeout(context.Background(), time.Second*20)
157	client := newTestClient(ctx, t)
158	defer client.Close()
159
160	type X struct {
161		I int
162		S string
163		T time.Time
164	}
165
166	x0 := X{66, "99", timeNow.Truncate(time.Millisecond)}
167	k, err := client.Put(ctx, IncompleteKey("BasicsX", nil), &x0)
168	if err != nil {
169		t.Fatalf("client.Put: %v", err)
170	}
171	x1 := X{}
172	err = client.Get(ctx, k, &x1)
173	if err != nil {
174		t.Errorf("client.Get: %v", err)
175	}
176	err = client.Delete(ctx, k)
177	if err != nil {
178		t.Errorf("client.Delete: %v", err)
179	}
180	if !testutil.Equal(x0, x1) {
181		t.Errorf("compare: x0=%v, x1=%v", x0, x1)
182	}
183}
184
185func TestIntegration_TopLevelKeyLoaded(t *testing.T) {
186	ctx, _ := context.WithTimeout(context.Background(), time.Second*20)
187	client := newTestClient(ctx, t)
188	defer client.Close()
189
190	completeKey := NameKey("EntityWithKey", "myent", nil)
191
192	type EntityWithKey struct {
193		I int
194		S string
195		K *Key `datastore:"__key__"`
196	}
197
198	in := &EntityWithKey{
199		I: 12,
200		S: "abcd",
201	}
202
203	k, err := client.Put(ctx, completeKey, in)
204	if err != nil {
205		t.Fatalf("client.Put: %v", err)
206	}
207
208	var e EntityWithKey
209	err = client.Get(ctx, k, &e)
210	if err != nil {
211		t.Fatalf("client.Get: %v", err)
212	}
213
214	// The two keys should be absolutely identical.
215	if !testutil.Equal(e.K, k) {
216		t.Fatalf("e.K not equal to k; got %#v, want %#v", e.K, k)
217	}
218
219}
220
221func TestIntegration_ListValues(t *testing.T) {
222	ctx := context.Background()
223	client := newTestClient(ctx, t)
224	defer client.Close()
225
226	p0 := PropertyList{
227		{Name: "L", Value: []interface{}{int64(12), "string", true}},
228	}
229	k, err := client.Put(ctx, IncompleteKey("ListValue", nil), &p0)
230	if err != nil {
231		t.Fatalf("client.Put: %v", err)
232	}
233	var p1 PropertyList
234	if err := client.Get(ctx, k, &p1); err != nil {
235		t.Errorf("client.Get: %v", err)
236	}
237	if !testutil.Equal(p0, p1) {
238		t.Errorf("compare:\np0=%v\np1=%#v", p0, p1)
239	}
240	if err = client.Delete(ctx, k); err != nil {
241		t.Errorf("client.Delete: %v", err)
242	}
243}
244
245func TestIntegration_GetMulti(t *testing.T) {
246	ctx := context.Background()
247	client := newTestClient(ctx, t)
248	defer client.Close()
249
250	type X struct {
251		I int
252	}
253	p := NameKey("X", "x"+suffix, nil)
254
255	cases := []struct {
256		key *Key
257		put bool
258	}{
259		{key: NameKey("X", "item1", p), put: true},
260		{key: NameKey("X", "item2", p), put: false},
261		{key: NameKey("X", "item3", p), put: false},
262		{key: NameKey("X", "item3", p), put: false},
263		{key: NameKey("X", "item4", p), put: true},
264	}
265
266	var src, dst []*X
267	var srcKeys, dstKeys []*Key
268	for _, c := range cases {
269		dst = append(dst, &X{})
270		dstKeys = append(dstKeys, c.key)
271		if c.put {
272			src = append(src, &X{})
273			srcKeys = append(srcKeys, c.key)
274		}
275	}
276	if _, err := client.PutMulti(ctx, srcKeys, src); err != nil {
277		t.Error(err)
278	}
279	err := client.GetMulti(ctx, dstKeys, dst)
280	if err == nil {
281		t.Errorf("client.GetMulti got %v, expected error", err)
282	}
283	e, ok := err.(MultiError)
284	if !ok {
285		t.Errorf("client.GetMulti got %T, expected MultiError", err)
286	}
287	for i, err := range e {
288		got, want := err, (error)(nil)
289		if !cases[i].put {
290			got, want = err, ErrNoSuchEntity
291		}
292		if got != want {
293			t.Errorf("MultiError[%d] == %v, want %v", i, got, want)
294		}
295	}
296}
297
298type Z struct {
299	S string
300	T string `datastore:",noindex"`
301	P []byte
302	K []byte `datastore:",noindex"`
303}
304
305func (z Z) String() string {
306	var lens []string
307	v := reflect.ValueOf(z)
308	for i := 0; i < v.NumField(); i++ {
309		if l := v.Field(i).Len(); l > 0 {
310			lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l))
311		}
312	}
313	return fmt.Sprintf("Z{ %s }", strings.Join(lens, ","))
314}
315
316func TestIntegration_UnindexableValues(t *testing.T) {
317	ctx := context.Background()
318	client := newTestClient(ctx, t)
319	defer client.Close()
320
321	x1500 := strings.Repeat("x", 1500)
322	x1501 := strings.Repeat("x", 1501)
323	testCases := []struct {
324		in      Z
325		wantErr bool
326	}{
327		{in: Z{S: x1500}, wantErr: false},
328		{in: Z{S: x1501}, wantErr: true},
329		{in: Z{T: x1500}, wantErr: false},
330		{in: Z{T: x1501}, wantErr: false},
331		{in: Z{P: []byte(x1500)}, wantErr: false},
332		{in: Z{P: []byte(x1501)}, wantErr: true},
333		{in: Z{K: []byte(x1500)}, wantErr: false},
334		{in: Z{K: []byte(x1501)}, wantErr: false},
335	}
336	for _, tt := range testCases {
337		_, err := client.Put(ctx, IncompleteKey("BasicsZ", nil), &tt.in)
338		if (err != nil) != tt.wantErr {
339			t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr)
340		}
341	}
342}
343
344func TestIntegration_NilKey(t *testing.T) {
345	ctx := context.Background()
346	client := newTestClient(ctx, t)
347	defer client.Close()
348
349	testCases := []struct {
350		in      K0
351		wantErr bool
352	}{
353		{in: K0{K: testKey0}, wantErr: false},
354		{in: K0{}, wantErr: false},
355	}
356	for _, tt := range testCases {
357		_, err := client.Put(ctx, IncompleteKey("NilKey", nil), &tt.in)
358		if (err != nil) != tt.wantErr {
359			t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr)
360		}
361	}
362}
363
364type SQChild struct {
365	I, J int
366	T, U int64
367}
368
369type SQTestCase struct {
370	desc      string
371	q         *Query
372	wantCount int
373	wantSum   int
374}
375
376func testSmallQueries(ctx context.Context, t *testing.T, client *Client, parent *Key, children []*SQChild,
377	testCases []SQTestCase, extraTests ...func()) {
378	keys := make([]*Key, len(children))
379	for i := range keys {
380		keys[i] = IncompleteKey("SQChild", parent)
381	}
382	keys, err := client.PutMulti(ctx, keys, children)
383	if err != nil {
384		t.Fatalf("client.PutMulti: %v", err)
385	}
386	defer func() {
387		err := client.DeleteMulti(ctx, keys)
388		if err != nil {
389			t.Errorf("client.DeleteMulti: %v", err)
390		}
391	}()
392
393	for _, tc := range testCases {
394		count, err := client.Count(ctx, tc.q)
395		if err != nil {
396			t.Errorf("Count %q: %v", tc.desc, err)
397			continue
398		}
399		if count != tc.wantCount {
400			t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount)
401			continue
402		}
403	}
404
405	for _, tc := range testCases {
406		var got []SQChild
407		_, err := client.GetAll(ctx, tc.q, &got)
408		if err != nil {
409			t.Errorf("client.GetAll %q: %v", tc.desc, err)
410			continue
411		}
412		sum := 0
413		for _, c := range got {
414			sum += c.I + c.J
415		}
416		if sum != tc.wantSum {
417			t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum)
418			continue
419		}
420	}
421	for _, x := range extraTests {
422		x()
423	}
424}
425
426func TestIntegration_Filters(t *testing.T) {
427	ctx := context.Background()
428	client := newTestClient(ctx, t)
429	defer client.Close()
430
431	parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil)
432	now := timeNow.Truncate(time.Millisecond).Unix()
433	children := []*SQChild{
434		{I: 0, T: now, U: now},
435		{I: 1, T: now, U: now},
436		{I: 2, T: now, U: now},
437		{I: 3, T: now, U: now},
438		{I: 4, T: now, U: now},
439		{I: 5, T: now, U: now},
440		{I: 6, T: now, U: now},
441		{I: 7, T: now, U: now},
442	}
443	baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now)
444	testSmallQueries(ctx, t, client, parent, children, []SQTestCase{
445		{
446			"I>1",
447			baseQuery.Filter("I>", 1),
448			6,
449			2 + 3 + 4 + 5 + 6 + 7,
450		},
451		{
452			"I>2 AND I<=5",
453			baseQuery.Filter("I>", 2).Filter("I<=", 5),
454			3,
455			3 + 4 + 5,
456		},
457		{
458			"I>=3 AND I<3",
459			baseQuery.Filter("I>=", 3).Filter("I<", 3),
460			0,
461			0,
462		},
463		{
464			"I=4",
465			baseQuery.Filter("I=", 4),
466			1,
467			4,
468		},
469	}, func() {
470		got := []*SQChild{}
471		want := []*SQChild{
472			{I: 0, T: now, U: now},
473			{I: 1, T: now, U: now},
474			{I: 2, T: now, U: now},
475			{I: 3, T: now, U: now},
476			{I: 4, T: now, U: now},
477			{I: 5, T: now, U: now},
478			{I: 6, T: now, U: now},
479			{I: 7, T: now, U: now},
480		}
481		_, err := client.GetAll(ctx, baseQuery.Order("I"), &got)
482		if err != nil {
483			t.Errorf("client.GetAll: %v", err)
484		}
485		if !testutil.Equal(got, want) {
486			t.Errorf("compare: got=%v, want=%v", got, want)
487		}
488	}, func() {
489		got := []*SQChild{}
490		want := []*SQChild{
491			{I: 7, T: now, U: now},
492			{I: 6, T: now, U: now},
493			{I: 5, T: now, U: now},
494			{I: 4, T: now, U: now},
495			{I: 3, T: now, U: now},
496			{I: 2, T: now, U: now},
497			{I: 1, T: now, U: now},
498			{I: 0, T: now, U: now},
499		}
500		_, err := client.GetAll(ctx, baseQuery.Order("-I"), &got)
501		if err != nil {
502			t.Errorf("client.GetAll: %v", err)
503		}
504		if !testutil.Equal(got, want) {
505			t.Errorf("compare: got=%v, want=%v", got, want)
506		}
507	})
508}
509
510type ckey struct{}
511
512func TestIntegration_LargeQuery(t *testing.T) {
513	ctx := context.Background()
514	client := newTestClient(ctx, t)
515	defer client.Close()
516
517	parent := NameKey("LQParent", "TestIntegration_LargeQuery"+suffix, nil)
518	now := timeNow.Truncate(time.Millisecond).Unix()
519
520	// Make a large number of children entities.
521	const n = 800
522	children := make([]*SQChild, 0, n)
523	keys := make([]*Key, 0, n)
524	for i := 0; i < n; i++ {
525		children = append(children, &SQChild{I: i, T: now, U: now})
526		keys = append(keys, IncompleteKey("SQChild", parent))
527	}
528
529	// Store using PutMulti in batches.
530	const batchSize = 500
531	for i := 0; i < n; i = i + 500 {
532		j := i + batchSize
533		if j > n {
534			j = n
535		}
536		fullKeys, err := client.PutMulti(ctx, keys[i:j], children[i:j])
537		if err != nil {
538			t.Fatalf("PutMulti(%d, %d): %v", i, j, err)
539		}
540		defer func() {
541			err := client.DeleteMulti(ctx, fullKeys)
542			if err != nil {
543				t.Errorf("client.DeleteMulti: %v", err)
544			}
545		}()
546	}
547
548	q := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Order("I")
549
550	// Wait group to allow us to run query tests in parallel below.
551	var wg sync.WaitGroup
552
553	// Check we get the expected count and results for various limits/offsets.
554	queryTests := []struct {
555		limit, offset, want int
556	}{
557		// Just limit.
558		{limit: 0, want: 0},
559		{limit: 100, want: 100},
560		{limit: 501, want: 501},
561		{limit: n, want: n},
562		{limit: n * 2, want: n},
563		{limit: -1, want: n},
564		// Just offset.
565		{limit: -1, offset: 100, want: n - 100},
566		{limit: -1, offset: 500, want: n - 500},
567		{limit: -1, offset: n, want: 0},
568		// Limit and offset.
569		{limit: 100, offset: 100, want: 100},
570		{limit: 1000, offset: 100, want: n - 100},
571		{limit: 500, offset: 500, want: n - 500},
572	}
573	for _, tt := range queryTests {
574		q := q.Limit(tt.limit).Offset(tt.offset)
575		wg.Add(1)
576
577		go func(limit, offset, want int) {
578			defer wg.Done()
579			// Check Count returns the expected number of results.
580			count, err := client.Count(ctx, q)
581			if err != nil {
582				t.Errorf("client.Count(limit=%d offset=%d): %v", limit, offset, err)
583				return
584			}
585			if count != want {
586				t.Errorf("Count(limit=%d offset=%d) returned %d, want %d", limit, offset, count, want)
587			}
588
589			var got []SQChild
590			_, err = client.GetAll(ctx, q, &got)
591			if err != nil {
592				t.Errorf("client.GetAll(limit=%d offset=%d): %v", limit, offset, err)
593				return
594			}
595			if len(got) != want {
596				t.Errorf("GetAll(limit=%d offset=%d) returned %d, want %d", limit, offset, len(got), want)
597			}
598			for i, child := range got {
599				if got, want := child.I, i+offset; got != want {
600					t.Errorf("GetAll(limit=%d offset=%d) got[%d].I == %d; want %d", limit, offset, i, got, want)
601					break
602				}
603			}
604		}(tt.limit, tt.offset, tt.want)
605	}
606
607	// Also check iterator cursor behaviour.
608	cursorTests := []struct {
609		limit, offset int // Query limit and offset.
610		count         int // The number of times to call "next"
611		want          int // The I value of the desired element, -1 for "Done".
612	}{
613		// No limits.
614		{count: 0, limit: -1, want: 0},
615		{count: 5, limit: -1, want: 5},
616		{count: 500, limit: -1, want: 500},
617		{count: 1000, limit: -1, want: -1}, // No more results.
618		// Limits.
619		{count: 5, limit: 5, want: 5},
620		{count: 500, limit: 5, want: 5},
621		{count: 1000, limit: 1000, want: -1}, // No more results.
622		// Offsets.
623		{count: 0, offset: 5, limit: -1, want: 5},
624		{count: 5, offset: 5, limit: -1, want: 10},
625		{count: 200, offset: 500, limit: -1, want: 700},
626		{count: 200, offset: 1000, limit: -1, want: -1}, // No more results.
627	}
628	for _, tt := range cursorTests {
629		wg.Add(1)
630
631		go func(count, limit, offset, want int) {
632			defer wg.Done()
633
634			ctx := context.WithValue(ctx, ckey{}, fmt.Sprintf("c=%d,l=%d,o=%d", count, limit, offset))
635			// Run iterator through count calls to Next.
636			it := client.Run(ctx, q.Limit(limit).Offset(offset).KeysOnly())
637			for i := 0; i < count; i++ {
638				_, err := it.Next(nil)
639				if err == iterator.Done {
640					break
641				}
642				if err != nil {
643					t.Errorf("count=%d, limit=%d, offset=%d: it.Next failed at i=%d", count, limit, offset, i)
644					return
645				}
646			}
647
648			// Grab the cursor.
649			cursor, err := it.Cursor()
650			if err != nil {
651				t.Errorf("count=%d, limit=%d, offset=%d: it.Cursor: %v", count, limit, offset, err)
652				return
653			}
654
655			// Make a request for the next element.
656			it = client.Run(ctx, q.Limit(1).Start(cursor))
657			var entity SQChild
658			_, err = it.Next(&entity)
659			switch {
660			case want == -1:
661				if err != iterator.Done {
662					t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor %v, want Done", count, limit, offset, err)
663				}
664			case err != nil:
665				t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor: %v, want nil", count, limit, offset, err)
666			case entity.I != want:
667				t.Errorf("count=%d, limit=%d, offset=%d: got.I = %d, want %d", count, limit, offset, entity.I, want)
668			}
669		}(tt.count, tt.limit, tt.offset, tt.want)
670	}
671	wg.Wait()
672}
673
674func TestIntegration_EventualConsistency(t *testing.T) {
675	// TODO(jba): either make this actually test eventual consistency, or
676	// delete it. Currently it behaves the same with or without the
677	// EventualConsistency call.
678	ctx := context.Background()
679	client := newTestClient(ctx, t)
680	defer client.Close()
681
682	parent := NameKey("SQParent", "TestIntegration_EventualConsistency"+suffix, nil)
683	now := timeNow.Truncate(time.Millisecond).Unix()
684	children := []*SQChild{
685		{I: 0, T: now, U: now},
686		{I: 1, T: now, U: now},
687		{I: 2, T: now, U: now},
688	}
689	query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency()
690	testSmallQueries(ctx, t, client, parent, children, nil, func() {
691		got, err := client.Count(ctx, query)
692		if err != nil {
693			t.Fatalf("Count: %v", err)
694		}
695		if got < 0 || 3 < got {
696			t.Errorf("Count: got %d, want [0,3]", got)
697		}
698	})
699}
700
701func TestIntegration_Projection(t *testing.T) {
702	ctx := context.Background()
703	client := newTestClient(ctx, t)
704	defer client.Close()
705
706	parent := NameKey("SQParent", "TestIntegration_Projection"+suffix, nil)
707	now := timeNow.Truncate(time.Millisecond).Unix()
708	children := []*SQChild{
709		{I: 1 << 0, J: 100, T: now, U: now},
710		{I: 1 << 1, J: 100, T: now, U: now},
711		{I: 1 << 2, J: 200, T: now, U: now},
712		{I: 1 << 3, J: 300, T: now, U: now},
713		{I: 1 << 4, J: 300, T: now, U: now},
714	}
715	baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150)
716	testSmallQueries(ctx, t, client, parent, children, []SQTestCase{
717		{
718			"project",
719			baseQuery.Project("J"),
720			3,
721			200 + 300 + 300,
722		},
723		{
724			"distinct",
725			baseQuery.Project("J").Distinct(),
726			2,
727			200 + 300,
728		},
729		{
730			"distinct on",
731			baseQuery.Project("J").DistinctOn("J"),
732			2,
733			200 + 300,
734		},
735		{
736			"project on meaningful (GD_WHEN) field",
737			baseQuery.Project("U"),
738			3,
739			0,
740		},
741	})
742}
743
744func TestIntegration_AllocateIDs(t *testing.T) {
745	ctx := context.Background()
746	client := newTestClient(ctx, t)
747	defer client.Close()
748
749	keys := make([]*Key, 5)
750	for i := range keys {
751		keys[i] = IncompleteKey("AllocID", nil)
752	}
753	keys, err := client.AllocateIDs(ctx, keys)
754	if err != nil {
755		t.Errorf("AllocID #0 failed: %v", err)
756	}
757	if want := len(keys); want != 5 {
758		t.Errorf("Expected to allocate 5 keys, %d keys are found", want)
759	}
760	for _, k := range keys {
761		if k.Incomplete() {
762			t.Errorf("Unexpeceted incomplete key found: %v", k)
763		}
764	}
765}
766
767func TestIntegration_GetAllWithFieldMismatch(t *testing.T) {
768	ctx := context.Background()
769	client := newTestClient(ctx, t)
770	defer client.Close()
771
772	type Fat struct {
773		X, Y int
774	}
775	type Thin struct {
776		X int
777	}
778
779	// Ancestor queries (those within an entity group) are strongly consistent
780	// by default, which prevents a test from being flaky.
781	// See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency
782	// for more information.
783	parent := NameKey("SQParent", "TestIntegration_GetAllWithFieldMismatch"+suffix, nil)
784	putKeys := make([]*Key, 3)
785	for i := range putKeys {
786		putKeys[i] = IDKey("GetAllThing", int64(10+i), parent)
787		_, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i})
788		if err != nil {
789			t.Fatalf("client.Put: %v", err)
790		}
791	}
792
793	var got []Thin
794	want := []Thin{
795		{X: 20},
796		{X: 21},
797		{X: 22},
798	}
799	getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got)
800	if len(getKeys) != 3 && !testutil.Equal(getKeys, putKeys) {
801		t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys)
802	}
803	if !testutil.Equal(got, want) {
804		t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want)
805	}
806	if _, ok := err.(*ErrFieldMismatch); !ok {
807		t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err)
808	}
809}
810
811func TestIntegration_KindlessQueries(t *testing.T) {
812	ctx := context.Background()
813	client := newTestClient(ctx, t)
814	defer client.Close()
815
816	type Dee struct {
817		I   int
818		Why string
819	}
820	type Dum struct {
821		I     int
822		Pling string
823	}
824
825	parent := NameKey("Tweedle", "tweedle"+suffix, nil)
826
827	keys := []*Key{
828		NameKey("Dee", "dee0", parent),
829		NameKey("Dum", "dum1", parent),
830		NameKey("Dum", "dum2", parent),
831		NameKey("Dum", "dum3", parent),
832	}
833	src := []interface{}{
834		&Dee{1, "binary0001"},
835		&Dum{2, "binary0010"},
836		&Dum{4, "binary0100"},
837		&Dum{8, "binary1000"},
838	}
839	keys, err := client.PutMulti(ctx, keys, src)
840	if err != nil {
841		t.Fatalf("put: %v", err)
842	}
843
844	testCases := []struct {
845		desc    string
846		query   *Query
847		want    []int
848		wantErr string
849	}{
850		{
851			desc:  "Dee",
852			query: NewQuery("Dee"),
853			want:  []int{1},
854		},
855		{
856			desc:  "Doh",
857			query: NewQuery("Doh"),
858			want:  nil},
859		{
860			desc:  "Dum",
861			query: NewQuery("Dum"),
862			want:  []int{2, 4, 8},
863		},
864		{
865			desc:  "",
866			query: NewQuery(""),
867			want:  []int{1, 2, 4, 8},
868		},
869		{
870			desc:  "Kindless filter",
871			query: NewQuery("").Filter("__key__ =", keys[2]),
872			want:  []int{4},
873		},
874		{
875			desc:  "Kindless order",
876			query: NewQuery("").Order("__key__"),
877			want:  []int{1, 2, 4, 8},
878		},
879		{
880			desc:    "Kindless bad filter",
881			query:   NewQuery("").Filter("I =", 4),
882			wantErr: "kind is required",
883		},
884		{
885			desc:    "Kindless bad order",
886			query:   NewQuery("").Order("-__key__"),
887			wantErr: "kind is required for all orders except __key__ ascending",
888		},
889	}
890	for _, test := range testCases {
891		t.Run(test.desc, func(t *testing.T) {
892			q := test.query.Ancestor(parent)
893			gotCount, err := client.Count(ctx, q)
894			if err != nil {
895				if test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr) {
896					t.Fatalf("count %q: err %v, want err %q", test.desc, err, test.wantErr)
897				}
898				return
899			}
900			if test.wantErr != "" {
901				t.Fatalf("count %q: want err %q", test.desc, test.wantErr)
902			}
903			if gotCount != len(test.want) {
904				t.Fatalf("count %q: got %d want %d", test.desc, gotCount, len(test.want))
905			}
906			var got []int
907			for iter := client.Run(ctx, q); ; {
908				var dst struct {
909					I          int
910					Why, Pling string
911				}
912				_, err := iter.Next(&dst)
913				if err == iterator.Done {
914					break
915				}
916				if err != nil {
917					t.Fatalf("iter.Next %q: %v", test.desc, err)
918				}
919				got = append(got, dst.I)
920			}
921			sort.Ints(got)
922			if !testutil.Equal(got, test.want) {
923				t.Fatalf("elems %q: got %+v want %+v", test.desc, got, test.want)
924			}
925		})
926	}
927}
928
929func TestIntegration_Transaction(t *testing.T) {
930	ctx := context.Background()
931	client := newTestClient(ctx, t)
932	defer client.Close()
933
934	type Counter struct {
935		N int
936		T time.Time
937	}
938
939	bangErr := errors.New("bang")
940	tests := []struct {
941		desc          string
942		causeConflict []bool
943		retErr        []error
944		want          int
945		wantErr       error
946	}{
947		{
948			desc:          "3 attempts, no conflicts",
949			causeConflict: []bool{false},
950			retErr:        []error{nil},
951			want:          11,
952		},
953		{
954			desc:          "1 attempt, user error",
955			causeConflict: []bool{false},
956			retErr:        []error{bangErr},
957			wantErr:       bangErr,
958		},
959		{
960			desc:          "2 attempts, 1 conflict",
961			causeConflict: []bool{true, false},
962			retErr:        []error{nil, nil},
963			want:          13, // Each conflict increments by 2.
964		},
965		{
966			desc:          "3 attempts, 3 conflicts",
967			causeConflict: []bool{true, true, true},
968			retErr:        []error{nil, nil, nil},
969			wantErr:       ErrConcurrentTransaction,
970		},
971	}
972	for i, test := range tests {
973		t.Run(test.desc, func(t *testing.T) {
974			// Put a new counter.
975			c := &Counter{N: 10, T: timeNow}
976			key, err := client.Put(ctx, IncompleteKey("TransCounter", nil), c)
977			if err != nil {
978				t.Fatal(err)
979			}
980			defer client.Delete(ctx, key)
981
982			// Increment the counter in a transaction.
983			// The test case can manually cause a conflict or return an
984			// error at each attempt.
985			var attempts int
986			_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
987				attempts++
988				if attempts > len(test.causeConflict) {
989					return fmt.Errorf("too many attempts. Got %d, max %d", attempts, len(test.causeConflict))
990				}
991
992				var c Counter
993				if err := tx.Get(key, &c); err != nil {
994					return err
995				}
996				c.N++
997				if _, err := tx.Put(key, &c); err != nil {
998					return err
999				}
1000
1001				if test.causeConflict[attempts-1] {
1002					c.N++
1003					if _, err := client.Put(ctx, key, &c); err != nil {
1004						return err
1005					}
1006				}
1007
1008				return test.retErr[attempts-1]
1009			}, MaxAttempts(i))
1010
1011			// Check the error returned by RunInTransaction.
1012			if err != test.wantErr {
1013				t.Fatalf("got err %v, want %v", err, test.wantErr)
1014			}
1015			if test.wantErr != nil {
1016				// If we were expecting an error, this is where the test ends.
1017				return
1018			}
1019
1020			// Check the final value of the counter.
1021			if err := client.Get(ctx, key, c); err != nil {
1022				t.Fatal(err)
1023			}
1024			if c.N != test.want {
1025				t.Fatalf("counter N=%d, want N=%d", c.N, test.want)
1026			}
1027		})
1028	}
1029}
1030
1031func TestIntegration_ReadOnlyTransaction(t *testing.T) {
1032	if testing.Short() {
1033		t.Skip("Integration tests skipped in short mode")
1034	}
1035	ctx := context.Background()
1036	client := newClient(ctx, t, nil)
1037	defer client.Close()
1038
1039	type value struct{ N int }
1040
1041	// Put a value.
1042	const n = 5
1043	v := &value{N: n}
1044	key, err := client.Put(ctx, IncompleteKey("roTxn", nil), v)
1045	if err != nil {
1046		t.Fatal(err)
1047	}
1048	defer client.Delete(ctx, key)
1049
1050	// Read it from a read-only transaction.
1051	_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
1052		if err := tx.Get(key, v); err != nil {
1053			return err
1054		}
1055		return nil
1056	}, ReadOnly)
1057	if err != nil {
1058		t.Fatal(err)
1059	}
1060	if v.N != n {
1061		t.Fatalf("got %d, want %d", v.N, n)
1062	}
1063
1064	// Attempting to write from a read-only transaction is an error.
1065	_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
1066		if _, err := tx.Put(key, v); err != nil {
1067			return err
1068		}
1069		return nil
1070	}, ReadOnly)
1071	if err == nil {
1072		t.Fatal("got nil, want error")
1073	}
1074}
1075
1076func TestIntegration_NilPointers(t *testing.T) {
1077	ctx := context.Background()
1078	client := newTestClient(ctx, t)
1079	defer client.Close()
1080
1081	type X struct {
1082		S string
1083	}
1084
1085	src := []*X{{"zero"}, {"one"}}
1086	keys := []*Key{IncompleteKey("NilX", nil), IncompleteKey("NilX", nil)}
1087	keys, err := client.PutMulti(ctx, keys, src)
1088	if err != nil {
1089		t.Fatalf("PutMulti: %v", err)
1090	}
1091
1092	// It's okay to store into a slice of nil *X.
1093	xs := make([]*X, 2)
1094	if err := client.GetMulti(ctx, keys, xs); err != nil {
1095		t.Errorf("GetMulti: %v", err)
1096	} else if !testutil.Equal(xs, src) {
1097		t.Errorf("GetMulti fetched %v, want %v", xs, src)
1098	}
1099
1100	// It isn't okay to store into a single nil *X.
1101	var x0 *X
1102	if err, want := client.Get(ctx, keys[0], x0), ErrInvalidEntityType; err != want {
1103		t.Errorf("Get: err %v; want %v", err, want)
1104	}
1105
1106	// Test that deleting with duplicate keys work.
1107	keys = append(keys, keys...)
1108	if err := client.DeleteMulti(ctx, keys); err != nil {
1109		t.Errorf("Delete: %v", err)
1110	}
1111}
1112
1113func TestIntegration_NestedRepeatedElementNoIndex(t *testing.T) {
1114	ctx := context.Background()
1115	client := newTestClient(ctx, t)
1116	defer client.Close()
1117
1118	type Inner struct {
1119		Name  string
1120		Value string `datastore:",noindex"`
1121	}
1122	type Outer struct {
1123		Config []Inner
1124	}
1125	m := &Outer{
1126		Config: []Inner{
1127			{Name: "short", Value: "a"},
1128			{Name: "long", Value: strings.Repeat("a", 2000)},
1129		},
1130	}
1131
1132	key := NameKey("Nested", "Nested"+suffix, nil)
1133	if _, err := client.Put(ctx, key, m); err != nil {
1134		t.Fatalf("client.Put: %v", err)
1135	}
1136	if err := client.Delete(ctx, key); err != nil {
1137		t.Fatalf("client.Delete: %v", err)
1138	}
1139}
1140
1141func TestIntegration_PointerFields(t *testing.T) {
1142	ctx := context.Background()
1143	client := newTestClient(ctx, t)
1144	defer client.Close()
1145
1146	want := populatedPointers()
1147	key, err := client.Put(ctx, IncompleteKey("pointers", nil), want)
1148	if err != nil {
1149		t.Fatal(err)
1150	}
1151	var got Pointers
1152	if err := client.Get(ctx, key, &got); err != nil {
1153		t.Fatal(err)
1154	}
1155	if got.Pi == nil || *got.Pi != *want.Pi {
1156		t.Errorf("Pi: got %v, want %v", got.Pi, *want.Pi)
1157	}
1158	if got.Ps == nil || *got.Ps != *want.Ps {
1159		t.Errorf("Ps: got %v, want %v", got.Ps, *want.Ps)
1160	}
1161	if got.Pb == nil || *got.Pb != *want.Pb {
1162		t.Errorf("Pb: got %v, want %v", got.Pb, *want.Pb)
1163	}
1164	if got.Pf == nil || *got.Pf != *want.Pf {
1165		t.Errorf("Pf: got %v, want %v", got.Pf, *want.Pf)
1166	}
1167	if got.Pg == nil || *got.Pg != *want.Pg {
1168		t.Errorf("Pg: got %v, want %v", got.Pg, *want.Pg)
1169	}
1170	if got.Pt == nil || !got.Pt.Equal(*want.Pt) {
1171		t.Errorf("Pt: got %v, want %v", got.Pt, *want.Pt)
1172	}
1173}
1174
1175func TestIntegration_Mutate(t *testing.T) {
1176	// test Client.Mutate
1177	testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) {
1178		return client.Mutate(ctx, muts...)
1179	})
1180	// test Transaction.Mutate
1181	testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) {
1182		var pkeys []*PendingKey
1183		commit, err := client.RunInTransaction(ctx, func(tx *Transaction) error {
1184			var err error
1185			pkeys, err = tx.Mutate(muts...)
1186			return err
1187		})
1188		if err != nil {
1189			return nil, err
1190		}
1191		var keys []*Key
1192		for _, pk := range pkeys {
1193			keys = append(keys, commit.Key(pk))
1194		}
1195		return keys, nil
1196	})
1197}
1198
1199func testMutate(t *testing.T, mutate func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error)) {
1200	ctx := context.Background()
1201	client := newTestClient(ctx, t)
1202	defer client.Close()
1203
1204	type T struct{ I int }
1205
1206	check := func(k *Key, want interface{}) {
1207		var x T
1208		err := client.Get(ctx, k, &x)
1209		switch want := want.(type) {
1210		case error:
1211			if err != want {
1212				t.Errorf("key %s: got error %v, want %v", k, err, want)
1213			}
1214		case int:
1215			if err != nil {
1216				t.Fatalf("key %s: %v", k, err)
1217			}
1218			if x.I != want {
1219				t.Errorf("key %s: got %d, want %d", k, x.I, want)
1220			}
1221		default:
1222			panic("check: bad arg")
1223		}
1224	}
1225
1226	keys, err := mutate(ctx, client,
1227		NewInsert(IncompleteKey("t", nil), &T{1}),
1228		NewUpsert(IncompleteKey("t", nil), &T{2}),
1229	)
1230	if err != nil {
1231		t.Fatal(err)
1232	}
1233	check(keys[0], 1)
1234	check(keys[1], 2)
1235
1236	_, err = mutate(ctx, client,
1237		NewUpdate(keys[0], &T{3}),
1238		NewDelete(keys[1]),
1239	)
1240	if err != nil {
1241		t.Fatal(err)
1242	}
1243	check(keys[0], 3)
1244	check(keys[1], ErrNoSuchEntity)
1245
1246	_, err = mutate(ctx, client, NewInsert(keys[0], &T{4}))
1247	if got, want := status.Code(err), codes.AlreadyExists; got != want {
1248		t.Errorf("Insert existing key: got %s, want %s", got, want)
1249	}
1250
1251	_, err = mutate(ctx, client, NewUpdate(keys[1], &T{4}))
1252	if got, want := status.Code(err), codes.NotFound; got != want {
1253		t.Errorf("Update non-existing key: got %s, want %s", got, want)
1254	}
1255}
1256
1257func TestIntegration_DetectProjectID(t *testing.T) {
1258	if testing.Short() {
1259		t.Skip("Integration tests skipped in short mode")
1260	}
1261	ctx := context.Background()
1262
1263	creds := testutil.Credentials(ctx, ScopeDatastore)
1264	if creds == nil {
1265		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
1266	}
1267
1268	// Use creds with project ID.
1269	if _, err := NewClient(ctx, DetectProjectID, option.WithCredentials(creds)); err != nil {
1270		t.Errorf("NewClient: %v", err)
1271	}
1272
1273	ts := testutil.ErroringTokenSource{}
1274	// Try to use creds without project ID.
1275	_, err := NewClient(ctx, DetectProjectID, option.WithTokenSource(ts))
1276	if err == nil || err.Error() != "datastore: see the docs on DetectProjectID" {
1277		t.Errorf("expected an error while using TokenSource that does not have a project ID")
1278	}
1279}
1280
1281var genKeyName = uid.NewSpace("datastore-integration", nil)
1282
1283func TestIntegration_Project_TimestampStoreAndRetrieve(t *testing.T) {
1284	t.Skip("https://github.com/googleapis/google-cloud-go/issues/1479")
1285
1286	ctx := context.Background()
1287	client := newTestClient(ctx, t)
1288	defer client.Close()
1289
1290	type T struct{ Created time.Time }
1291
1292	// We need to generate a new key to prevent any clashes with concurrent test runs,
1293	// as per:  https://github.com/googleapis/google-cloud-go/issues/1479
1294	keyName := genKeyName.New()
1295
1296	now := time.Now()
1297	k, err := client.Put(ctx, IncompleteKey(keyName, nil), &T{Created: now})
1298	if err != nil {
1299		t.Fatal(err)
1300	}
1301	defer func() {
1302		if err := client.Delete(ctx, k); err != nil {
1303			log.Println(err)
1304		}
1305	}()
1306
1307	q := NewQuery(keyName).Order("Created").Project("Created")
1308	res := []T{}
1309	if _, err := client.GetAll(ctx, q, &res); err != nil {
1310		t.Fatal(err)
1311	}
1312	if len(res) != 1 {
1313		t.Fatalf("expected 1 result, got %d", len(res))
1314	}
1315	if got, want := res[0].Created.Unix(), now.Unix(); got != want {
1316		t.Fatalf("got %v, want %v", got, want)
1317	}
1318}
1319