1// Copyright 2014 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package datastore
16
17import (
18	"context"
19	"encoding/json"
20	"errors"
21	"flag"
22	"fmt"
23	"log"
24	"os"
25	"reflect"
26	"sort"
27	"strings"
28	"sync"
29	"testing"
30	"time"
31
32	"cloud.google.com/go/internal/testutil"
33	"cloud.google.com/go/internal/uid"
34	"cloud.google.com/go/rpcreplay"
35	"google.golang.org/api/iterator"
36	"google.golang.org/api/option"
37	"google.golang.org/grpc"
38	"google.golang.org/grpc/codes"
39	"google.golang.org/grpc/status"
40)
41
42// TODO(djd): Make test entity clean up more robust: some test entities may
43// be left behind if tests are aborted, the transport fails, etc.
44
45var timeNow = time.Now()
46
47// suffix is a timestamp-based suffix which is appended to key names,
48// particularly for the root keys of entity groups. This reduces flakiness
49// when the tests are run in parallel.
50var suffix string
51
52const replayFilename = "datastore.replay"
53
54type replayInfo struct {
55	ProjectID string
56	Time      time.Time
57}
58
59var (
60	record = flag.Bool("record", false, "record RPCs")
61
62	newTestClient = func(ctx context.Context, t *testing.T) *Client {
63		return newClient(ctx, t, nil)
64	}
65)
66
67func TestMain(m *testing.M) {
68	os.Exit(testMain(m))
69}
70
71func testMain(m *testing.M) int {
72	flag.Parse()
73	if testing.Short() {
74		if *record {
75			log.Fatal("cannot combine -short and -record")
76		}
77		if testutil.CanReplay(replayFilename) {
78			initReplay()
79		}
80	} else if *record {
81		if testutil.ProjID() == "" {
82			log.Fatal("must record with a project ID")
83		}
84		b, err := json.Marshal(replayInfo{
85			ProjectID: testutil.ProjID(),
86			Time:      timeNow,
87		})
88		if err != nil {
89			log.Fatal(err)
90		}
91		rec, err := rpcreplay.NewRecorder(replayFilename, b)
92		if err != nil {
93			log.Fatal(err)
94		}
95		defer func() {
96			if err := rec.Close(); err != nil {
97				log.Fatalf("closing recorder: %v", err)
98			}
99		}()
100		newTestClient = func(ctx context.Context, t *testing.T) *Client {
101			return newClient(ctx, t, rec.DialOptions())
102		}
103		log.Printf("recording to %s", replayFilename)
104	}
105	suffix = fmt.Sprintf("-t%d", timeNow.UnixNano())
106	return m.Run()
107}
108
109func initReplay() {
110	rep, err := rpcreplay.NewReplayer(replayFilename)
111	if err != nil {
112		log.Fatal(err)
113	}
114	defer rep.Close()
115
116	var ri replayInfo
117	if err := json.Unmarshal(rep.Initial(), &ri); err != nil {
118		log.Fatalf("unmarshaling initial replay info: %v", err)
119	}
120	timeNow = ri.Time.In(time.Local)
121
122	conn, err := rep.Connection()
123	if err != nil {
124		log.Fatal(err)
125	}
126
127	newTestClient = func(ctx context.Context, t *testing.T) *Client {
128		grpcHeadersEnforcer := &testutil.HeadersEnforcer{
129			OnFailure: t.Fatalf,
130			Checkers: []*testutil.HeaderChecker{
131				testutil.XGoogClientHeaderChecker,
132			},
133		}
134
135		opts := append(grpcHeadersEnforcer.CallOptions(), option.WithGRPCConn(conn))
136		client, err := NewClient(ctx, ri.ProjectID, opts...)
137		if err != nil {
138			t.Fatalf("NewClient: %v", err)
139		}
140		return client
141	}
142	log.Printf("replaying from %s", replayFilename)
143}
144
145func newClient(ctx context.Context, t *testing.T, dialOpts []grpc.DialOption) *Client {
146	if testing.Short() {
147		t.Skip("Integration tests skipped in short mode")
148	}
149	ts := testutil.TokenSource(ctx, ScopeDatastore)
150	if ts == nil {
151		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
152	}
153
154	grpcHeadersEnforcer := &testutil.HeadersEnforcer{
155		OnFailure: t.Fatalf,
156		Checkers: []*testutil.HeaderChecker{
157			testutil.XGoogClientHeaderChecker,
158		},
159	}
160	opts := append(grpcHeadersEnforcer.CallOptions(), option.WithTokenSource(ts))
161	for _, opt := range dialOpts {
162		opts = append(opts, option.WithGRPCDialOption(opt))
163	}
164	client, err := NewClient(ctx, testutil.ProjID(), opts...)
165	if err != nil {
166		t.Fatalf("NewClient: %v", err)
167	}
168	return client
169}
170
171func TestIntegration_Basics(t *testing.T) {
172	ctx, _ := context.WithTimeout(context.Background(), time.Second*20)
173	client := newTestClient(ctx, t)
174	defer client.Close()
175
176	type X struct {
177		I int
178		S string
179		T time.Time
180		U interface{}
181	}
182
183	x0 := X{66, "99", timeNow.Truncate(time.Millisecond), "X"}
184	k, err := client.Put(ctx, IncompleteKey("BasicsX", nil), &x0)
185	if err != nil {
186		t.Fatalf("client.Put: %v", err)
187	}
188	x1 := X{}
189	err = client.Get(ctx, k, &x1)
190	if err != nil {
191		t.Errorf("client.Get: %v", err)
192	}
193	err = client.Delete(ctx, k)
194	if err != nil {
195		t.Errorf("client.Delete: %v", err)
196	}
197	if !testutil.Equal(x0, x1) {
198		t.Errorf("compare: x0=%v, x1=%v", x0, x1)
199	}
200}
201
202func TestIntegration_TopLevelKeyLoaded(t *testing.T) {
203	ctx, _ := context.WithTimeout(context.Background(), time.Second*20)
204	client := newTestClient(ctx, t)
205	defer client.Close()
206
207	completeKey := NameKey("EntityWithKey", "myent", nil)
208
209	type EntityWithKey struct {
210		I int
211		S string
212		K *Key `datastore:"__key__"`
213	}
214
215	in := &EntityWithKey{
216		I: 12,
217		S: "abcd",
218	}
219
220	k, err := client.Put(ctx, completeKey, in)
221	if err != nil {
222		t.Fatalf("client.Put: %v", err)
223	}
224
225	var e EntityWithKey
226	err = client.Get(ctx, k, &e)
227	if err != nil {
228		t.Fatalf("client.Get: %v", err)
229	}
230
231	// The two keys should be absolutely identical.
232	if !testutil.Equal(e.K, k) {
233		t.Fatalf("e.K not equal to k; got %#v, want %#v", e.K, k)
234	}
235
236}
237
238func TestIntegration_ListValues(t *testing.T) {
239	ctx := context.Background()
240	client := newTestClient(ctx, t)
241	defer client.Close()
242
243	p0 := PropertyList{
244		{Name: "L", Value: []interface{}{int64(12), "string", true}},
245	}
246	k, err := client.Put(ctx, IncompleteKey("ListValue", nil), &p0)
247	if err != nil {
248		t.Fatalf("client.Put: %v", err)
249	}
250	var p1 PropertyList
251	if err := client.Get(ctx, k, &p1); err != nil {
252		t.Errorf("client.Get: %v", err)
253	}
254	if !testutil.Equal(p0, p1) {
255		t.Errorf("compare:\np0=%v\np1=%#v", p0, p1)
256	}
257	if err = client.Delete(ctx, k); err != nil {
258		t.Errorf("client.Delete: %v", err)
259	}
260}
261
262func TestIntegration_GetMulti(t *testing.T) {
263	ctx := context.Background()
264	client := newTestClient(ctx, t)
265	defer client.Close()
266
267	type X struct {
268		I int
269	}
270	p := NameKey("X", "x"+suffix, nil)
271
272	cases := []struct {
273		key *Key
274		put bool
275	}{
276		{key: NameKey("X", "item1", p), put: true},
277		{key: NameKey("X", "item2", p), put: false},
278		{key: NameKey("X", "item3", p), put: false},
279		{key: NameKey("X", "item3", p), put: false},
280		{key: NameKey("X", "item4", p), put: true},
281	}
282
283	var src, dst []*X
284	var srcKeys, dstKeys []*Key
285	for _, c := range cases {
286		dst = append(dst, &X{})
287		dstKeys = append(dstKeys, c.key)
288		if c.put {
289			src = append(src, &X{})
290			srcKeys = append(srcKeys, c.key)
291		}
292	}
293	if _, err := client.PutMulti(ctx, srcKeys, src); err != nil {
294		t.Error(err)
295	}
296	err := client.GetMulti(ctx, dstKeys, dst)
297	if err == nil {
298		t.Errorf("client.GetMulti got %v, expected error", err)
299	}
300	e, ok := err.(MultiError)
301	if !ok {
302		t.Errorf("client.GetMulti got %T, expected MultiError", err)
303	}
304	for i, err := range e {
305		got, want := err, (error)(nil)
306		if !cases[i].put {
307			got, want = err, ErrNoSuchEntity
308		}
309		if got != want {
310			t.Errorf("MultiError[%d] == %v, want %v", i, got, want)
311		}
312	}
313}
314
315type Z struct {
316	S string
317	T string `datastore:",noindex"`
318	P []byte
319	K []byte `datastore:",noindex"`
320}
321
322func (z Z) String() string {
323	var lens []string
324	v := reflect.ValueOf(z)
325	for i := 0; i < v.NumField(); i++ {
326		if l := v.Field(i).Len(); l > 0 {
327			lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l))
328		}
329	}
330	return fmt.Sprintf("Z{ %s }", strings.Join(lens, ","))
331}
332
333func TestIntegration_UnindexableValues(t *testing.T) {
334	ctx := context.Background()
335	client := newTestClient(ctx, t)
336	defer client.Close()
337
338	x1500 := strings.Repeat("x", 1500)
339	x1501 := strings.Repeat("x", 1501)
340	testCases := []struct {
341		in      Z
342		wantErr bool
343	}{
344		{in: Z{S: x1500}, wantErr: false},
345		{in: Z{S: x1501}, wantErr: true},
346		{in: Z{T: x1500}, wantErr: false},
347		{in: Z{T: x1501}, wantErr: false},
348		{in: Z{P: []byte(x1500)}, wantErr: false},
349		{in: Z{P: []byte(x1501)}, wantErr: true},
350		{in: Z{K: []byte(x1500)}, wantErr: false},
351		{in: Z{K: []byte(x1501)}, wantErr: false},
352	}
353	for _, tt := range testCases {
354		_, err := client.Put(ctx, IncompleteKey("BasicsZ", nil), &tt.in)
355		if (err != nil) != tt.wantErr {
356			t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr)
357		}
358	}
359}
360
361func TestIntegration_NilKey(t *testing.T) {
362	ctx := context.Background()
363	client := newTestClient(ctx, t)
364	defer client.Close()
365
366	testCases := []struct {
367		in      K0
368		wantErr bool
369	}{
370		{in: K0{K: testKey0}, wantErr: false},
371		{in: K0{}, wantErr: false},
372	}
373	for _, tt := range testCases {
374		_, err := client.Put(ctx, IncompleteKey("NilKey", nil), &tt.in)
375		if (err != nil) != tt.wantErr {
376			t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr)
377		}
378	}
379}
380
381type SQChild struct {
382	I, J int
383	T, U int64
384}
385
386type SQTestCase struct {
387	desc      string
388	q         *Query
389	wantCount int
390	wantSum   int
391}
392
393func testSmallQueries(ctx context.Context, t *testing.T, client *Client, parent *Key, children []*SQChild,
394	testCases []SQTestCase, extraTests ...func()) {
395	keys := make([]*Key, len(children))
396	for i := range keys {
397		keys[i] = IncompleteKey("SQChild", parent)
398	}
399	keys, err := client.PutMulti(ctx, keys, children)
400	if err != nil {
401		t.Fatalf("client.PutMulti: %v", err)
402	}
403	defer func() {
404		err := client.DeleteMulti(ctx, keys)
405		if err != nil {
406			t.Errorf("client.DeleteMulti: %v", err)
407		}
408	}()
409
410	for _, tc := range testCases {
411		count, err := client.Count(ctx, tc.q)
412		if err != nil {
413			t.Errorf("Count %q: %v", tc.desc, err)
414			continue
415		}
416		if count != tc.wantCount {
417			t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount)
418			continue
419		}
420	}
421
422	for _, tc := range testCases {
423		var got []SQChild
424		_, err := client.GetAll(ctx, tc.q, &got)
425		if err != nil {
426			t.Errorf("client.GetAll %q: %v", tc.desc, err)
427			continue
428		}
429		sum := 0
430		for _, c := range got {
431			sum += c.I + c.J
432		}
433		if sum != tc.wantSum {
434			t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum)
435			continue
436		}
437	}
438	for _, x := range extraTests {
439		x()
440	}
441}
442
443func TestIntegration_Filters(t *testing.T) {
444	ctx := context.Background()
445	client := newTestClient(ctx, t)
446	defer client.Close()
447
448	parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil)
449	now := timeNow.Truncate(time.Millisecond).Unix()
450	children := []*SQChild{
451		{I: 0, T: now, U: now},
452		{I: 1, T: now, U: now},
453		{I: 2, T: now, U: now},
454		{I: 3, T: now, U: now},
455		{I: 4, T: now, U: now},
456		{I: 5, T: now, U: now},
457		{I: 6, T: now, U: now},
458		{I: 7, T: now, U: now},
459	}
460	baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now)
461	testSmallQueries(ctx, t, client, parent, children, []SQTestCase{
462		{
463			"I>1",
464			baseQuery.Filter("I>", 1),
465			6,
466			2 + 3 + 4 + 5 + 6 + 7,
467		},
468		{
469			"I>2 AND I<=5",
470			baseQuery.Filter("I>", 2).Filter("I<=", 5),
471			3,
472			3 + 4 + 5,
473		},
474		{
475			"I>=3 AND I<3",
476			baseQuery.Filter("I>=", 3).Filter("I<", 3),
477			0,
478			0,
479		},
480		{
481			"I=4",
482			baseQuery.Filter("I=", 4),
483			1,
484			4,
485		},
486	}, func() {
487		got := []*SQChild{}
488		want := []*SQChild{
489			{I: 0, T: now, U: now},
490			{I: 1, T: now, U: now},
491			{I: 2, T: now, U: now},
492			{I: 3, T: now, U: now},
493			{I: 4, T: now, U: now},
494			{I: 5, T: now, U: now},
495			{I: 6, T: now, U: now},
496			{I: 7, T: now, U: now},
497		}
498		_, err := client.GetAll(ctx, baseQuery.Order("I"), &got)
499		if err != nil {
500			t.Errorf("client.GetAll: %v", err)
501		}
502		if !testutil.Equal(got, want) {
503			t.Errorf("compare: got=%v, want=%v", got, want)
504		}
505	}, func() {
506		got := []*SQChild{}
507		want := []*SQChild{
508			{I: 7, T: now, U: now},
509			{I: 6, T: now, U: now},
510			{I: 5, T: now, U: now},
511			{I: 4, T: now, U: now},
512			{I: 3, T: now, U: now},
513			{I: 2, T: now, U: now},
514			{I: 1, T: now, U: now},
515			{I: 0, T: now, U: now},
516		}
517		_, err := client.GetAll(ctx, baseQuery.Order("-I"), &got)
518		if err != nil {
519			t.Errorf("client.GetAll: %v", err)
520		}
521		if !testutil.Equal(got, want) {
522			t.Errorf("compare: got=%v, want=%v", got, want)
523		}
524	})
525}
526
527type ckey struct{}
528
529func TestIntegration_LargeQuery(t *testing.T) {
530	ctx := context.Background()
531	client := newTestClient(ctx, t)
532	defer client.Close()
533
534	parent := NameKey("LQParent", "TestIntegration_LargeQuery"+suffix, nil)
535	now := timeNow.Truncate(time.Millisecond).Unix()
536
537	// Make a large number of children entities.
538	const n = 800
539	children := make([]*SQChild, 0, n)
540	keys := make([]*Key, 0, n)
541	for i := 0; i < n; i++ {
542		children = append(children, &SQChild{I: i, T: now, U: now})
543		keys = append(keys, IncompleteKey("SQChild", parent))
544	}
545
546	// Store using PutMulti in batches.
547	const batchSize = 500
548	for i := 0; i < n; i = i + 500 {
549		j := i + batchSize
550		if j > n {
551			j = n
552		}
553		fullKeys, err := client.PutMulti(ctx, keys[i:j], children[i:j])
554		if err != nil {
555			t.Fatalf("PutMulti(%d, %d): %v", i, j, err)
556		}
557		defer func() {
558			err := client.DeleteMulti(ctx, fullKeys)
559			if err != nil {
560				t.Errorf("client.DeleteMulti: %v", err)
561			}
562		}()
563	}
564
565	q := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Order("I")
566
567	// Wait group to allow us to run query tests in parallel below.
568	var wg sync.WaitGroup
569
570	// Check we get the expected count and results for various limits/offsets.
571	queryTests := []struct {
572		limit, offset, want int
573	}{
574		// Just limit.
575		{limit: 0, want: 0},
576		{limit: 100, want: 100},
577		{limit: 501, want: 501},
578		{limit: n, want: n},
579		{limit: n * 2, want: n},
580		{limit: -1, want: n},
581		// Just offset.
582		{limit: -1, offset: 100, want: n - 100},
583		{limit: -1, offset: 500, want: n - 500},
584		{limit: -1, offset: n, want: 0},
585		// Limit and offset.
586		{limit: 100, offset: 100, want: 100},
587		{limit: 1000, offset: 100, want: n - 100},
588		{limit: 500, offset: 500, want: n - 500},
589	}
590	for _, tt := range queryTests {
591		q := q.Limit(tt.limit).Offset(tt.offset)
592		wg.Add(1)
593
594		go func(limit, offset, want int) {
595			defer wg.Done()
596			// Check Count returns the expected number of results.
597			count, err := client.Count(ctx, q)
598			if err != nil {
599				t.Errorf("client.Count(limit=%d offset=%d): %v", limit, offset, err)
600				return
601			}
602			if count != want {
603				t.Errorf("Count(limit=%d offset=%d) returned %d, want %d", limit, offset, count, want)
604			}
605
606			var got []SQChild
607			_, err = client.GetAll(ctx, q, &got)
608			if err != nil {
609				t.Errorf("client.GetAll(limit=%d offset=%d): %v", limit, offset, err)
610				return
611			}
612			if len(got) != want {
613				t.Errorf("GetAll(limit=%d offset=%d) returned %d, want %d", limit, offset, len(got), want)
614			}
615			for i, child := range got {
616				if got, want := child.I, i+offset; got != want {
617					t.Errorf("GetAll(limit=%d offset=%d) got[%d].I == %d; want %d", limit, offset, i, got, want)
618					break
619				}
620			}
621		}(tt.limit, tt.offset, tt.want)
622	}
623
624	// Also check iterator cursor behaviour.
625	cursorTests := []struct {
626		limit, offset int // Query limit and offset.
627		count         int // The number of times to call "next"
628		want          int // The I value of the desired element, -1 for "Done".
629	}{
630		// No limits.
631		{count: 0, limit: -1, want: 0},
632		{count: 5, limit: -1, want: 5},
633		{count: 500, limit: -1, want: 500},
634		{count: 1000, limit: -1, want: -1}, // No more results.
635		// Limits.
636		{count: 5, limit: 5, want: 5},
637		{count: 500, limit: 5, want: 5},
638		{count: 1000, limit: 1000, want: -1}, // No more results.
639		// Offsets.
640		{count: 0, offset: 5, limit: -1, want: 5},
641		{count: 5, offset: 5, limit: -1, want: 10},
642		{count: 200, offset: 500, limit: -1, want: 700},
643		{count: 200, offset: 1000, limit: -1, want: -1}, // No more results.
644	}
645	for _, tt := range cursorTests {
646		wg.Add(1)
647
648		go func(count, limit, offset, want int) {
649			defer wg.Done()
650
651			ctx := context.WithValue(ctx, ckey{}, fmt.Sprintf("c=%d,l=%d,o=%d", count, limit, offset))
652			// Run iterator through count calls to Next.
653			it := client.Run(ctx, q.Limit(limit).Offset(offset).KeysOnly())
654			for i := 0; i < count; i++ {
655				_, err := it.Next(nil)
656				if err == iterator.Done {
657					break
658				}
659				if err != nil {
660					t.Errorf("count=%d, limit=%d, offset=%d: it.Next failed at i=%d", count, limit, offset, i)
661					return
662				}
663			}
664
665			// Grab the cursor.
666			cursor, err := it.Cursor()
667			if err != nil {
668				t.Errorf("count=%d, limit=%d, offset=%d: it.Cursor: %v", count, limit, offset, err)
669				return
670			}
671
672			// Make a request for the next element.
673			it = client.Run(ctx, q.Limit(1).Start(cursor))
674			var entity SQChild
675			_, err = it.Next(&entity)
676			switch {
677			case want == -1:
678				if err != iterator.Done {
679					t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor %v, want Done", count, limit, offset, err)
680				}
681			case err != nil:
682				t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor: %v, want nil", count, limit, offset, err)
683			case entity.I != want:
684				t.Errorf("count=%d, limit=%d, offset=%d: got.I = %d, want %d", count, limit, offset, entity.I, want)
685			}
686		}(tt.count, tt.limit, tt.offset, tt.want)
687	}
688	wg.Wait()
689}
690
691func TestIntegration_EventualConsistency(t *testing.T) {
692	// TODO(jba): either make this actually test eventual consistency, or
693	// delete it. Currently it behaves the same with or without the
694	// EventualConsistency call.
695	ctx := context.Background()
696	client := newTestClient(ctx, t)
697	defer client.Close()
698
699	parent := NameKey("SQParent", "TestIntegration_EventualConsistency"+suffix, nil)
700	now := timeNow.Truncate(time.Millisecond).Unix()
701	children := []*SQChild{
702		{I: 0, T: now, U: now},
703		{I: 1, T: now, U: now},
704		{I: 2, T: now, U: now},
705	}
706	query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency()
707	testSmallQueries(ctx, t, client, parent, children, nil, func() {
708		got, err := client.Count(ctx, query)
709		if err != nil {
710			t.Fatalf("Count: %v", err)
711		}
712		if got < 0 || 3 < got {
713			t.Errorf("Count: got %d, want [0,3]", got)
714		}
715	})
716}
717
718func TestIntegration_Projection(t *testing.T) {
719	ctx := context.Background()
720	client := newTestClient(ctx, t)
721	defer client.Close()
722
723	parent := NameKey("SQParent", "TestIntegration_Projection"+suffix, nil)
724	now := timeNow.Truncate(time.Millisecond).Unix()
725	children := []*SQChild{
726		{I: 1 << 0, J: 100, T: now, U: now},
727		{I: 1 << 1, J: 100, T: now, U: now},
728		{I: 1 << 2, J: 200, T: now, U: now},
729		{I: 1 << 3, J: 300, T: now, U: now},
730		{I: 1 << 4, J: 300, T: now, U: now},
731	}
732	baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150)
733	testSmallQueries(ctx, t, client, parent, children, []SQTestCase{
734		{
735			"project",
736			baseQuery.Project("J"),
737			3,
738			200 + 300 + 300,
739		},
740		{
741			"distinct",
742			baseQuery.Project("J").Distinct(),
743			2,
744			200 + 300,
745		},
746		{
747			"distinct on",
748			baseQuery.Project("J").DistinctOn("J"),
749			2,
750			200 + 300,
751		},
752		{
753			"project on meaningful (GD_WHEN) field",
754			baseQuery.Project("U"),
755			3,
756			0,
757		},
758	})
759}
760
761func TestIntegration_AllocateIDs(t *testing.T) {
762	ctx := context.Background()
763	client := newTestClient(ctx, t)
764	defer client.Close()
765
766	keys := make([]*Key, 5)
767	for i := range keys {
768		keys[i] = IncompleteKey("AllocID", nil)
769	}
770	keys, err := client.AllocateIDs(ctx, keys)
771	if err != nil {
772		t.Errorf("AllocID #0 failed: %v", err)
773	}
774	if want := len(keys); want != 5 {
775		t.Errorf("Expected to allocate 5 keys, %d keys are found", want)
776	}
777	for _, k := range keys {
778		if k.Incomplete() {
779			t.Errorf("Unexpeceted incomplete key found: %v", k)
780		}
781	}
782}
783
784func TestIntegration_GetAllWithFieldMismatch(t *testing.T) {
785	ctx := context.Background()
786	client := newTestClient(ctx, t)
787	defer client.Close()
788
789	type Fat struct {
790		X, Y int
791	}
792	type Thin struct {
793		X int
794	}
795
796	// Ancestor queries (those within an entity group) are strongly consistent
797	// by default, which prevents a test from being flaky.
798	// See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency
799	// for more information.
800	parent := NameKey("SQParent", "TestIntegration_GetAllWithFieldMismatch"+suffix, nil)
801	putKeys := make([]*Key, 3)
802	for i := range putKeys {
803		putKeys[i] = IDKey("GetAllThing", int64(10+i), parent)
804		_, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i})
805		if err != nil {
806			t.Fatalf("client.Put: %v", err)
807		}
808	}
809
810	var got []Thin
811	want := []Thin{
812		{X: 20},
813		{X: 21},
814		{X: 22},
815	}
816	getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got)
817	if len(getKeys) != 3 && !testutil.Equal(getKeys, putKeys) {
818		t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys)
819	}
820	if !testutil.Equal(got, want) {
821		t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want)
822	}
823	if _, ok := err.(*ErrFieldMismatch); !ok {
824		t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err)
825	}
826}
827
828func TestIntegration_KindlessQueries(t *testing.T) {
829	ctx := context.Background()
830	client := newTestClient(ctx, t)
831	defer client.Close()
832
833	type Dee struct {
834		I   int
835		Why string
836	}
837	type Dum struct {
838		I     int
839		Pling string
840	}
841
842	parent := NameKey("Tweedle", "tweedle"+suffix, nil)
843
844	keys := []*Key{
845		NameKey("Dee", "dee0", parent),
846		NameKey("Dum", "dum1", parent),
847		NameKey("Dum", "dum2", parent),
848		NameKey("Dum", "dum3", parent),
849	}
850	src := []interface{}{
851		&Dee{1, "binary0001"},
852		&Dum{2, "binary0010"},
853		&Dum{4, "binary0100"},
854		&Dum{8, "binary1000"},
855	}
856	keys, err := client.PutMulti(ctx, keys, src)
857	if err != nil {
858		t.Fatalf("put: %v", err)
859	}
860
861	testCases := []struct {
862		desc    string
863		query   *Query
864		want    []int
865		wantErr string
866	}{
867		{
868			desc:  "Dee",
869			query: NewQuery("Dee"),
870			want:  []int{1},
871		},
872		{
873			desc:  "Doh",
874			query: NewQuery("Doh"),
875			want:  nil},
876		{
877			desc:  "Dum",
878			query: NewQuery("Dum"),
879			want:  []int{2, 4, 8},
880		},
881		{
882			desc:  "",
883			query: NewQuery(""),
884			want:  []int{1, 2, 4, 8},
885		},
886		{
887			desc:  "Kindless filter",
888			query: NewQuery("").Filter("__key__ =", keys[2]),
889			want:  []int{4},
890		},
891		{
892			desc:  "Kindless order",
893			query: NewQuery("").Order("__key__"),
894			want:  []int{1, 2, 4, 8},
895		},
896		{
897			desc:    "Kindless bad filter",
898			query:   NewQuery("").Filter("I =", 4),
899			wantErr: "kind is required",
900		},
901		{
902			desc:    "Kindless bad order",
903			query:   NewQuery("").Order("-__key__"),
904			wantErr: "kind is required for all orders except __key__ ascending",
905		},
906	}
907	for _, test := range testCases {
908		t.Run(test.desc, func(t *testing.T) {
909			q := test.query.Ancestor(parent)
910			gotCount, err := client.Count(ctx, q)
911			if err != nil {
912				if test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr) {
913					t.Fatalf("count %q: err %v, want err %q", test.desc, err, test.wantErr)
914				}
915				return
916			}
917			if test.wantErr != "" {
918				t.Fatalf("count %q: want err %q", test.desc, test.wantErr)
919			}
920			if gotCount != len(test.want) {
921				t.Fatalf("count %q: got %d want %d", test.desc, gotCount, len(test.want))
922			}
923			var got []int
924			for iter := client.Run(ctx, q); ; {
925				var dst struct {
926					I          int
927					Why, Pling string
928				}
929				_, err := iter.Next(&dst)
930				if err == iterator.Done {
931					break
932				}
933				if err != nil {
934					t.Fatalf("iter.Next %q: %v", test.desc, err)
935				}
936				got = append(got, dst.I)
937			}
938			sort.Ints(got)
939			if !testutil.Equal(got, test.want) {
940				t.Fatalf("elems %q: got %+v want %+v", test.desc, got, test.want)
941			}
942		})
943	}
944}
945
946func TestIntegration_Transaction(t *testing.T) {
947	ctx := context.Background()
948	client := newTestClient(ctx, t)
949	defer client.Close()
950
951	type Counter struct {
952		N int
953		T time.Time
954	}
955
956	bangErr := errors.New("bang")
957	tests := []struct {
958		desc          string
959		causeConflict []bool
960		retErr        []error
961		want          int
962		wantErr       error
963	}{
964		{
965			desc:          "3 attempts, no conflicts",
966			causeConflict: []bool{false},
967			retErr:        []error{nil},
968			want:          11,
969		},
970		{
971			desc:          "1 attempt, user error",
972			causeConflict: []bool{false},
973			retErr:        []error{bangErr},
974			wantErr:       bangErr,
975		},
976		{
977			desc:          "2 attempts, 1 conflict",
978			causeConflict: []bool{true, false},
979			retErr:        []error{nil, nil},
980			want:          13, // Each conflict increments by 2.
981		},
982		{
983			desc:          "3 attempts, 3 conflicts",
984			causeConflict: []bool{true, true, true},
985			retErr:        []error{nil, nil, nil},
986			wantErr:       ErrConcurrentTransaction,
987		},
988	}
989	for i, test := range tests {
990		t.Run(test.desc, func(t *testing.T) {
991			// Put a new counter.
992			c := &Counter{N: 10, T: timeNow}
993			key, err := client.Put(ctx, IncompleteKey("TransCounter", nil), c)
994			if err != nil {
995				t.Fatal(err)
996			}
997			defer client.Delete(ctx, key)
998
999			// Increment the counter in a transaction.
1000			// The test case can manually cause a conflict or return an
1001			// error at each attempt.
1002			var attempts int
1003			_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
1004				attempts++
1005				if attempts > len(test.causeConflict) {
1006					return fmt.Errorf("too many attempts. Got %d, max %d", attempts, len(test.causeConflict))
1007				}
1008
1009				var c Counter
1010				if err := tx.Get(key, &c); err != nil {
1011					return err
1012				}
1013				c.N++
1014				if _, err := tx.Put(key, &c); err != nil {
1015					return err
1016				}
1017
1018				if test.causeConflict[attempts-1] {
1019					c.N++
1020					if _, err := client.Put(ctx, key, &c); err != nil {
1021						return err
1022					}
1023				}
1024
1025				return test.retErr[attempts-1]
1026			}, MaxAttempts(i))
1027
1028			// Check the error returned by RunInTransaction.
1029			if err != test.wantErr {
1030				t.Fatalf("got err %v, want %v", err, test.wantErr)
1031			}
1032			if test.wantErr != nil {
1033				// If we were expecting an error, this is where the test ends.
1034				return
1035			}
1036
1037			// Check the final value of the counter.
1038			if err := client.Get(ctx, key, c); err != nil {
1039				t.Fatal(err)
1040			}
1041			if c.N != test.want {
1042				t.Fatalf("counter N=%d, want N=%d", c.N, test.want)
1043			}
1044		})
1045	}
1046}
1047
1048func TestIntegration_ReadOnlyTransaction(t *testing.T) {
1049	if testing.Short() {
1050		t.Skip("Integration tests skipped in short mode")
1051	}
1052	ctx := context.Background()
1053	client := newClient(ctx, t, nil)
1054	defer client.Close()
1055
1056	type value struct{ N int }
1057
1058	// Put a value.
1059	const n = 5
1060	v := &value{N: n}
1061	key, err := client.Put(ctx, IncompleteKey("roTxn", nil), v)
1062	if err != nil {
1063		t.Fatal(err)
1064	}
1065	defer client.Delete(ctx, key)
1066
1067	// Read it from a read-only transaction.
1068	_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
1069		if err := tx.Get(key, v); err != nil {
1070			return err
1071		}
1072		return nil
1073	}, ReadOnly)
1074	if err != nil {
1075		t.Fatal(err)
1076	}
1077	if v.N != n {
1078		t.Fatalf("got %d, want %d", v.N, n)
1079	}
1080
1081	// Attempting to write from a read-only transaction is an error.
1082	_, err = client.RunInTransaction(ctx, func(tx *Transaction) error {
1083		if _, err := tx.Put(key, v); err != nil {
1084			return err
1085		}
1086		return nil
1087	}, ReadOnly)
1088	if err == nil {
1089		t.Fatal("got nil, want error")
1090	}
1091}
1092
1093func TestIntegration_NilPointers(t *testing.T) {
1094	ctx := context.Background()
1095	client := newTestClient(ctx, t)
1096	defer client.Close()
1097
1098	type X struct {
1099		S string
1100	}
1101
1102	src := []*X{{"zero"}, {"one"}}
1103	keys := []*Key{IncompleteKey("NilX", nil), IncompleteKey("NilX", nil)}
1104	keys, err := client.PutMulti(ctx, keys, src)
1105	if err != nil {
1106		t.Fatalf("PutMulti: %v", err)
1107	}
1108
1109	// It's okay to store into a slice of nil *X.
1110	xs := make([]*X, 2)
1111	if err := client.GetMulti(ctx, keys, xs); err != nil {
1112		t.Errorf("GetMulti: %v", err)
1113	} else if !testutil.Equal(xs, src) {
1114		t.Errorf("GetMulti fetched %v, want %v", xs, src)
1115	}
1116
1117	// It isn't okay to store into a single nil *X.
1118	var x0 *X
1119	if err, want := client.Get(ctx, keys[0], x0), ErrInvalidEntityType; err != want {
1120		t.Errorf("Get: err %v; want %v", err, want)
1121	}
1122
1123	// Test that deleting with duplicate keys work.
1124	keys = append(keys, keys...)
1125	if err := client.DeleteMulti(ctx, keys); err != nil {
1126		t.Errorf("Delete: %v", err)
1127	}
1128}
1129
1130func TestIntegration_NestedRepeatedElementNoIndex(t *testing.T) {
1131	ctx := context.Background()
1132	client := newTestClient(ctx, t)
1133	defer client.Close()
1134
1135	type Inner struct {
1136		Name  string
1137		Value string `datastore:",noindex"`
1138	}
1139	type Outer struct {
1140		Config []Inner
1141	}
1142	m := &Outer{
1143		Config: []Inner{
1144			{Name: "short", Value: "a"},
1145			{Name: "long", Value: strings.Repeat("a", 2000)},
1146		},
1147	}
1148
1149	key := NameKey("Nested", "Nested"+suffix, nil)
1150	if _, err := client.Put(ctx, key, m); err != nil {
1151		t.Fatalf("client.Put: %v", err)
1152	}
1153	if err := client.Delete(ctx, key); err != nil {
1154		t.Fatalf("client.Delete: %v", err)
1155	}
1156}
1157
1158func TestIntegration_PointerFields(t *testing.T) {
1159	ctx := context.Background()
1160	client := newTestClient(ctx, t)
1161	defer client.Close()
1162
1163	want := populatedPointers()
1164	key, err := client.Put(ctx, IncompleteKey("pointers", nil), want)
1165	if err != nil {
1166		t.Fatal(err)
1167	}
1168	var got Pointers
1169	if err := client.Get(ctx, key, &got); err != nil {
1170		t.Fatal(err)
1171	}
1172	if got.Pi == nil || *got.Pi != *want.Pi {
1173		t.Errorf("Pi: got %v, want %v", got.Pi, *want.Pi)
1174	}
1175	if got.Ps == nil || *got.Ps != *want.Ps {
1176		t.Errorf("Ps: got %v, want %v", got.Ps, *want.Ps)
1177	}
1178	if got.Pb == nil || *got.Pb != *want.Pb {
1179		t.Errorf("Pb: got %v, want %v", got.Pb, *want.Pb)
1180	}
1181	if got.Pf == nil || *got.Pf != *want.Pf {
1182		t.Errorf("Pf: got %v, want %v", got.Pf, *want.Pf)
1183	}
1184	if got.Pg == nil || *got.Pg != *want.Pg {
1185		t.Errorf("Pg: got %v, want %v", got.Pg, *want.Pg)
1186	}
1187	if got.Pt == nil || !got.Pt.Equal(*want.Pt) {
1188		t.Errorf("Pt: got %v, want %v", got.Pt, *want.Pt)
1189	}
1190}
1191
1192func TestIntegration_Mutate(t *testing.T) {
1193	// test Client.Mutate
1194	testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) {
1195		return client.Mutate(ctx, muts...)
1196	})
1197	// test Transaction.Mutate
1198	testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) {
1199		var pkeys []*PendingKey
1200		commit, err := client.RunInTransaction(ctx, func(tx *Transaction) error {
1201			var err error
1202			pkeys, err = tx.Mutate(muts...)
1203			return err
1204		})
1205		if err != nil {
1206			return nil, err
1207		}
1208		var keys []*Key
1209		for _, pk := range pkeys {
1210			keys = append(keys, commit.Key(pk))
1211		}
1212		return keys, nil
1213	})
1214}
1215
1216func testMutate(t *testing.T, mutate func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error)) {
1217	ctx := context.Background()
1218	client := newTestClient(ctx, t)
1219	defer client.Close()
1220
1221	type T struct{ I int }
1222
1223	check := func(k *Key, want interface{}) {
1224		var x T
1225		err := client.Get(ctx, k, &x)
1226		switch want := want.(type) {
1227		case error:
1228			if err != want {
1229				t.Errorf("key %s: got error %v, want %v", k, err, want)
1230			}
1231		case int:
1232			if err != nil {
1233				t.Fatalf("key %s: %v", k, err)
1234			}
1235			if x.I != want {
1236				t.Errorf("key %s: got %d, want %d", k, x.I, want)
1237			}
1238		default:
1239			panic("check: bad arg")
1240		}
1241	}
1242
1243	keys, err := mutate(ctx, client,
1244		NewInsert(IncompleteKey("t", nil), &T{1}),
1245		NewUpsert(IncompleteKey("t", nil), &T{2}),
1246	)
1247	if err != nil {
1248		t.Fatal(err)
1249	}
1250	check(keys[0], 1)
1251	check(keys[1], 2)
1252
1253	_, err = mutate(ctx, client,
1254		NewUpdate(keys[0], &T{3}),
1255		NewDelete(keys[1]),
1256	)
1257	if err != nil {
1258		t.Fatal(err)
1259	}
1260	check(keys[0], 3)
1261	check(keys[1], ErrNoSuchEntity)
1262
1263	_, err = mutate(ctx, client, NewInsert(keys[0], &T{4}))
1264	if got, want := status.Code(err), codes.AlreadyExists; got != want {
1265		t.Errorf("Insert existing key: got %s, want %s", got, want)
1266	}
1267
1268	_, err = mutate(ctx, client, NewUpdate(keys[1], &T{4}))
1269	if got, want := status.Code(err), codes.NotFound; got != want {
1270		t.Errorf("Update non-existing key: got %s, want %s", got, want)
1271	}
1272}
1273
1274func TestIntegration_DetectProjectID(t *testing.T) {
1275	if testing.Short() {
1276		t.Skip("Integration tests skipped in short mode")
1277	}
1278	ctx := context.Background()
1279
1280	creds := testutil.Credentials(ctx, ScopeDatastore)
1281	if creds == nil {
1282		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
1283	}
1284
1285	// Use creds with project ID.
1286	if _, err := NewClient(ctx, DetectProjectID, option.WithCredentials(creds)); err != nil {
1287		t.Errorf("NewClient: %v", err)
1288	}
1289
1290	ts := testutil.ErroringTokenSource{}
1291	// Try to use creds without project ID.
1292	_, err := NewClient(ctx, DetectProjectID, option.WithTokenSource(ts))
1293	if err == nil || err.Error() != "datastore: see the docs on DetectProjectID" {
1294		t.Errorf("expected an error while using TokenSource that does not have a project ID")
1295	}
1296}
1297
1298var genKeyName = uid.NewSpace("datastore-integration", nil)
1299
1300func TestIntegration_Project_TimestampStoreAndRetrieve(t *testing.T) {
1301	ctx := context.Background()
1302	client := newTestClient(ctx, t)
1303	defer client.Close()
1304
1305	type T struct{ Created time.Time }
1306
1307	keyName := genKeyName.New()
1308
1309	now := time.Now()
1310	k, err := client.Put(ctx, IncompleteKey(keyName, nil), &T{Created: now})
1311	if err != nil {
1312		t.Fatal(err)
1313	}
1314	defer func() {
1315		if err := client.Delete(ctx, k); err != nil {
1316			log.Println(err)
1317		}
1318	}()
1319
1320	// Without .Ancestor, this query is eventually consistent (so this test
1321	// would be flakey). Ancestor queries, however, are strongly consistent.
1322	// See more at https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/.
1323	q := NewQuery(k.Kind).Ancestor(k)
1324	res := []T{}
1325	if _, err := client.GetAll(ctx, q, &res); err != nil {
1326		t.Fatal(err)
1327	}
1328	if len(res) != 1 {
1329		t.Fatalf("expected 1 result, got %d", len(res))
1330	}
1331	if got, want := res[0].Created.Unix(), now.Unix(); got != want {
1332		t.Fatalf("got %v, want %v", got, want)
1333	}
1334}
1335