1// Copyright 2018 The etcd Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package snapshot_test
16
17import (
18	"context"
19	"fmt"
20	"testing"
21	"time"
22
23	"go.etcd.io/etcd/client/pkg/v3/testutil"
24	"go.etcd.io/etcd/client/v3"
25	"go.etcd.io/etcd/server/v3/embed"
26	"go.etcd.io/etcd/server/v3/etcdserver"
27	"go.etcd.io/etcd/tests/v3/integration"
28)
29
30// TestSnapshotV3RestoreMultiMemberAdd ensures that multiple members
31// can boot into the same cluster after being restored from a same
32// snapshot file, and also be able to add another member to the cluster.
33func TestSnapshotV3RestoreMultiMemberAdd(t *testing.T) {
34	integration.BeforeTest(t)
35
36	kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}}
37	dbPath := createSnapshotFile(t, kvs)
38
39	clusterN := 3
40	cURLs, pURLs, srvs := restoreCluster(t, clusterN, dbPath)
41
42	defer func() {
43		for i := 0; i < clusterN; i++ {
44			srvs[i].Close()
45		}
46	}()
47
48	// wait for health interval + leader election
49	time.Sleep(etcdserver.HealthInterval + 2*time.Second)
50
51	cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{cURLs[0].String()}})
52	if err != nil {
53		t.Fatal(err)
54	}
55	defer cli.Close()
56
57	urls := newEmbedURLs(2)
58	newCURLs, newPURLs := urls[:1], urls[1:]
59	if _, err = cli.MemberAdd(context.Background(), []string{newPURLs[0].String()}); err != nil {
60		t.Fatal(err)
61	}
62
63	// wait for membership reconfiguration apply
64	time.Sleep(testutil.ApplyTimeout)
65
66	cfg := integration.NewEmbedConfig(t, "3")
67	cfg.InitialClusterToken = testClusterTkn
68	cfg.ClusterState = "existing"
69	cfg.LCUrls, cfg.ACUrls = newCURLs, newCURLs
70	cfg.LPUrls, cfg.APUrls = newPURLs, newPURLs
71	cfg.InitialCluster = ""
72	for i := 0; i < clusterN; i++ {
73		cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, pURLs[i].String())
74	}
75	cfg.InitialCluster = cfg.InitialCluster[1:]
76	cfg.InitialCluster += fmt.Sprintf(",%s=%s", cfg.Name, newPURLs[0].String())
77
78	srv, err := embed.StartEtcd(cfg)
79	if err != nil {
80		t.Fatal(err)
81	}
82	defer func() {
83		srv.Close()
84	}()
85	select {
86	case <-srv.Server.ReadyNotify():
87	case <-time.After(10 * time.Second):
88		t.Fatalf("failed to start the newly added etcd member")
89	}
90
91	cli2, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{newCURLs[0].String()}})
92	if err != nil {
93		t.Fatal(err)
94	}
95	defer cli2.Close()
96
97	ctx, cancel := context.WithTimeout(context.Background(), testutil.RequestTimeout)
98	mresp, err := cli2.MemberList(ctx)
99	cancel()
100	if err != nil {
101		t.Fatal(err)
102	}
103	if len(mresp.Members) != 4 {
104		t.Fatalf("expected 4 members, got %+v", mresp)
105	}
106
107	// make sure restored cluster has kept all data on recovery
108	var gresp *clientv3.GetResponse
109	ctx, cancel = context.WithTimeout(context.Background(), testutil.RequestTimeout)
110	gresp, err = cli2.Get(ctx, "foo", clientv3.WithPrefix())
111	cancel()
112	if err != nil {
113		t.Fatal(err)
114	}
115	for i := range gresp.Kvs {
116		if string(gresp.Kvs[i].Key) != kvs[i].k {
117			t.Fatalf("#%d: key expected %s, got %s", i, kvs[i].k, string(gresp.Kvs[i].Key))
118		}
119		if string(gresp.Kvs[i].Value) != kvs[i].v {
120			t.Fatalf("#%d: value expected %s, got %s", i, kvs[i].v, string(gresp.Kvs[i].Value))
121		}
122	}
123}
124