1// Copyright 2018 The etcd Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package snapshot_test 16 17import ( 18 "context" 19 "fmt" 20 "testing" 21 "time" 22 23 "go.etcd.io/etcd/client/pkg/v3/testutil" 24 "go.etcd.io/etcd/client/v3" 25 "go.etcd.io/etcd/server/v3/embed" 26 "go.etcd.io/etcd/server/v3/etcdserver" 27 "go.etcd.io/etcd/tests/v3/integration" 28) 29 30// TestSnapshotV3RestoreMultiMemberAdd ensures that multiple members 31// can boot into the same cluster after being restored from a same 32// snapshot file, and also be able to add another member to the cluster. 33func TestSnapshotV3RestoreMultiMemberAdd(t *testing.T) { 34 integration.BeforeTest(t) 35 36 kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}} 37 dbPath := createSnapshotFile(t, kvs) 38 39 clusterN := 3 40 cURLs, pURLs, srvs := restoreCluster(t, clusterN, dbPath) 41 42 defer func() { 43 for i := 0; i < clusterN; i++ { 44 srvs[i].Close() 45 } 46 }() 47 48 // wait for health interval + leader election 49 time.Sleep(etcdserver.HealthInterval + 2*time.Second) 50 51 cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{cURLs[0].String()}}) 52 if err != nil { 53 t.Fatal(err) 54 } 55 defer cli.Close() 56 57 urls := newEmbedURLs(2) 58 newCURLs, newPURLs := urls[:1], urls[1:] 59 if _, err = cli.MemberAdd(context.Background(), []string{newPURLs[0].String()}); err != nil { 60 t.Fatal(err) 61 } 62 63 // wait for membership reconfiguration apply 64 time.Sleep(testutil.ApplyTimeout) 65 66 cfg := integration.NewEmbedConfig(t, "3") 67 cfg.InitialClusterToken = testClusterTkn 68 cfg.ClusterState = "existing" 69 cfg.LCUrls, cfg.ACUrls = newCURLs, newCURLs 70 cfg.LPUrls, cfg.APUrls = newPURLs, newPURLs 71 cfg.InitialCluster = "" 72 for i := 0; i < clusterN; i++ { 73 cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, pURLs[i].String()) 74 } 75 cfg.InitialCluster = cfg.InitialCluster[1:] 76 cfg.InitialCluster += fmt.Sprintf(",%s=%s", cfg.Name, newPURLs[0].String()) 77 78 srv, err := embed.StartEtcd(cfg) 79 if err != nil { 80 t.Fatal(err) 81 } 82 defer func() { 83 srv.Close() 84 }() 85 select { 86 case <-srv.Server.ReadyNotify(): 87 case <-time.After(10 * time.Second): 88 t.Fatalf("failed to start the newly added etcd member") 89 } 90 91 cli2, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{newCURLs[0].String()}}) 92 if err != nil { 93 t.Fatal(err) 94 } 95 defer cli2.Close() 96 97 ctx, cancel := context.WithTimeout(context.Background(), testutil.RequestTimeout) 98 mresp, err := cli2.MemberList(ctx) 99 cancel() 100 if err != nil { 101 t.Fatal(err) 102 } 103 if len(mresp.Members) != 4 { 104 t.Fatalf("expected 4 members, got %+v", mresp) 105 } 106 107 // make sure restored cluster has kept all data on recovery 108 var gresp *clientv3.GetResponse 109 ctx, cancel = context.WithTimeout(context.Background(), testutil.RequestTimeout) 110 gresp, err = cli2.Get(ctx, "foo", clientv3.WithPrefix()) 111 cancel() 112 if err != nil { 113 t.Fatal(err) 114 } 115 for i := range gresp.Kvs { 116 if string(gresp.Kvs[i].Key) != kvs[i].k { 117 t.Fatalf("#%d: key expected %s, got %s", i, kvs[i].k, string(gresp.Kvs[i].Key)) 118 } 119 if string(gresp.Kvs[i].Value) != kvs[i].v { 120 t.Fatalf("#%d: value expected %s, got %s", i, kvs[i].v, string(gresp.Kvs[i].Value)) 121 } 122 } 123} 124