1// Copyright 2016 The etcd Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package e2e
16
17import (
18	"encoding/json"
19	"fmt"
20	"io"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"strings"
25	"testing"
26	"time"
27
28	"go.etcd.io/etcd/clientv3/snapshot"
29	"go.etcd.io/etcd/pkg/expect"
30	"go.etcd.io/etcd/pkg/testutil"
31)
32
33func TestCtlV3Snapshot(t *testing.T) { testCtl(t, snapshotTest) }
34
35func snapshotTest(cx ctlCtx) {
36	maintenanceInitKeys(cx)
37
38	leaseID, err := ctlV3LeaseGrant(cx, 100)
39	if err != nil {
40		cx.t.Fatalf("snapshot: ctlV3LeaseGrant error (%v)", err)
41	}
42	if err = ctlV3Put(cx, "withlease", "withlease", leaseID); err != nil {
43		cx.t.Fatalf("snapshot: ctlV3Put error (%v)", err)
44	}
45
46	fpath := "test1.snapshot"
47	defer os.RemoveAll(fpath)
48
49	if err = ctlV3SnapshotSave(cx, fpath); err != nil {
50		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
51	}
52
53	st, err := getSnapshotStatus(cx, fpath)
54	if err != nil {
55		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
56	}
57	if st.Revision != 5 {
58		cx.t.Fatalf("expected 4, got %d", st.Revision)
59	}
60	if st.TotalKey < 4 {
61		cx.t.Fatalf("expected at least 4, got %d", st.TotalKey)
62	}
63}
64
65func TestCtlV3SnapshotCorrupt(t *testing.T) { testCtl(t, snapshotCorruptTest) }
66
67func snapshotCorruptTest(cx ctlCtx) {
68	fpath := "test2.snapshot"
69	defer os.RemoveAll(fpath)
70
71	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
72		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
73	}
74
75	// corrupt file
76	f, oerr := os.OpenFile(fpath, os.O_WRONLY, 0)
77	if oerr != nil {
78		cx.t.Fatal(oerr)
79	}
80	if _, err := f.Write(make([]byte, 512)); err != nil {
81		cx.t.Fatal(err)
82	}
83	f.Close()
84
85	defer os.RemoveAll("snap.etcd")
86	serr := spawnWithExpect(
87		append(cx.PrefixArgs(), "snapshot", "restore",
88			"--data-dir", "snap.etcd",
89			fpath),
90		"expected sha256")
91
92	if serr != nil {
93		cx.t.Fatal(serr)
94	}
95}
96
97// This test ensures that the snapshot status does not modify the snapshot file
98func TestCtlV3SnapshotStatusBeforeRestore(t *testing.T) { testCtl(t, snapshotStatusBeforeRestoreTest) }
99
100func snapshotStatusBeforeRestoreTest(cx ctlCtx) {
101	fpath := "test3.snapshot"
102	defer os.RemoveAll(fpath)
103
104	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
105		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
106	}
107
108	// snapshot status on the fresh snapshot file
109	_, err := getSnapshotStatus(cx, fpath)
110	if err != nil {
111		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
112	}
113
114	defer os.RemoveAll("snap.etcd")
115	serr := spawnWithExpect(
116		append(cx.PrefixArgs(), "snapshot", "restore",
117			"--data-dir", "snap.etcd",
118			fpath),
119		"added member")
120	if serr != nil {
121		cx.t.Fatal(serr)
122	}
123}
124
125func ctlV3SnapshotSave(cx ctlCtx, fpath string) error {
126	cmdArgs := append(cx.PrefixArgs(), "snapshot", "save", fpath)
127	return spawnWithExpect(cmdArgs, fmt.Sprintf("Snapshot saved at %s", fpath))
128}
129
130func getSnapshotStatus(cx ctlCtx, fpath string) (snapshot.Status, error) {
131	cmdArgs := append(cx.PrefixArgs(), "--write-out", "json", "snapshot", "status", fpath)
132
133	proc, err := spawnCmd(cmdArgs)
134	if err != nil {
135		return snapshot.Status{}, err
136	}
137	var txt string
138	txt, err = proc.Expect("totalKey")
139	if err != nil {
140		return snapshot.Status{}, err
141	}
142	if err = proc.Close(); err != nil {
143		return snapshot.Status{}, err
144	}
145
146	resp := snapshot.Status{}
147	dec := json.NewDecoder(strings.NewReader(txt))
148	if err := dec.Decode(&resp); err == io.EOF {
149		return snapshot.Status{}, err
150	}
151	return resp, nil
152}
153
154// TestIssue6361 ensures new member that starts with snapshot correctly
155// syncs up with other members and serve correct data.
156func TestIssue6361(t *testing.T) {
157	defer testutil.AfterTest(t)
158	mustEtcdctl(t)
159	os.Setenv("ETCDCTL_API", "3")
160	defer os.Unsetenv("ETCDCTL_API")
161
162	epc, err := newEtcdProcessCluster(&etcdProcessClusterConfig{
163		clusterSize:  1,
164		initialToken: "new",
165		keepDataDir:  true,
166	})
167	if err != nil {
168		t.Fatalf("could not start etcd process cluster (%v)", err)
169	}
170	defer func() {
171		if errC := epc.Close(); errC != nil {
172			t.Fatalf("error closing etcd processes (%v)", errC)
173		}
174	}()
175
176	dialTimeout := 7 * time.Second
177	prefixArgs := []string{ctlBinPath, "--endpoints", strings.Join(epc.EndpointsV3(), ","), "--dial-timeout", dialTimeout.String()}
178
179	// write some keys
180	kvs := []kv{{"foo1", "val1"}, {"foo2", "val2"}, {"foo3", "val3"}}
181	for i := range kvs {
182		if err = spawnWithExpect(append(prefixArgs, "put", kvs[i].key, kvs[i].val), "OK"); err != nil {
183			t.Fatal(err)
184		}
185	}
186
187	fpath := filepath.Join(os.TempDir(), "test.snapshot")
188	defer os.RemoveAll(fpath)
189
190	// etcdctl save snapshot
191	if err = spawnWithExpect(append(prefixArgs, "snapshot", "save", fpath), fmt.Sprintf("Snapshot saved at %s", fpath)); err != nil {
192		t.Fatal(err)
193	}
194
195	if err = epc.procs[0].Stop(); err != nil {
196		t.Fatal(err)
197	}
198
199	newDataDir := filepath.Join(os.TempDir(), "test.data")
200	defer os.RemoveAll(newDataDir)
201
202	// etcdctl restore the snapshot
203	err = spawnWithExpect([]string{ctlBinPath, "snapshot", "restore", fpath, "--name", epc.procs[0].Config().name, "--initial-cluster", epc.procs[0].Config().initialCluster, "--initial-cluster-token", epc.procs[0].Config().initialToken, "--initial-advertise-peer-urls", epc.procs[0].Config().purl.String(), "--data-dir", newDataDir}, "added member")
204	if err != nil {
205		t.Fatal(err)
206	}
207
208	// start the etcd member using the restored snapshot
209	epc.procs[0].Config().dataDirPath = newDataDir
210	for i := range epc.procs[0].Config().args {
211		if epc.procs[0].Config().args[i] == "--data-dir" {
212			epc.procs[0].Config().args[i+1] = newDataDir
213		}
214	}
215	if err = epc.procs[0].Restart(); err != nil {
216		t.Fatal(err)
217	}
218
219	// ensure the restored member has the correct data
220	for i := range kvs {
221		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
222			t.Fatal(err)
223		}
224	}
225
226	// add a new member into the cluster
227	clientURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+30)
228	peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+31)
229	err = spawnWithExpect(append(prefixArgs, "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)), " added to cluster ")
230	if err != nil {
231		t.Fatal(err)
232	}
233
234	var newDataDir2 string
235	newDataDir2, err = ioutil.TempDir("", "newdata2")
236	if err != nil {
237		t.Fatal(err)
238	}
239	defer os.RemoveAll(newDataDir2)
240
241	name2 := "infra2"
242	initialCluster2 := epc.procs[0].Config().initialCluster + fmt.Sprintf(",%s=%s", name2, peerURL)
243
244	// start the new member
245	var nepc *expect.ExpectProcess
246	nepc, err = spawnCmd([]string{epc.procs[0].Config().execPath, "--name", name2,
247		"--listen-client-urls", clientURL, "--advertise-client-urls", clientURL,
248		"--listen-peer-urls", peerURL, "--initial-advertise-peer-urls", peerURL,
249		"--initial-cluster", initialCluster2, "--initial-cluster-state", "existing", "--data-dir", newDataDir2})
250	if err != nil {
251		t.Fatal(err)
252	}
253	if _, err = nepc.Expect("enabled capabilities for version"); err != nil {
254		t.Fatal(err)
255	}
256
257	prefixArgs = []string{ctlBinPath, "--endpoints", clientURL, "--dial-timeout", dialTimeout.String()}
258
259	// ensure added member has data from incoming snapshot
260	for i := range kvs {
261		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
262			t.Fatal(err)
263		}
264	}
265
266	if err = nepc.Stop(); err != nil {
267		t.Fatal(err)
268	}
269}
270