1package autopilot
2
3import (
4	"errors"
5	"fmt"
6	"testing"
7	"time"
8
9	"github.com/hashicorp/raft"
10	"github.com/stretchr/testify/require"
11)
12
13var (
14	injectedErr = fmt.Errorf("injected err")
15
16	test3VoterRaftConfiguration = raft.Configuration{
17		Servers: []raft.Server{
18			{
19				Suffrage: raft.Voter,
20				ID:       "7875975d-d54b-49c1-a400-9fefcc706c67",
21				Address:  "198.18.0.1:8300",
22			},
23			{
24				Suffrage: raft.Voter,
25				ID:       "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1",
26				Address:  "198.18.0.2:8300",
27			},
28			{
29				Suffrage: raft.Voter,
30				ID:       "e72eb8da-604d-47cd-bd7f-69ec120ea2b7",
31				Address:  "198.18.0.3:8300",
32			},
33		},
34	}
35)
36
37func isInjectedError(err error) bool {
38	return errors.Is(err, injectedErr)
39}
40
41func mockedRaftAutopilot(t *testing.T) (*Autopilot, *MockRaft) {
42	t.Helper()
43	mdel := NewMockApplicationIntegration(t)
44	mraft := NewMockRaft(t)
45
46	return New(mraft, mdel), mraft
47}
48
49func TestNumVoters(t *testing.T) {
50	type testCase struct {
51		future raftConfigFuture
52
53		expected int
54	}
55
56	cases := map[string]testCase{
57		"error": {
58			future: raftConfigFuture{
59				err: injectedErr,
60			},
61		},
62		"all-voters": {
63			future: raftConfigFuture{
64				config: test3VoterRaftConfiguration,
65			},
66			expected: 3,
67		},
68		"ignore-staging": {
69			future: raftConfigFuture{
70				config: raft.Configuration{
71					Servers: []raft.Server{
72						{
73							Suffrage: raft.Voter,
74							ID:       "7875975d-d54b-49c1-a400-9fefcc706c67",
75							Address:  "198.18.0.1:8300",
76						},
77						{
78							Suffrage: raft.Staging,
79							ID:       "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1",
80							Address:  "198.18.0.2:8300",
81						},
82						{
83							Suffrage: raft.Voter,
84							ID:       "e72eb8da-604d-47cd-bd7f-69ec120ea2b7",
85							Address:  "198.18.0.3:8300",
86						},
87					},
88				},
89			},
90			expected: 2,
91		},
92		"ignore-non-voter": {
93			future: raftConfigFuture{
94				config: raft.Configuration{
95					Servers: []raft.Server{
96						{
97							Suffrage: raft.Voter,
98							ID:       "7875975d-d54b-49c1-a400-9fefcc706c67",
99							Address:  "198.18.0.1:8300",
100						},
101						{
102							Suffrage: raft.Nonvoter,
103							ID:       "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1",
104							Address:  "198.18.0.2:8300",
105						},
106						{
107							Suffrage: raft.Voter,
108							ID:       "e72eb8da-604d-47cd-bd7f-69ec120ea2b7",
109							Address:  "198.18.0.3:8300",
110						},
111					},
112				},
113			},
114			expected: 2,
115		},
116	}
117
118	for name, tcase := range cases {
119		t.Run(name, func(t *testing.T) {
120			ap, mraft := mockedRaftAutopilot(t)
121			mraft.On("GetConfiguration").Return(&tcase.future).Once()
122
123			voters, err := ap.NumVoters()
124			if tcase.future.Error() != nil {
125				require.Zero(t, voters)
126				// error should just be passed through without modification
127				require.Equal(t, tcase.future.Error(), err)
128			} else {
129				require.Equal(t, tcase.expected, voters)
130				require.Nil(t, err)
131			}
132		})
133	}
134}
135
136func TestAddServer(t *testing.T) {
137	t.Run("existing-no-change", func(t *testing.T) {
138		ap, mraft := mockedRaftAutopilot(t)
139
140		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
141
142		require.Nil(t, ap.AddServer(&Server{ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", Address: "198.18.0.3:8300"}))
143		require.False(t, chanIsSelectable(ap.removeDeadCh))
144	})
145
146	t.Run("config-error", func(t *testing.T) {
147		ap, mraft := mockedRaftAutopilot(t)
148
149		mraft.On("GetConfiguration").Return(&raftConfigFuture{err: injectedErr}).Once()
150
151		err := ap.AddServer(&Server{ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", Address: "198.18.0.3:8300"})
152		require.Error(t, err)
153		require.True(t, isInjectedError(err))
154		require.False(t, chanIsSelectable(ap.removeDeadCh))
155	})
156
157	t.Run("new", func(t *testing.T) {
158		ap, mraft := mockedRaftAutopilot(t)
159
160		var newID raft.ServerID = "5e816fb6-d4e6-4b3a-b15a-afb3e6d5664b"
161		var newAddr raft.ServerAddress = "198.18.0.4:8300"
162
163		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
164		mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
165
166		require.Nil(t, ap.AddServer(&Server{ID: newID, Address: newAddr}))
167		require.True(t, chanIsSelectable(ap.removeDeadCh))
168	})
169
170	t.Run("existing-addr-change", func(t *testing.T) {
171		ap, mraft := mockedRaftAutopilot(t)
172
173		var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"
174		var newAddr raft.ServerAddress = "198.18.0.4:8300"
175
176		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
177		mraft.On("AddVoter", existingID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
178
179		require.Nil(t, ap.AddServer(&Server{ID: existingID, Address: newAddr}))
180		require.True(t, chanIsSelectable(ap.removeDeadCh))
181	})
182
183	t.Run("existing-id-change", func(t *testing.T) {
184		ap, mraft := mockedRaftAutopilot(t)
185
186		var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"
187		var newID raft.ServerID = "95e2f84d-ff36-4a48-bee7-a50863f17f55"
188		var existingAddr raft.ServerAddress = "198.18.0.2:8300"
189
190		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
191		mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
192		mraft.On("AddNonvoter", newID, existingAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
193
194		require.Nil(t, ap.AddServer(&Server{ID: newID, Address: existingAddr}))
195		require.True(t, chanIsSelectable(ap.removeDeadCh))
196	})
197
198	t.Run("remove-failure", func(t *testing.T) {
199		ap, mraft := mockedRaftAutopilot(t)
200
201		var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"
202		var newID raft.ServerID = "95e2f84d-ff36-4a48-bee7-a50863f17f55"
203		var existingAddr raft.ServerAddress = "198.18.0.2:8300"
204
205		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
206		mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once()
207
208		require.True(t, isInjectedError(ap.AddServer(&Server{ID: newID, Address: existingAddr})))
209		require.False(t, chanIsSelectable(ap.removeDeadCh))
210	})
211
212	t.Run("add-failure", func(t *testing.T) {
213		ap, mraft := mockedRaftAutopilot(t)
214
215		var newID raft.ServerID = "5e816fb6-d4e6-4b3a-b15a-afb3e6d5664b"
216		var newAddr raft.ServerAddress = "198.18.0.4:8300"
217
218		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
219		mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once()
220
221		require.True(t, isInjectedError(ap.AddServer(&Server{ID: newID, Address: newAddr})))
222		require.False(t, chanIsSelectable(ap.removeDeadCh))
223	})
224
225	t.Run("error-for-unsafe-removals", func(t *testing.T) {
226		ap, mraft := mockedRaftAutopilot(t)
227
228		var newID raft.ServerID = "c3195ec1-c229-48c6-bdab-4db54ed04807"
229		var newAddr raft.ServerAddress = "198.18.0.2:8300"
230
231		mraft.On("GetConfiguration").Once().Return(&raftConfigFuture{
232			config: raft.Configuration{
233				Servers: []raft.Server{
234					{
235						ID:       "36ced65e-be2c-478d-a2f4-56a9706041d0",
236						Address:  "198.18.0.1:8300",
237						Suffrage: raft.Voter,
238					},
239					{
240						ID:       "e0c54c7c-1363-46d0-950b-2cb4aad347a8",
241						Address:  "198.18.0.2:8300",
242						Suffrage: raft.Voter,
243					},
244				},
245			},
246		})
247
248		err := ap.AddServer(&Server{ID: newID, Address: newAddr})
249		require.Error(t, err)
250		require.Contains(t, err.Error(), "Preventing server addition that would require removal of too many servers and cause cluster instability")
251	})
252
253	t.Run("safe-non-voter-removals", func(t *testing.T) {
254		ap, mraft := mockedRaftAutopilot(t)
255
256		var existingID raft.ServerID = "e0c54c7c-1363-46d0-950b-2cb4aad347a8"
257		var newID raft.ServerID = "c3195ec1-c229-48c6-bdab-4db54ed04807"
258		var newAddr raft.ServerAddress = "198.18.0.2:8300"
259
260		mraft.On("GetConfiguration").Once().Return(&raftConfigFuture{
261			config: raft.Configuration{
262				Servers: []raft.Server{
263					{
264						ID:       "36ced65e-be2c-478d-a2f4-56a9706041d0",
265						Address:  "198.18.0.1:8300",
266						Suffrage: raft.Voter,
267					},
268					{
269						ID:       existingID,
270						Address:  "198.18.0.2:8300",
271						Suffrage: raft.Nonvoter,
272					},
273				},
274			},
275		})
276
277		mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
278		mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
279		require.NoError(t, ap.AddServer(&Server{ID: newID, Address: newAddr}))
280		require.True(t, chanIsSelectable(ap.removeDeadCh))
281	})
282}
283
284func TestRemoveServer(t *testing.T) {
285	t.Run("config-failure", func(t *testing.T) {
286		ap, mraft := mockedRaftAutopilot(t)
287
288		mraft.On("GetConfiguration").Return(&raftConfigFuture{err: injectedErr}).Once()
289		require.True(t, isInjectedError(ap.RemoveServer("ecfc5237-63c3-4b09-94b9-d5682d9ae5b1")))
290	})
291
292	t.Run("not-found", func(t *testing.T) {
293		ap, mraft := mockedRaftAutopilot(t)
294
295		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
296		require.NoError(t, ap.RemoveServer("29a3d904-6848-4e2f-928f-9abafc3f87ba"))
297	})
298
299	t.Run("removed", func(t *testing.T) {
300		ap, mraft := mockedRaftAutopilot(t)
301
302		var id raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"
303
304		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
305		mraft.On("RemoveServer", id, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once()
306		require.NoError(t, ap.RemoveServer(id))
307	})
308
309	t.Run("remove-failure", func(t *testing.T) {
310		ap, mraft := mockedRaftAutopilot(t)
311
312		var id raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"
313
314		mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once()
315		mraft.On("RemoveServer", id, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once()
316		require.True(t, isInjectedError(ap.RemoveServer(id)))
317	})
318}
319