1package autopilot 2 3import ( 4 "errors" 5 "fmt" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/raft" 10 "github.com/stretchr/testify/require" 11) 12 13var ( 14 injectedErr = fmt.Errorf("injected err") 15 16 test3VoterRaftConfiguration = raft.Configuration{ 17 Servers: []raft.Server{ 18 { 19 Suffrage: raft.Voter, 20 ID: "7875975d-d54b-49c1-a400-9fefcc706c67", 21 Address: "198.18.0.1:8300", 22 }, 23 { 24 Suffrage: raft.Voter, 25 ID: "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1", 26 Address: "198.18.0.2:8300", 27 }, 28 { 29 Suffrage: raft.Voter, 30 ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", 31 Address: "198.18.0.3:8300", 32 }, 33 }, 34 } 35) 36 37func isInjectedError(err error) bool { 38 return errors.Is(err, injectedErr) 39} 40 41func mockedRaftAutopilot(t *testing.T) (*Autopilot, *MockRaft) { 42 t.Helper() 43 mdel := NewMockApplicationIntegration(t) 44 mraft := NewMockRaft(t) 45 46 return New(mraft, mdel), mraft 47} 48 49func TestNumVoters(t *testing.T) { 50 type testCase struct { 51 future raftConfigFuture 52 53 expected int 54 } 55 56 cases := map[string]testCase{ 57 "error": { 58 future: raftConfigFuture{ 59 err: injectedErr, 60 }, 61 }, 62 "all-voters": { 63 future: raftConfigFuture{ 64 config: test3VoterRaftConfiguration, 65 }, 66 expected: 3, 67 }, 68 "ignore-staging": { 69 future: raftConfigFuture{ 70 config: raft.Configuration{ 71 Servers: []raft.Server{ 72 { 73 Suffrage: raft.Voter, 74 ID: "7875975d-d54b-49c1-a400-9fefcc706c67", 75 Address: "198.18.0.1:8300", 76 }, 77 { 78 Suffrage: raft.Staging, 79 ID: "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1", 80 Address: "198.18.0.2:8300", 81 }, 82 { 83 Suffrage: raft.Voter, 84 ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", 85 Address: "198.18.0.3:8300", 86 }, 87 }, 88 }, 89 }, 90 expected: 2, 91 }, 92 "ignore-non-voter": { 93 future: raftConfigFuture{ 94 config: raft.Configuration{ 95 Servers: []raft.Server{ 96 { 97 Suffrage: raft.Voter, 98 ID: "7875975d-d54b-49c1-a400-9fefcc706c67", 99 Address: "198.18.0.1:8300", 100 }, 101 { 102 Suffrage: raft.Nonvoter, 103 ID: "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1", 104 Address: "198.18.0.2:8300", 105 }, 106 { 107 Suffrage: raft.Voter, 108 ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", 109 Address: "198.18.0.3:8300", 110 }, 111 }, 112 }, 113 }, 114 expected: 2, 115 }, 116 } 117 118 for name, tcase := range cases { 119 t.Run(name, func(t *testing.T) { 120 ap, mraft := mockedRaftAutopilot(t) 121 mraft.On("GetConfiguration").Return(&tcase.future).Once() 122 123 voters, err := ap.NumVoters() 124 if tcase.future.Error() != nil { 125 require.Zero(t, voters) 126 // error should just be passed through without modification 127 require.Equal(t, tcase.future.Error(), err) 128 } else { 129 require.Equal(t, tcase.expected, voters) 130 require.Nil(t, err) 131 } 132 }) 133 } 134} 135 136func TestAddServer(t *testing.T) { 137 t.Run("existing-no-change", func(t *testing.T) { 138 ap, mraft := mockedRaftAutopilot(t) 139 140 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 141 142 require.Nil(t, ap.AddServer(&Server{ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", Address: "198.18.0.3:8300"})) 143 require.False(t, chanIsSelectable(ap.removeDeadCh)) 144 }) 145 146 t.Run("config-error", func(t *testing.T) { 147 ap, mraft := mockedRaftAutopilot(t) 148 149 mraft.On("GetConfiguration").Return(&raftConfigFuture{err: injectedErr}).Once() 150 151 err := ap.AddServer(&Server{ID: "e72eb8da-604d-47cd-bd7f-69ec120ea2b7", Address: "198.18.0.3:8300"}) 152 require.Error(t, err) 153 require.True(t, isInjectedError(err)) 154 require.False(t, chanIsSelectable(ap.removeDeadCh)) 155 }) 156 157 t.Run("new", func(t *testing.T) { 158 ap, mraft := mockedRaftAutopilot(t) 159 160 var newID raft.ServerID = "5e816fb6-d4e6-4b3a-b15a-afb3e6d5664b" 161 var newAddr raft.ServerAddress = "198.18.0.4:8300" 162 163 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 164 mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 165 166 require.Nil(t, ap.AddServer(&Server{ID: newID, Address: newAddr})) 167 require.True(t, chanIsSelectable(ap.removeDeadCh)) 168 }) 169 170 t.Run("existing-addr-change", func(t *testing.T) { 171 ap, mraft := mockedRaftAutopilot(t) 172 173 var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1" 174 var newAddr raft.ServerAddress = "198.18.0.4:8300" 175 176 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 177 mraft.On("AddVoter", existingID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 178 179 require.Nil(t, ap.AddServer(&Server{ID: existingID, Address: newAddr})) 180 require.True(t, chanIsSelectable(ap.removeDeadCh)) 181 }) 182 183 t.Run("existing-id-change", func(t *testing.T) { 184 ap, mraft := mockedRaftAutopilot(t) 185 186 var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1" 187 var newID raft.ServerID = "95e2f84d-ff36-4a48-bee7-a50863f17f55" 188 var existingAddr raft.ServerAddress = "198.18.0.2:8300" 189 190 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 191 mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 192 mraft.On("AddNonvoter", newID, existingAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 193 194 require.Nil(t, ap.AddServer(&Server{ID: newID, Address: existingAddr})) 195 require.True(t, chanIsSelectable(ap.removeDeadCh)) 196 }) 197 198 t.Run("remove-failure", func(t *testing.T) { 199 ap, mraft := mockedRaftAutopilot(t) 200 201 var existingID raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1" 202 var newID raft.ServerID = "95e2f84d-ff36-4a48-bee7-a50863f17f55" 203 var existingAddr raft.ServerAddress = "198.18.0.2:8300" 204 205 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 206 mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once() 207 208 require.True(t, isInjectedError(ap.AddServer(&Server{ID: newID, Address: existingAddr}))) 209 require.False(t, chanIsSelectable(ap.removeDeadCh)) 210 }) 211 212 t.Run("add-failure", func(t *testing.T) { 213 ap, mraft := mockedRaftAutopilot(t) 214 215 var newID raft.ServerID = "5e816fb6-d4e6-4b3a-b15a-afb3e6d5664b" 216 var newAddr raft.ServerAddress = "198.18.0.4:8300" 217 218 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 219 mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once() 220 221 require.True(t, isInjectedError(ap.AddServer(&Server{ID: newID, Address: newAddr}))) 222 require.False(t, chanIsSelectable(ap.removeDeadCh)) 223 }) 224 225 t.Run("error-for-unsafe-removals", func(t *testing.T) { 226 ap, mraft := mockedRaftAutopilot(t) 227 228 var newID raft.ServerID = "c3195ec1-c229-48c6-bdab-4db54ed04807" 229 var newAddr raft.ServerAddress = "198.18.0.2:8300" 230 231 mraft.On("GetConfiguration").Once().Return(&raftConfigFuture{ 232 config: raft.Configuration{ 233 Servers: []raft.Server{ 234 { 235 ID: "36ced65e-be2c-478d-a2f4-56a9706041d0", 236 Address: "198.18.0.1:8300", 237 Suffrage: raft.Voter, 238 }, 239 { 240 ID: "e0c54c7c-1363-46d0-950b-2cb4aad347a8", 241 Address: "198.18.0.2:8300", 242 Suffrage: raft.Voter, 243 }, 244 }, 245 }, 246 }) 247 248 err := ap.AddServer(&Server{ID: newID, Address: newAddr}) 249 require.Error(t, err) 250 require.Contains(t, err.Error(), "Preventing server addition that would require removal of too many servers and cause cluster instability") 251 }) 252 253 t.Run("safe-non-voter-removals", func(t *testing.T) { 254 ap, mraft := mockedRaftAutopilot(t) 255 256 var existingID raft.ServerID = "e0c54c7c-1363-46d0-950b-2cb4aad347a8" 257 var newID raft.ServerID = "c3195ec1-c229-48c6-bdab-4db54ed04807" 258 var newAddr raft.ServerAddress = "198.18.0.2:8300" 259 260 mraft.On("GetConfiguration").Once().Return(&raftConfigFuture{ 261 config: raft.Configuration{ 262 Servers: []raft.Server{ 263 { 264 ID: "36ced65e-be2c-478d-a2f4-56a9706041d0", 265 Address: "198.18.0.1:8300", 266 Suffrage: raft.Voter, 267 }, 268 { 269 ID: existingID, 270 Address: "198.18.0.2:8300", 271 Suffrage: raft.Nonvoter, 272 }, 273 }, 274 }, 275 }) 276 277 mraft.On("RemoveServer", existingID, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 278 mraft.On("AddNonvoter", newID, newAddr, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 279 require.NoError(t, ap.AddServer(&Server{ID: newID, Address: newAddr})) 280 require.True(t, chanIsSelectable(ap.removeDeadCh)) 281 }) 282} 283 284func TestRemoveServer(t *testing.T) { 285 t.Run("config-failure", func(t *testing.T) { 286 ap, mraft := mockedRaftAutopilot(t) 287 288 mraft.On("GetConfiguration").Return(&raftConfigFuture{err: injectedErr}).Once() 289 require.True(t, isInjectedError(ap.RemoveServer("ecfc5237-63c3-4b09-94b9-d5682d9ae5b1"))) 290 }) 291 292 t.Run("not-found", func(t *testing.T) { 293 ap, mraft := mockedRaftAutopilot(t) 294 295 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 296 require.NoError(t, ap.RemoveServer("29a3d904-6848-4e2f-928f-9abafc3f87ba")) 297 }) 298 299 t.Run("removed", func(t *testing.T) { 300 ap, mraft := mockedRaftAutopilot(t) 301 302 var id raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1" 303 304 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 305 mraft.On("RemoveServer", id, uint64(0), time.Duration(0)).Return(&raftIndexFuture{}).Once() 306 require.NoError(t, ap.RemoveServer(id)) 307 }) 308 309 t.Run("remove-failure", func(t *testing.T) { 310 ap, mraft := mockedRaftAutopilot(t) 311 312 var id raft.ServerID = "ecfc5237-63c3-4b09-94b9-d5682d9ae5b1" 313 314 mraft.On("GetConfiguration").Return(&raftConfigFuture{config: test3VoterRaftConfiguration}).Once() 315 mraft.On("RemoveServer", id, uint64(0), time.Duration(0)).Return(&raftIndexFuture{err: injectedErr}).Once() 316 require.True(t, isInjectedError(ap.RemoveServer(id))) 317 }) 318} 319