1
2 /**
3 * Copyright (C) 2018-present MongoDB, Inc.
4 *
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the Server Side Public License, version 1,
7 * as published by MongoDB, Inc.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * Server Side Public License for more details.
13 *
14 * You should have received a copy of the Server Side Public License
15 * along with this program. If not, see
16 * <http://www.mongodb.com/licensing/server-side-public-license>.
17 *
18 * As a special exception, the copyright holders give permission to link the
19 * code of portions of this program with the OpenSSL library under certain
20 * conditions as described in each individual source file and distribute
21 * linked combinations including the program with the OpenSSL library. You
22 * must comply with the Server Side Public License in all respects for
23 * all of the code used other than as permitted herein. If you modify file(s)
24 * with this exception, you may extend this exception to your version of the
25 * file(s), but you are not obligated to do so. If you do not wish to do so,
26 * delete this exception statement from your version. If you delete this
27 * exception statement from all source files in the program, then also delete
28 * it in the license file.
29 */
30
31 #include "mongo/platform/basic.h"
32
33 #include <iostream>
34
35 #include "mongo/bson/json.h"
36 #include "mongo/db/repl/heartbeat_response_action.h"
37 #include "mongo/db/repl/member_data.h"
38 #include "mongo/db/repl/repl_set_heartbeat_args.h"
39 #include "mongo/db/repl/repl_set_heartbeat_response.h"
40 #include "mongo/db/repl/repl_set_request_votes_args.h"
41 #include "mongo/db/repl/topology_coordinator.h"
42 #include "mongo/db/server_options.h"
43 #include "mongo/executor/task_executor.h"
44 #include "mongo/logger/logger.h"
45 #include "mongo/rpc/metadata/oplog_query_metadata.h"
46 #include "mongo/rpc/metadata/repl_set_metadata.h"
47 #include "mongo/unittest/unittest.h"
48 #include "mongo/util/assert_util.h"
49 #include "mongo/util/net/hostandport.h"
50 #include "mongo/util/scopeguard.h"
51 #include "mongo/util/time_support.h"
52
53 #define ASSERT_NO_ACTION(EXPRESSION) \
54 ASSERT_EQUALS(mongo::repl::HeartbeatResponseAction::NoAction, (EXPRESSION))
55
56 using std::unique_ptr;
57 using mongo::rpc::ReplSetMetadata;
58 using mongo::rpc::OplogQueryMetadata;
59
60 namespace mongo {
61 namespace repl {
62 namespace {
63
operator ++(Date_t & d,int)64 Date_t operator++(Date_t& d, int) {
65 Date_t result = d;
66 d += Milliseconds(1);
67 return result;
68 }
69
stringContains(const std::string & haystack,const std::string & needle)70 bool stringContains(const std::string& haystack, const std::string& needle) {
71 return haystack.find(needle) != std::string::npos;
72 }
73
74 class TopoCoordTest : public mongo::unittest::Test {
75 public:
setUp()76 virtual void setUp() {
77 _options = TopologyCoordinator::Options{};
78 _options.maxSyncSourceLagSecs = Seconds{100};
79 _topo.reset(new TopologyCoordinator(_options));
80 _now = Date_t();
81 _selfIndex = -1;
82 _cbData.reset(new executor::TaskExecutor::CallbackArgs(
83 NULL, executor::TaskExecutor::CallbackHandle(), Status::OK()));
84 }
85
tearDown()86 virtual void tearDown() {
87 _topo.reset(NULL);
88 _cbData.reset(NULL);
89 }
90
91 protected:
getTopoCoord()92 TopologyCoordinator& getTopoCoord() {
93 return *_topo;
94 }
cbData()95 executor::TaskExecutor::CallbackArgs cbData() {
96 return *_cbData;
97 }
now()98 Date_t& now() {
99 return _now;
100 }
101
setOptions(const TopologyCoordinator::Options & options)102 void setOptions(const TopologyCoordinator::Options& options) {
103 _options = options;
104 _topo.reset(new TopologyCoordinator(_options));
105 }
106
countLogLinesContaining(const std::string & needle)107 int64_t countLogLinesContaining(const std::string& needle) {
108 return std::count_if(getCapturedLogMessages().begin(),
109 getCapturedLogMessages().end(),
110 stdx::bind(stringContains, stdx::placeholders::_1, needle));
111 }
112
makeSelfPrimary(const Timestamp & electionTimestamp=Timestamp (0,0),const OpTime & firstOpTimeOfTerm=OpTime ())113 void makeSelfPrimary(const Timestamp& electionTimestamp = Timestamp(0, 0),
114 const OpTime& firstOpTimeOfTerm = OpTime()) {
115 getTopoCoord().changeMemberState_forTest(MemberState::RS_PRIMARY, electionTimestamp);
116 getTopoCoord()._setCurrentPrimaryForTest(_selfIndex);
117 ASSERT_OK(getTopoCoord().completeTransitionToPrimary(firstOpTimeOfTerm));
118 }
119
setSelfMemberState(const MemberState & newState)120 void setSelfMemberState(const MemberState& newState) {
121 getTopoCoord().changeMemberState_forTest(newState);
122 }
123
getCurrentPrimaryIndex()124 int getCurrentPrimaryIndex() {
125 return getTopoCoord().getCurrentPrimaryIndex();
126 }
127
getCurrentPrimaryHost()128 HostAndPort getCurrentPrimaryHost() {
129 return _currentConfig.getMemberAt(getTopoCoord().getCurrentPrimaryIndex()).getHostAndPort();
130 }
131
132 // Update config and set selfIndex
133 // If "now" is passed in, set _now to now+1
updateConfig(BSONObj cfg,int selfIndex,Date_t now=Date_t::fromMillisSinceEpoch (-1),const OpTime & lastOp=OpTime ())134 void updateConfig(BSONObj cfg,
135 int selfIndex,
136 Date_t now = Date_t::fromMillisSinceEpoch(-1),
137 const OpTime& lastOp = OpTime()) {
138 ReplSetConfig config;
139 ASSERT_OK(config.initialize(cfg));
140 ASSERT_OK(config.validate());
141
142 _selfIndex = selfIndex;
143
144 if (now == Date_t::fromMillisSinceEpoch(-1)) {
145 getTopoCoord().updateConfig(config, selfIndex, _now);
146 _now += Milliseconds(1);
147 } else {
148 invariant(now > _now);
149 getTopoCoord().updateConfig(config, selfIndex, now);
150 _now = now + Milliseconds(1);
151 }
152
153 _currentConfig = config;
154 }
155
156 // Make the metadata coming from sync source. Only set visibleOpTime.
makeReplSetMetadata(OpTime visibleOpTime=OpTime ())157 ReplSetMetadata makeReplSetMetadata(OpTime visibleOpTime = OpTime()) {
158 return ReplSetMetadata(_topo->getTerm(),
159 OpTime(),
160 visibleOpTime,
161 _currentConfig.getConfigVersion(),
162 OID(),
163 -1,
164 -1);
165 }
166
167 // Make the metadata coming from sync source. Only set lastAppliedOpTime.
makeOplogQueryMetadata(OpTime lastAppliedOpTime=OpTime ())168 OplogQueryMetadata makeOplogQueryMetadata(OpTime lastAppliedOpTime = OpTime()) {
169 return OplogQueryMetadata(OpTime(), lastAppliedOpTime, -1, -1, -1);
170 }
171
172
receiveUpHeartbeat(const HostAndPort & member,const std::string & setName,MemberState memberState,const OpTime & electionTime,const OpTime & lastOpTimeSender)173 HeartbeatResponseAction receiveUpHeartbeat(const HostAndPort& member,
174 const std::string& setName,
175 MemberState memberState,
176 const OpTime& electionTime,
177 const OpTime& lastOpTimeSender) {
178 return _receiveHeartbeatHelper(Status::OK(),
179 member,
180 setName,
181 memberState,
182 electionTime.getTimestamp(),
183 lastOpTimeSender,
184 Milliseconds(1));
185 }
186
receiveDownHeartbeat(const HostAndPort & member,const std::string & setName,const OpTime & lastOpTimeReceiver,ErrorCodes::Error errcode=ErrorCodes::HostUnreachable)187 HeartbeatResponseAction receiveDownHeartbeat(
188 const HostAndPort& member,
189 const std::string& setName,
190 const OpTime& lastOpTimeReceiver,
191 ErrorCodes::Error errcode = ErrorCodes::HostUnreachable) {
192 // timed out heartbeat to mark a node as down
193
194 Milliseconds roundTripTime{ReplSetConfig::kDefaultHeartbeatTimeoutPeriod};
195 return _receiveHeartbeatHelper(Status(errcode, ""),
196 member,
197 setName,
198 MemberState::RS_UNKNOWN,
199 Timestamp(),
200 OpTime(),
201 roundTripTime);
202 }
203
heartbeatFromMember(const HostAndPort & member,const std::string & setName,MemberState memberState,const OpTime & lastOpTimeSender,Milliseconds roundTripTime=Milliseconds (1))204 HeartbeatResponseAction heartbeatFromMember(const HostAndPort& member,
205 const std::string& setName,
206 MemberState memberState,
207 const OpTime& lastOpTimeSender,
208 Milliseconds roundTripTime = Milliseconds(1)) {
209 return _receiveHeartbeatHelper(Status::OK(),
210 member,
211 setName,
212 memberState,
213 Timestamp(),
214 lastOpTimeSender,
215 roundTripTime);
216 }
217
218 private:
_receiveHeartbeatHelper(Status responseStatus,const HostAndPort & member,const std::string & setName,MemberState memberState,Timestamp electionTime,const OpTime & lastOpTimeSender,Milliseconds roundTripTime)219 HeartbeatResponseAction _receiveHeartbeatHelper(Status responseStatus,
220 const HostAndPort& member,
221 const std::string& setName,
222 MemberState memberState,
223 Timestamp electionTime,
224 const OpTime& lastOpTimeSender,
225 Milliseconds roundTripTime) {
226 ReplSetHeartbeatResponse hb;
227 hb.setConfigVersion(1);
228 hb.setState(memberState);
229 hb.setDurableOpTime(lastOpTimeSender);
230 hb.setAppliedOpTime(lastOpTimeSender);
231 hb.setElectionTime(electionTime);
232
233 StatusWith<ReplSetHeartbeatResponse> hbResponse = responseStatus.isOK()
234 ? StatusWith<ReplSetHeartbeatResponse>(hb)
235 : StatusWith<ReplSetHeartbeatResponse>(responseStatus);
236
237 getTopoCoord().prepareHeartbeatRequest(now(), setName, member);
238 now() += roundTripTime;
239 return getTopoCoord().processHeartbeatResponse(now(), roundTripTime, member, hbResponse);
240 }
241
242 private:
243 unique_ptr<TopologyCoordinator> _topo;
244 unique_ptr<executor::TaskExecutor::CallbackArgs> _cbData;
245 ReplSetConfig _currentConfig;
246 Date_t _now;
247 int _selfIndex;
248 TopologyCoordinator::Options _options;
249 };
250
TEST_F(TopoCoordTest,NodeReturnsSecondaryWithMostRecentDataAsSyncSource)251 TEST_F(TopoCoordTest, NodeReturnsSecondaryWithMostRecentDataAsSyncSource) {
252 // if we do not have an index in the config, we should get an empty syncsource
253 HostAndPort newSyncSource = getTopoCoord().chooseNewSyncSource(
254 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
255 ASSERT_TRUE(newSyncSource.empty());
256
257 updateConfig(BSON("_id"
258 << "rs0"
259 << "version"
260 << 1
261 << "members"
262 << BSON_ARRAY(BSON("_id" << 10 << "host"
263 << "hself")
264 << BSON("_id" << 20 << "host"
265 << "h2")
266 << BSON("_id" << 30 << "host"
267 << "h3"))),
268 0);
269 setSelfMemberState(MemberState::RS_SECONDARY);
270
271 // member h2 is the furthest ahead
272 heartbeatFromMember(
273 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
274 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime());
275
276 // We start with no sync source
277 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
278
279 // Fail due to insufficient number of pings
280 newSyncSource = getTopoCoord().chooseNewSyncSource(
281 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
282 ASSERT_EQUALS(getTopoCoord().getSyncSourceAddress(), newSyncSource);
283 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
284
285 // Record 2nd round of pings to allow choosing a new sync source; all members equidistant
286 heartbeatFromMember(
287 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
288 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime());
289
290 // Should choose h2, since it is furthest ahead
291 newSyncSource = getTopoCoord().chooseNewSyncSource(
292 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
293 ASSERT_EQUALS(getTopoCoord().getSyncSourceAddress(), newSyncSource);
294 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
295
296 // h3 becomes further ahead, so it should be chosen
297 heartbeatFromMember(
298 HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(2, 0), 0));
299 getTopoCoord().chooseNewSyncSource(
300 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
301 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
302
303 // h3 becomes an invalid candidate for sync source; should choose h2 again
304 heartbeatFromMember(
305 HostAndPort("h3"), "rs0", MemberState::RS_RECOVERING, OpTime(Timestamp(2, 0), 0));
306 getTopoCoord().chooseNewSyncSource(
307 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
308 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
309
310 // h3 back in SECONDARY and ahead
311 heartbeatFromMember(
312 HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(2, 0), 0));
313 getTopoCoord().chooseNewSyncSource(
314 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
315 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
316
317 // h3 goes down
318 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime());
319 getTopoCoord().chooseNewSyncSource(
320 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
321 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
322
323 // h3 back up and ahead
324 heartbeatFromMember(
325 HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(2, 0), 0));
326 getTopoCoord().chooseNewSyncSource(
327 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
328 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
329 }
330
TEST_F(TopoCoordTest,NodeReturnsClosestValidSyncSourceAsSyncSource)331 TEST_F(TopoCoordTest, NodeReturnsClosestValidSyncSourceAsSyncSource) {
332 updateConfig(BSON("_id"
333 << "rs0"
334 << "version"
335 << 1
336 << "members"
337 << BSON_ARRAY(BSON("_id" << 1 << "host"
338 << "hself")
339 << BSON("_id" << 10 << "host"
340 << "h1")
341 << BSON("_id" << 20 << "host"
342 << "h2"
343 << "buildIndexes"
344 << false
345 << "priority"
346 << 0)
347 << BSON("_id" << 30 << "host"
348 << "h3"
349 << "hidden"
350 << true
351 << "priority"
352 << 0
353 << "votes"
354 << 0)
355 << BSON("_id" << 40 << "host"
356 << "h4"
357 << "arbiterOnly"
358 << true)
359 << BSON("_id" << 50 << "host"
360 << "h5"
361 << "slaveDelay"
362 << 1
363 << "priority"
364 << 0)
365 << BSON("_id" << 60 << "host"
366 << "h6")
367 << BSON("_id" << 70 << "host"
368 << "hprimary"))),
369 0);
370
371 setSelfMemberState(MemberState::RS_SECONDARY);
372 OpTime lastOpTimeWeApplied = OpTime(Timestamp(100, 0), 0);
373
374 heartbeatFromMember(HostAndPort("h1"),
375 "rs0",
376 MemberState::RS_SECONDARY,
377 OpTime(Timestamp(501, 0), 0),
378 Milliseconds(700));
379 heartbeatFromMember(HostAndPort("h2"),
380 "rs0",
381 MemberState::RS_SECONDARY,
382 OpTime(Timestamp(501, 0), 0),
383 Milliseconds(600));
384 heartbeatFromMember(HostAndPort("h3"),
385 "rs0",
386 MemberState::RS_SECONDARY,
387 OpTime(Timestamp(501, 0), 0),
388 Milliseconds(500));
389 heartbeatFromMember(HostAndPort("h4"),
390 "rs0",
391 MemberState::RS_SECONDARY,
392 OpTime(Timestamp(501, 0), 0),
393 Milliseconds(400));
394 heartbeatFromMember(HostAndPort("h5"),
395 "rs0",
396 MemberState::RS_SECONDARY,
397 OpTime(Timestamp(501, 0), 0),
398 Milliseconds(300));
399
400 // This node is lagged further than maxSyncSourceLagSeconds.
401 heartbeatFromMember(HostAndPort("h6"),
402 "rs0",
403 MemberState::RS_SECONDARY,
404 OpTime(Timestamp(499, 0), 0),
405 Milliseconds(200));
406
407 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
408 heartbeatFromMember(HostAndPort("hprimary"),
409 "rs0",
410 MemberState::RS_PRIMARY,
411 OpTime(Timestamp(600, 0), 0),
412 Milliseconds(100));
413 ASSERT_EQUALS(7, getCurrentPrimaryIndex());
414
415 // Record 2nd round of pings to allow choosing a new sync source
416 heartbeatFromMember(HostAndPort("h1"),
417 "rs0",
418 MemberState::RS_SECONDARY,
419 OpTime(Timestamp(501, 0), 0),
420 Milliseconds(700));
421 heartbeatFromMember(HostAndPort("h2"),
422 "rs0",
423 MemberState::RS_SECONDARY,
424 OpTime(Timestamp(501, 0), 0),
425 Milliseconds(600));
426 heartbeatFromMember(HostAndPort("h3"),
427 "rs0",
428 MemberState::RS_SECONDARY,
429 OpTime(Timestamp(501, 0), 0),
430 Milliseconds(500));
431 heartbeatFromMember(HostAndPort("h4"),
432 "rs0",
433 MemberState::RS_SECONDARY,
434 OpTime(Timestamp(501, 0), 0),
435 Milliseconds(400));
436 heartbeatFromMember(HostAndPort("h5"),
437 "rs0",
438 MemberState::RS_SECONDARY,
439 OpTime(Timestamp(501, 0), 0),
440 Milliseconds(300));
441 heartbeatFromMember(HostAndPort("h6"),
442 "rs0",
443 MemberState::RS_SECONDARY,
444 OpTime(Timestamp(499, 0), 0),
445 Milliseconds(200));
446 heartbeatFromMember(HostAndPort("hprimary"),
447 "rs0",
448 MemberState::RS_PRIMARY,
449 OpTime(Timestamp(600, 0), 0),
450 Milliseconds(100));
451
452 // Should choose primary first; it's closest
453 getTopoCoord().chooseNewSyncSource(
454 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
455 ASSERT_EQUALS(HostAndPort("hprimary"), getTopoCoord().getSyncSourceAddress());
456
457 // Primary goes far far away
458 heartbeatFromMember(HostAndPort("hprimary"),
459 "rs0",
460 MemberState::RS_PRIMARY,
461 OpTime(Timestamp(600, 0), 0),
462 Milliseconds(100000000));
463
464 // Should choose h4. (if an arbiter has an oplog, it's a valid sync source)
465 // h6 is not considered because it is outside the maxSyncLagSeconds window,
466 getTopoCoord().chooseNewSyncSource(
467 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
468 ASSERT_EQUALS(HostAndPort("h4"), getTopoCoord().getSyncSourceAddress());
469
470 // h4 goes down; should choose h1
471 receiveDownHeartbeat(HostAndPort("h4"), "rs0", OpTime());
472 getTopoCoord().chooseNewSyncSource(
473 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
474 ASSERT_EQUALS(HostAndPort("h1"), getTopoCoord().getSyncSourceAddress());
475
476 // Primary and h1 go down; should choose h6
477 receiveDownHeartbeat(HostAndPort("h1"), "rs0", OpTime());
478 receiveDownHeartbeat(HostAndPort("hprimary"), "rs0", OpTime());
479 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
480 getTopoCoord().chooseNewSyncSource(
481 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
482 ASSERT_EQUALS(HostAndPort("h6"), getTopoCoord().getSyncSourceAddress());
483
484 // h6 goes down; should choose h5
485 receiveDownHeartbeat(HostAndPort("h6"), "rs0", OpTime());
486 getTopoCoord().chooseNewSyncSource(
487 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
488 ASSERT_EQUALS(HostAndPort("h5"), getTopoCoord().getSyncSourceAddress());
489
490 // h5 goes down; should choose h3
491 receiveDownHeartbeat(HostAndPort("h5"), "rs0", OpTime());
492 getTopoCoord().chooseNewSyncSource(
493 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
494 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
495
496 // h3 goes down; no sync source candidates remain
497 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime());
498 getTopoCoord().chooseNewSyncSource(
499 now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration);
500 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
501 }
502
503
TEST_F(TopoCoordTest,ChooseOnlyPrimaryAsSyncSourceWhenChainingIsDisallowed)504 TEST_F(TopoCoordTest, ChooseOnlyPrimaryAsSyncSourceWhenChainingIsDisallowed) {
505 updateConfig(BSON("_id"
506 << "rs0"
507 << "version"
508 << 1
509 << "settings"
510 << BSON("chainingAllowed" << false)
511 << "members"
512 << BSON_ARRAY(BSON("_id" << 10 << "host"
513 << "hself")
514 << BSON("_id" << 20 << "host"
515 << "h2")
516 << BSON("_id" << 30 << "host"
517 << "h3"))),
518 0);
519
520 setSelfMemberState(MemberState::RS_SECONDARY);
521
522 heartbeatFromMember(HostAndPort("h2"),
523 "rs0",
524 MemberState::RS_SECONDARY,
525 OpTime(Timestamp(11, 0), 0),
526 Milliseconds(100));
527 heartbeatFromMember(HostAndPort("h2"),
528 "rs0",
529 MemberState::RS_SECONDARY,
530 OpTime(Timestamp(11, 0), 0),
531 Milliseconds(100));
532 heartbeatFromMember(HostAndPort("h3"),
533 "rs0",
534 MemberState::RS_SECONDARY,
535 OpTime(Timestamp(0, 0), 0),
536 Milliseconds(300));
537 heartbeatFromMember(HostAndPort("h3"),
538 "rs0",
539 MemberState::RS_SECONDARY,
540 OpTime(Timestamp(0, 0), 0),
541 Milliseconds(300));
542
543 // No primary situation: should choose no sync source.
544 ASSERT_EQUALS(
545 HostAndPort(),
546 getTopoCoord().chooseNewSyncSource(
547 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration));
548 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
549
550 // Add primary
551 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
552 heartbeatFromMember(HostAndPort("h3"),
553 "rs0",
554 MemberState::RS_PRIMARY,
555 OpTime(Timestamp(0, 0), 0),
556 Milliseconds(300));
557 ASSERT_EQUALS(2, getCurrentPrimaryIndex());
558
559 // h3 is primary and should be chosen as the sync source when we are not in catch-up mode,
560 // despite being further away than h2 and the primary (h3) being behind our most recently
561 // applied optime.
562 ASSERT_EQUALS(HostAndPort("h3"),
563 getTopoCoord().chooseNewSyncSource(
564 now()++,
565 OpTime(Timestamp(10, 0), 0),
566 TopologyCoordinator::ChainingPreference::kUseConfiguration));
567 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
568
569 // When we are in catch-up mode, the chainingAllowed setting is ignored. h2 should be chosen as
570 // the sync source.
571 ASSERT_EQUALS(HostAndPort("h2"),
572 getTopoCoord().chooseNewSyncSource(
573 now()++,
574 OpTime(Timestamp(10, 0), 0),
575 TopologyCoordinator::ChainingPreference::kAllowChaining));
576 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
577
578 // Become primary: should not choose self as sync source.
579 heartbeatFromMember(HostAndPort("h3"),
580 "rs0",
581 MemberState::RS_SECONDARY,
582 OpTime(Timestamp(0, 0), 0),
583 Milliseconds(300));
584 makeSelfPrimary(Timestamp(3.0));
585 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
586 ASSERT_EQUALS(
587 HostAndPort(),
588 getTopoCoord().chooseNewSyncSource(
589 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration));
590 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
591 }
592
TEST_F(TopoCoordTest,ChooseOnlyVotersAsSyncSourceWhenNodeIsAVoter)593 TEST_F(TopoCoordTest, ChooseOnlyVotersAsSyncSourceWhenNodeIsAVoter) {
594 updateConfig(fromjson("{_id:'rs0', version:1, members:["
595 "{_id:10, host:'hself'}, "
596 "{_id:20, host:'h2', votes:0, priority:0}, "
597 "{_id:30, host:'h3'} "
598 "]}"),
599 0);
600
601 setSelfMemberState(MemberState::RS_SECONDARY);
602
603 HostAndPort h2("h2"), h3("h3");
604 Timestamp t1(1, 0), t5(5, 0), t10(10, 0);
605 OpTime ot1(t1, 0), ot5(t5, 0), ot10(t10, 0);
606 Milliseconds hbRTT100(100), hbRTT300(300);
607
608 // Two rounds of heartbeat pings from each member.
609 heartbeatFromMember(h2, "rs0", MemberState::RS_SECONDARY, ot5, hbRTT100);
610 heartbeatFromMember(h2, "rs0", MemberState::RS_SECONDARY, ot5, hbRTT100);
611 heartbeatFromMember(h3, "rs0", MemberState::RS_SECONDARY, ot1, hbRTT300);
612 heartbeatFromMember(h3, "rs0", MemberState::RS_SECONDARY, ot1, hbRTT300);
613
614 // Should choose h3 as it is a voter
615 auto newSource = getTopoCoord().chooseNewSyncSource(
616 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
617 ASSERT_EQUALS(h3, newSource);
618
619 // Can't choose h2 as it is not a voter
620 newSource = getTopoCoord().chooseNewSyncSource(
621 now()++, ot10, TopologyCoordinator::ChainingPreference::kUseConfiguration);
622 ASSERT_EQUALS(HostAndPort(), newSource);
623
624 // Should choose h3 as it is a voter, and ahead
625 heartbeatFromMember(h3, "rs0", MemberState::RS_SECONDARY, ot5, hbRTT300);
626 newSource = getTopoCoord().chooseNewSyncSource(
627 now()++, ot1, TopologyCoordinator::ChainingPreference::kUseConfiguration);
628 ASSERT_EQUALS(h3, newSource);
629 }
630
TEST_F(TopoCoordTest,ChooseSameSyncSourceEvenWhenPrimary)631 TEST_F(TopoCoordTest, ChooseSameSyncSourceEvenWhenPrimary) {
632 updateConfig(BSON("_id"
633 << "rs0"
634 << "version"
635 << 1
636 << "members"
637 << BSON_ARRAY(BSON("_id" << 10 << "host"
638 << "hself")
639 << BSON("_id" << 20 << "host"
640 << "h2")
641 << BSON("_id" << 30 << "host"
642 << "h3"))),
643 0);
644
645 setSelfMemberState(MemberState::RS_SECONDARY);
646
647 // Two rounds of heartbeat pings from each member.
648 heartbeatFromMember(HostAndPort("h2"),
649 "rs0",
650 MemberState::RS_SECONDARY,
651 OpTime(Timestamp(1, 0), 0),
652 Milliseconds(100));
653 heartbeatFromMember(HostAndPort("h2"),
654 "rs0",
655 MemberState::RS_SECONDARY,
656 OpTime(Timestamp(1, 0), 0),
657 Milliseconds(100));
658 heartbeatFromMember(HostAndPort("h3"),
659 "rs0",
660 MemberState::RS_SECONDARY,
661 OpTime(Timestamp(0, 0), 0),
662 Milliseconds(300));
663 heartbeatFromMember(HostAndPort("h3"),
664 "rs0",
665 MemberState::RS_SECONDARY,
666 OpTime(Timestamp(0, 0), 0),
667 Milliseconds(300));
668
669 // No primary situation: should choose h2 sync source.
670 ASSERT_EQUALS(
671 HostAndPort("h2"),
672 getTopoCoord().chooseNewSyncSource(
673 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration));
674 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
675
676 // Become primary
677 makeSelfPrimary(Timestamp(3.0));
678 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
679
680 // Choose same sync source even when primary.
681 ASSERT_EQUALS(
682 HostAndPort("h2"),
683 getTopoCoord().chooseNewSyncSource(
684 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration));
685 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
686 }
687
TEST_F(TopoCoordTest,ChooseRequestedSyncSourceOnlyTheFirstTimeAfterTheSyncSourceIsForciblySet)688 TEST_F(TopoCoordTest, ChooseRequestedSyncSourceOnlyTheFirstTimeAfterTheSyncSourceIsForciblySet) {
689 updateConfig(BSON("_id"
690 << "rs0"
691 << "version"
692 << 1
693 << "members"
694 << BSON_ARRAY(BSON("_id" << 10 << "host"
695 << "hself")
696 << BSON("_id" << 20 << "host"
697 << "h2")
698 << BSON("_id" << 30 << "host"
699 << "h3"))),
700 0);
701
702 setSelfMemberState(MemberState::RS_SECONDARY);
703
704 // two rounds of heartbeat pings from each member
705 heartbeatFromMember(HostAndPort("h2"),
706 "rs0",
707 MemberState::RS_SECONDARY,
708 OpTime(Timestamp(1, 0), 0),
709 Milliseconds(300));
710 heartbeatFromMember(HostAndPort("h2"),
711 "rs0",
712 MemberState::RS_SECONDARY,
713 OpTime(Timestamp(1, 0), 0),
714 Milliseconds(300));
715 heartbeatFromMember(HostAndPort("h3"),
716 "rs0",
717 MemberState::RS_SECONDARY,
718 OpTime(Timestamp(2, 0), 0),
719 Milliseconds(100));
720 heartbeatFromMember(HostAndPort("h3"),
721 "rs0",
722 MemberState::RS_SECONDARY,
723 OpTime(Timestamp(2, 0), 0),
724 Milliseconds(100));
725
726 // force should overrule other defaults
727 getTopoCoord().chooseNewSyncSource(
728 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
729 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
730 getTopoCoord().setForceSyncSourceIndex(1);
731 // force should cause shouldChangeSyncSource() to return true
732 // even if the currentSource is the force target
733 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
734 HostAndPort("h2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
735 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
736 HostAndPort("h3"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
737 getTopoCoord().chooseNewSyncSource(
738 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
739 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
740
741 // force should only work for one call to chooseNewSyncSource
742 getTopoCoord().chooseNewSyncSource(
743 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
744 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
745 }
746
TEST_F(TopoCoordTest,NodeDoesNotChooseBlacklistedSyncSourceUntilBlacklistingExpires)747 TEST_F(TopoCoordTest, NodeDoesNotChooseBlacklistedSyncSourceUntilBlacklistingExpires) {
748 updateConfig(BSON("_id"
749 << "rs0"
750 << "version"
751 << 1
752 << "members"
753 << BSON_ARRAY(BSON("_id" << 10 << "host"
754 << "hself")
755 << BSON("_id" << 20 << "host"
756 << "h2")
757 << BSON("_id" << 30 << "host"
758 << "h3"))),
759 0);
760
761 setSelfMemberState(MemberState::RS_SECONDARY);
762
763 // Two rounds of heartbeat pings from each member.
764 heartbeatFromMember(HostAndPort("h2"),
765 "rs0",
766 MemberState::RS_SECONDARY,
767 OpTime(Timestamp(1, 0), 0),
768 Milliseconds(300));
769 heartbeatFromMember(HostAndPort("h2"),
770 "rs0",
771 MemberState::RS_SECONDARY,
772 OpTime(Timestamp(1, 0), 0),
773 Milliseconds(300));
774 heartbeatFromMember(HostAndPort("h3"),
775 "rs0",
776 MemberState::RS_SECONDARY,
777 OpTime(Timestamp(2, 0), 0),
778 Milliseconds(100));
779 heartbeatFromMember(HostAndPort("h3"),
780 "rs0",
781 MemberState::RS_SECONDARY,
782 OpTime(Timestamp(2, 0), 0),
783 Milliseconds(100));
784
785 getTopoCoord().chooseNewSyncSource(
786 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
787 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
788
789 Date_t expireTime = Date_t::fromMillisSinceEpoch(1000);
790 getTopoCoord().blacklistSyncSource(HostAndPort("h3"), expireTime);
791 getTopoCoord().chooseNewSyncSource(
792 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
793 // Should choose second best choice now that h3 is blacklisted.
794 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
795
796 // After time has passed, should go back to original sync source
797 getTopoCoord().chooseNewSyncSource(
798 expireTime, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
799 ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress());
800 }
801
TEST_F(TopoCoordTest,ChooseNoSyncSourceWhenPrimaryIsBlacklistedAndChainingIsDisallowed)802 TEST_F(TopoCoordTest, ChooseNoSyncSourceWhenPrimaryIsBlacklistedAndChainingIsDisallowed) {
803 updateConfig(BSON("_id"
804 << "rs0"
805 << "version"
806 << 1
807 << "settings"
808 << BSON("chainingAllowed" << false)
809 << "members"
810 << BSON_ARRAY(BSON("_id" << 10 << "host"
811 << "hself")
812 << BSON("_id" << 20 << "host"
813 << "h2")
814 << BSON("_id" << 30 << "host"
815 << "h3"))),
816 0);
817
818 setSelfMemberState(MemberState::RS_SECONDARY);
819
820 heartbeatFromMember(HostAndPort("h2"),
821 "rs0",
822 MemberState::RS_PRIMARY,
823 OpTime(Timestamp(2, 0), 0),
824 Milliseconds(100));
825 heartbeatFromMember(HostAndPort("h2"),
826 "rs0",
827 MemberState::RS_PRIMARY,
828 OpTime(Timestamp(2, 0), 0),
829 Milliseconds(100));
830 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
831
832 heartbeatFromMember(HostAndPort("h3"),
833 "rs0",
834 MemberState::RS_SECONDARY,
835 OpTime(Timestamp(2, 0), 0),
836 Milliseconds(100));
837 heartbeatFromMember(HostAndPort("h3"),
838 "rs0",
839 MemberState::RS_SECONDARY,
840 OpTime(Timestamp(2, 0), 0),
841 Milliseconds(100));
842
843 getTopoCoord().chooseNewSyncSource(
844 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
845 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
846
847 Date_t expireTime = Date_t::fromMillisSinceEpoch(1000);
848 getTopoCoord().blacklistSyncSource(HostAndPort("h2"), expireTime);
849 getTopoCoord().chooseNewSyncSource(
850 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
851 // Can't choose any sync source now.
852 ASSERT(getTopoCoord().getSyncSourceAddress().empty());
853
854 // After time has passed, should go back to the primary
855 getTopoCoord().chooseNewSyncSource(
856 expireTime, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
857 ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress());
858 }
859
TEST_F(TopoCoordTest,NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp)860 TEST_F(TopoCoordTest, NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp) {
861 updateConfig(BSON("_id"
862 << "rs0"
863 << "version"
864 << 1
865 << "members"
866 << BSON_ARRAY(BSON("_id" << 10 << "host"
867 << "hself")
868 << BSON("_id" << 20 << "host"
869 << "h2")
870 << BSON("_id" << 30 << "host"
871 << "h3"))),
872 0);
873
874 setSelfMemberState(MemberState::RS_SECONDARY);
875
876 // Generate enough heartbeats to select a sync source below
877 heartbeatFromMember(HostAndPort("h2"),
878 "rs0",
879 MemberState::RS_SECONDARY,
880 OpTime(Timestamp(1, 0), 0),
881 Milliseconds(300));
882 heartbeatFromMember(HostAndPort("h2"),
883 "rs0",
884 MemberState::RS_SECONDARY,
885 OpTime(Timestamp(1, 0), 0),
886 Milliseconds(300));
887 heartbeatFromMember(HostAndPort("h3"),
888 "rs0",
889 MemberState::RS_SECONDARY,
890 OpTime(Timestamp(2, 0), 0),
891 Milliseconds(100));
892 heartbeatFromMember(HostAndPort("h3"),
893 "rs0",
894 MemberState::RS_SECONDARY,
895 OpTime(Timestamp(2, 0), 0),
896 Milliseconds(100));
897
898 ASSERT_EQUALS(
899 HostAndPort("h3"),
900 getTopoCoord().chooseNewSyncSource(
901 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration));
902 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
903 // Good state setup done
904
905 // Mark nodes down, ensure that we have no source and are secondary
906 receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime(), ErrorCodes::NetworkTimeout);
907 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime(), ErrorCodes::NetworkTimeout);
908 ASSERT_TRUE(getTopoCoord()
909 .chooseNewSyncSource(now()++,
910 OpTime(),
911 TopologyCoordinator::ChainingPreference::kUseConfiguration)
912 .empty());
913 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
914
915 // Mark nodes unauth, ensure that we have no source and are secondary
916 receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime(), ErrorCodes::Unauthorized);
917 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime(), ErrorCodes::Unauthorized);
918 ASSERT_TRUE(getTopoCoord()
919 .chooseNewSyncSource(now()++,
920 OpTime(),
921 TopologyCoordinator::ChainingPreference::kUseConfiguration)
922 .empty());
923 ASSERT_EQUALS(MemberState::RS_RECOVERING, getTopoCoord().getMemberState().s);
924
925 // Having an auth error but with another node up should bring us out of RECOVERING
926 HeartbeatResponseAction action = receiveUpHeartbeat(
927 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(), OpTime(Timestamp(2, 0), 0));
928 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
929 // Test that the heartbeat that brings us from RECOVERING to SECONDARY doesn't initiate
930 // an election (SERVER-17164)
931 ASSERT_NO_ACTION(action.getAction());
932 }
933
TEST_F(TopoCoordTest,NodeDoesNotActOnHeartbeatsWhenAbsentFromConfig)934 TEST_F(TopoCoordTest, NodeDoesNotActOnHeartbeatsWhenAbsentFromConfig) {
935 updateConfig(BSON("_id"
936 << "rs0"
937 << "version"
938 << 1
939 << "members"
940 << BSON_ARRAY(BSON("_id" << 10 << "host"
941 << "h1")
942 << BSON("_id" << 20 << "host"
943 << "h2")
944 << BSON("_id" << 30 << "host"
945 << "h3"))),
946 -1);
947 ASSERT_NO_ACTION(heartbeatFromMember(HostAndPort("h2"),
948 "rs0",
949 MemberState::RS_SECONDARY,
950 OpTime(Timestamp(1, 0), 0),
951 Milliseconds(300))
952 .getAction());
953 }
954
TEST_F(TopoCoordTest,NodeReturnsNotSecondaryWhenSyncFromIsRunPriorToHavingAConfig)955 TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunPriorToHavingAConfig) {
956 Status result = Status::OK();
957 BSONObjBuilder response;
958
959 // if we do not have an index in the config, we should get ErrorCodes::NotSecondary
960 getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result);
961 ASSERT_EQUALS(ErrorCodes::NotSecondary, result);
962 ASSERT_EQUALS("Removed and uninitialized nodes do not sync", result.reason());
963 }
964
TEST_F(TopoCoordTest,NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter)965 TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter) {
966 Status result = Status::OK();
967 BSONObjBuilder response;
968
969
970 // Test trying to sync from another node when we are an arbiter
971 updateConfig(BSON("_id"
972 << "rs0"
973 << "version"
974 << 1
975 << "members"
976 << BSON_ARRAY(BSON("_id" << 0 << "host"
977 << "hself"
978 << "arbiterOnly"
979 << true)
980 << BSON("_id" << 1 << "host"
981 << "h1"))),
982 0);
983
984 getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result);
985 ASSERT_EQUALS(ErrorCodes::NotSecondary, result);
986 ASSERT_EQUALS("arbiters don't sync", result.reason());
987 }
988
TEST_F(TopoCoordTest,NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary)989 TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary) {
990 Status result = Status::OK();
991 BSONObjBuilder response;
992
993 updateConfig(BSON("_id"
994 << "rs0"
995 << "version"
996 << 1
997 << "members"
998 << BSON_ARRAY(BSON("_id" << 0 << "host"
999 << "hself")
1000 << BSON("_id" << 1 << "host"
1001 << "h1"
1002 << "arbiterOnly"
1003 << true)
1004 << BSON("_id" << 2 << "host"
1005 << "h2"
1006 << "priority"
1007 << 0
1008 << "buildIndexes"
1009 << false)
1010 << BSON("_id" << 3 << "host"
1011 << "h3")
1012 << BSON("_id" << 4 << "host"
1013 << "h4")
1014 << BSON("_id" << 5 << "host"
1015 << "h5")
1016 << BSON("_id" << 6 << "host"
1017 << "h6"))),
1018 0);
1019
1020 // Try to sync while PRIMARY
1021 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
1022 makeSelfPrimary();
1023 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
1024 getTopoCoord()._setCurrentPrimaryForTest(0);
1025 getTopoCoord().prepareSyncFromResponse(HostAndPort("h3"), &response, &result);
1026 ASSERT_EQUALS(ErrorCodes::NotSecondary, result);
1027 ASSERT_EQUALS("primaries don't sync", result.reason());
1028 ASSERT_EQUALS("h3:27017", response.obj()["syncFromRequested"].String());
1029 }
1030
TEST_F(TopoCoordTest,NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfig)1031 TEST_F(TopoCoordTest, NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfig) {
1032 Status result = Status::OK();
1033 BSONObjBuilder response;
1034
1035 updateConfig(BSON("_id"
1036 << "rs0"
1037 << "version"
1038 << 1
1039 << "members"
1040 << BSON_ARRAY(BSON("_id" << 0 << "host"
1041 << "hself")
1042 << BSON("_id" << 1 << "host"
1043 << "h1"
1044 << "arbiterOnly"
1045 << true)
1046 << BSON("_id" << 2 << "host"
1047 << "h2"
1048 << "priority"
1049 << 0
1050 << "buildIndexes"
1051 << false)
1052 << BSON("_id" << 3 << "host"
1053 << "h3")
1054 << BSON("_id" << 4 << "host"
1055 << "h4")
1056 << BSON("_id" << 5 << "host"
1057 << "h5")
1058 << BSON("_id" << 6 << "host"
1059 << "h6"))),
1060 0);
1061 setSelfMemberState(MemberState::RS_SECONDARY);
1062
1063 getTopoCoord().prepareSyncFromResponse(HostAndPort("fakemember"), &response, &result);
1064 ASSERT_EQUALS(ErrorCodes::NodeNotFound, result);
1065 ASSERT_EQUALS("Could not find member \"fakemember:27017\" in replica set", result.reason());
1066 }
1067
TEST_F(TopoCoordTest,NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf)1068 TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf) {
1069 Status result = Status::OK();
1070 BSONObjBuilder response;
1071
1072 updateConfig(BSON("_id"
1073 << "rs0"
1074 << "version"
1075 << 1
1076 << "members"
1077 << BSON_ARRAY(BSON("_id" << 0 << "host"
1078 << "hself")
1079 << BSON("_id" << 1 << "host"
1080 << "h1"
1081 << "arbiterOnly"
1082 << true)
1083 << BSON("_id" << 2 << "host"
1084 << "h2"
1085 << "priority"
1086 << 0
1087 << "buildIndexes"
1088 << false)
1089 << BSON("_id" << 3 << "host"
1090 << "h3")
1091 << BSON("_id" << 4 << "host"
1092 << "h4")
1093 << BSON("_id" << 5 << "host"
1094 << "h5")
1095 << BSON("_id" << 6 << "host"
1096 << "h6"))),
1097 0);
1098 setSelfMemberState(MemberState::RS_SECONDARY);
1099
1100 // Try to sync from self
1101 getTopoCoord().prepareSyncFromResponse(HostAndPort("hself"), &response, &result);
1102 ASSERT_EQUALS(ErrorCodes::InvalidOptions, result);
1103 ASSERT_EQUALS("I cannot sync from myself", result.reason());
1104 }
1105
TEST_F(TopoCoordTest,NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter)1106 TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter) {
1107 Status result = Status::OK();
1108 BSONObjBuilder response;
1109
1110 updateConfig(BSON("_id"
1111 << "rs0"
1112 << "version"
1113 << 1
1114 << "members"
1115 << BSON_ARRAY(BSON("_id" << 0 << "host"
1116 << "hself")
1117 << BSON("_id" << 1 << "host"
1118 << "h1"
1119 << "arbiterOnly"
1120 << true)
1121 << BSON("_id" << 2 << "host"
1122 << "h2"
1123 << "priority"
1124 << 0
1125 << "buildIndexes"
1126 << false)
1127 << BSON("_id" << 3 << "host"
1128 << "h3")
1129 << BSON("_id" << 4 << "host"
1130 << "h4")
1131 << BSON("_id" << 5 << "host"
1132 << "h5")
1133 << BSON("_id" << 6 << "host"
1134 << "h6"))),
1135 0);
1136 setSelfMemberState(MemberState::RS_SECONDARY);
1137
1138
1139 // Try to sync from an arbiter
1140 getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result);
1141 ASSERT_EQUALS(ErrorCodes::InvalidOptions, result);
1142 ASSERT_EQUALS("Cannot sync from \"h1:27017\" because it is an arbiter", result.reason());
1143 }
1144
TEST_F(TopoCoordTest,NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbuilder)1145 TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbuilder) {
1146 Status result = Status::OK();
1147 BSONObjBuilder response;
1148
1149 updateConfig(BSON("_id"
1150 << "rs0"
1151 << "version"
1152 << 1
1153 << "members"
1154 << BSON_ARRAY(BSON("_id" << 0 << "host"
1155 << "hself")
1156 << BSON("_id" << 1 << "host"
1157 << "h1"
1158 << "arbiterOnly"
1159 << true)
1160 << BSON("_id" << 2 << "host"
1161 << "h2"
1162 << "priority"
1163 << 0
1164 << "buildIndexes"
1165 << false)
1166 << BSON("_id" << 3 << "host"
1167 << "h3")
1168 << BSON("_id" << 4 << "host"
1169 << "h4")
1170 << BSON("_id" << 5 << "host"
1171 << "h5")
1172 << BSON("_id" << 6 << "host"
1173 << "h6"))),
1174 0);
1175 setSelfMemberState(MemberState::RS_SECONDARY);
1176
1177 // Try to sync from a node that doesn't build indexes
1178 getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result);
1179 ASSERT_EQUALS(ErrorCodes::InvalidOptions, result);
1180 ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it does not build indexes",
1181 result.reason());
1182 }
1183
TEST_F(TopoCoordTest,NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode)1184 TEST_F(TopoCoordTest, NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode) {
1185 Status result = Status::OK();
1186 BSONObjBuilder response;
1187
1188 updateConfig(BSON("_id"
1189 << "rs0"
1190 << "version"
1191 << 1
1192 << "members"
1193 << BSON_ARRAY(BSON("_id" << 0 << "host"
1194 << "hself")
1195 << BSON("_id" << 1 << "host"
1196 << "h1"
1197 << "arbiterOnly"
1198 << true)
1199 << BSON("_id" << 2 << "host"
1200 << "h2"
1201 << "priority"
1202 << 0
1203 << "buildIndexes"
1204 << false)
1205 << BSON("_id" << 3 << "host"
1206 << "h3")
1207 << BSON("_id" << 4 << "host"
1208 << "h4")
1209 << BSON("_id" << 5 << "host"
1210 << "h5")
1211 << BSON("_id" << 6 << "host"
1212 << "h6"))),
1213 0);
1214 setSelfMemberState(MemberState::RS_SECONDARY);
1215
1216 // Try to sync from a member that is down
1217 receiveDownHeartbeat(HostAndPort("h4"), "rs0", OpTime());
1218
1219 getTopoCoord().prepareSyncFromResponse(HostAndPort("h4"), &response, &result);
1220 ASSERT_EQUALS(ErrorCodes::HostUnreachable, result);
1221 ASSERT_EQUALS("I cannot reach the requested member: h4:27017", result.reason());
1222 }
1223
TEST_F(TopoCoordTest,ChooseRequestedNodeWhenSyncFromRequestsAStaleNode)1224 TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAStaleNode) {
1225 OpTime staleOpTime(Timestamp(1, 1), 0);
1226 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1227
1228 Status result = Status::OK();
1229 BSONObjBuilder response;
1230
1231 updateConfig(BSON("_id"
1232 << "rs0"
1233 << "version"
1234 << 1
1235 << "members"
1236 << BSON_ARRAY(BSON("_id" << 0 << "host"
1237 << "hself")
1238 << BSON("_id" << 1 << "host"
1239 << "h1"
1240 << "arbiterOnly"
1241 << true)
1242 << BSON("_id" << 2 << "host"
1243 << "h2"
1244 << "priority"
1245 << 0
1246 << "buildIndexes"
1247 << false)
1248 << BSON("_id" << 3 << "host"
1249 << "h3")
1250 << BSON("_id" << 4 << "host"
1251 << "h4")
1252 << BSON("_id" << 5 << "host"
1253 << "h5")
1254 << BSON("_id" << 6 << "host"
1255 << "h6"))),
1256 0);
1257 setSelfMemberState(MemberState::RS_SECONDARY);
1258 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1259
1260 // Sync successfully from a member that is stale
1261 heartbeatFromMember(
1262 HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, staleOpTime, Milliseconds(100));
1263
1264 getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result);
1265 ASSERT_OK(result);
1266 ASSERT_EQUALS("requested member \"h5:27017\" is more than 10 seconds behind us",
1267 response.obj()["warning"].String());
1268 getTopoCoord().chooseNewSyncSource(
1269 now()++, ourOpTime, TopologyCoordinator::ChainingPreference::kUseConfiguration);
1270 ASSERT_EQUALS(HostAndPort("h5"), getTopoCoord().getSyncSourceAddress());
1271 }
1272
TEST_F(TopoCoordTest,ChooseRequestedNodeWhenSyncFromRequestsAValidNode)1273 TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAValidNode) {
1274 OpTime staleOpTime(Timestamp(1, 1), 0);
1275 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1276
1277 Status result = Status::OK();
1278 BSONObjBuilder response;
1279
1280 updateConfig(BSON("_id"
1281 << "rs0"
1282 << "version"
1283 << 1
1284 << "members"
1285 << BSON_ARRAY(BSON("_id" << 0 << "host"
1286 << "hself")
1287 << BSON("_id" << 1 << "host"
1288 << "h1"
1289 << "arbiterOnly"
1290 << true)
1291 << BSON("_id" << 2 << "host"
1292 << "h2"
1293 << "priority"
1294 << 0
1295 << "buildIndexes"
1296 << false)
1297 << BSON("_id" << 3 << "host"
1298 << "h3")
1299 << BSON("_id" << 4 << "host"
1300 << "h4")
1301 << BSON("_id" << 5 << "host"
1302 << "h5")
1303 << BSON("_id" << 6 << "host"
1304 << "h6"))),
1305 0);
1306 setSelfMemberState(MemberState::RS_SECONDARY);
1307
1308 // Sync successfully from an up-to-date member
1309 heartbeatFromMember(
1310 HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100));
1311
1312 getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result);
1313 ASSERT_OK(result);
1314 BSONObj responseObj = response.obj();
1315 ASSERT_FALSE(responseObj.hasField("warning"));
1316 getTopoCoord().chooseNewSyncSource(
1317 now()++, ourOpTime, TopologyCoordinator::ChainingPreference::kUseConfiguration);
1318 ASSERT_EQUALS(HostAndPort("h6"), getTopoCoord().getSyncSourceAddress());
1319 }
1320
TEST_F(TopoCoordTest,NodeReturnsRequestedNodeWhenSyncFromRequestsAValidNodeEvenIfTheNodeHasSinceBeenMarkedDown)1321 TEST_F(TopoCoordTest,
1322 NodeReturnsRequestedNodeWhenSyncFromRequestsAValidNodeEvenIfTheNodeHasSinceBeenMarkedDown) {
1323 OpTime staleOpTime(Timestamp(1, 1), 0);
1324 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1325
1326 Status result = Status::OK();
1327 BSONObjBuilder response;
1328
1329 updateConfig(BSON("_id"
1330 << "rs0"
1331 << "version"
1332 << 1
1333 << "members"
1334 << BSON_ARRAY(BSON("_id" << 0 << "host"
1335 << "hself")
1336 << BSON("_id" << 1 << "host"
1337 << "h1"
1338 << "arbiterOnly"
1339 << true)
1340 << BSON("_id" << 2 << "host"
1341 << "h2"
1342 << "priority"
1343 << 0
1344 << "buildIndexes"
1345 << false)
1346 << BSON("_id" << 3 << "host"
1347 << "h3")
1348 << BSON("_id" << 4 << "host"
1349 << "h4")
1350 << BSON("_id" << 5 << "host"
1351 << "h5")
1352 << BSON("_id" << 6 << "host"
1353 << "h6"))),
1354 0);
1355 setSelfMemberState(MemberState::RS_SECONDARY);
1356
1357 heartbeatFromMember(
1358 HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100));
1359
1360 // node goes down between forceSync and chooseNewSyncSource
1361 getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result);
1362 BSONObj responseObj = response.obj();
1363 ASSERT_FALSE(responseObj.hasField("warning"));
1364 receiveDownHeartbeat(HostAndPort("h6"), "rs0", OpTime());
1365 HostAndPort syncSource = getTopoCoord().chooseNewSyncSource(
1366 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
1367 ASSERT_EQUALS(HostAndPort("h6"), syncSource);
1368 }
1369
TEST_F(TopoCoordTest,NodeReturnsUnauthorizedWhenSyncFromRequestsANodeWeAreNotAuthorizedFor)1370 TEST_F(TopoCoordTest, NodeReturnsUnauthorizedWhenSyncFromRequestsANodeWeAreNotAuthorizedFor) {
1371 OpTime staleOpTime(Timestamp(1, 1), 0);
1372 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1373
1374 Status result = Status::OK();
1375 BSONObjBuilder response;
1376
1377 updateConfig(BSON("_id"
1378 << "rs0"
1379 << "version"
1380 << 1
1381 << "members"
1382 << BSON_ARRAY(BSON("_id" << 0 << "host"
1383 << "hself")
1384 << BSON("_id" << 1 << "host"
1385 << "h1"
1386 << "arbiterOnly"
1387 << true)
1388 << BSON("_id" << 2 << "host"
1389 << "h2"
1390 << "priority"
1391 << 0
1392 << "buildIndexes"
1393 << false)
1394 << BSON("_id" << 3 << "host"
1395 << "h3")
1396 << BSON("_id" << 4 << "host"
1397 << "h4")
1398 << BSON("_id" << 5 << "host"
1399 << "h5")
1400 << BSON("_id" << 6 << "host"
1401 << "h6"))),
1402 0);
1403 setSelfMemberState(MemberState::RS_SECONDARY);
1404
1405 // Try to sync from a member that is unauth'd
1406 receiveDownHeartbeat(HostAndPort("h5"), "rs0", OpTime(), ErrorCodes::Unauthorized);
1407
1408 getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result);
1409 ASSERT_NOT_OK(result);
1410 ASSERT_EQUALS(ErrorCodes::Unauthorized, result.code());
1411 ASSERT_EQUALS("not authorized to communicate with h5:27017", result.reason());
1412 }
1413
TEST_F(TopoCoordTest,NodeReturnsInvalidOptionsWhenAskedToSyncFromANonVoterAsAVoter)1414 TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenAskedToSyncFromANonVoterAsAVoter) {
1415 OpTime staleOpTime(Timestamp(1, 1), 0);
1416 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1417
1418 Status result = Status::OK();
1419 BSONObjBuilder response;
1420
1421 // Test trying to sync from another node
1422 updateConfig(fromjson("{_id:'rs0', version:1, members:["
1423 "{_id:0, host:'self'},"
1424 "{_id:1, host:'h1'},"
1425 "{_id:2, host:'h2', votes:0, priority:0}"
1426 "]}"),
1427 0);
1428
1429 getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result);
1430 ASSERT_EQUALS(ErrorCodes::InvalidOptions, result);
1431 ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it is not a voter", result.reason());
1432 }
1433
TEST_F(TopoCoordTest,NodeShouldReturnPrevSyncTargetWhenItHasASyncTargetAndSyncFromMakesAValidRequest)1434 TEST_F(TopoCoordTest,
1435 NodeShouldReturnPrevSyncTargetWhenItHasASyncTargetAndSyncFromMakesAValidRequest) {
1436 OpTime staleOpTime(Timestamp(1, 1), 0);
1437 OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0);
1438
1439 Status result = Status::OK();
1440 BSONObjBuilder response;
1441 BSONObjBuilder response2;
1442
1443 updateConfig(BSON("_id"
1444 << "rs0"
1445 << "version"
1446 << 1
1447 << "members"
1448 << BSON_ARRAY(BSON("_id" << 0 << "host"
1449 << "hself")
1450 << BSON("_id" << 1 << "host"
1451 << "h1"
1452 << "arbiterOnly"
1453 << true)
1454 << BSON("_id" << 2 << "host"
1455 << "h2"
1456 << "priority"
1457 << 0
1458 << "buildIndexes"
1459 << false)
1460 << BSON("_id" << 3 << "host"
1461 << "h3")
1462 << BSON("_id" << 4 << "host"
1463 << "h4")
1464 << BSON("_id" << 5 << "host"
1465 << "h5")
1466 << BSON("_id" << 6 << "host"
1467 << "h6"))),
1468 0);
1469 setSelfMemberState(MemberState::RS_SECONDARY);
1470
1471 // Sync successfully from an up-to-date member.
1472 heartbeatFromMember(
1473 HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100));
1474
1475 getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result);
1476 ASSERT_OK(result);
1477 BSONObj responseObj = response.obj();
1478 ASSERT_FALSE(responseObj.hasField("warning"));
1479 ASSERT_FALSE(responseObj.hasField("prevSyncTarget"));
1480 getTopoCoord().chooseNewSyncSource(
1481 now()++, ourOpTime, TopologyCoordinator::ChainingPreference::kUseConfiguration);
1482 ASSERT_EQUALS(HostAndPort("h5"), getTopoCoord().getSyncSourceAddress());
1483
1484 heartbeatFromMember(
1485 HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100));
1486
1487 // Sync successfully from another up-to-date member.
1488 getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response2, &result);
1489 BSONObj response2Obj = response2.obj();
1490 ASSERT_FALSE(response2Obj.hasField("warning"));
1491 ASSERT_EQUALS(HostAndPort("h5").toString(), response2Obj["prevSyncTarget"].String());
1492 }
1493
1494 // TODO(dannenberg) figure out a concise name for this
TEST_F(TopoCoordTest,ReplSetGetStatus)1495 TEST_F(TopoCoordTest, ReplSetGetStatus) {
1496 // This test starts by configuring a TopologyCoordinator as a member of a 4 node replica
1497 // set, with each node in a different state.
1498 // The first node is DOWN, as if we tried heartbeating them and it failed in some way.
1499 // The second node is in state SECONDARY, as if we've received a valid heartbeat from them.
1500 // The third node is in state UNKNOWN, as if we've not yet had any heartbeating activity
1501 // with them yet. The fourth node is PRIMARY and corresponds to ourself, which gets its
1502 // information for replSetGetStatus from a different source than the nodes that aren't
1503 // ourself. After this setup, we call prepareStatusResponse and make sure that the fields
1504 // returned for each member match our expectations.
1505 Date_t startupTime = Date_t::fromMillisSinceEpoch(100);
1506 Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000);
1507 Seconds uptimeSecs(10);
1508 Date_t curTime = heartbeatTime + uptimeSecs;
1509 Timestamp electionTime(1, 2);
1510 OpTime oplogProgress(Timestamp(3, 4), 2);
1511 OpTime oplogDurable(Timestamp(3, 4), 1);
1512 OpTime lastCommittedOpTime(Timestamp(2, 3), -1);
1513 OpTime readConcernMajorityOpTime(Timestamp(4, 5), -1);
1514 std::string setName = "mySet";
1515
1516 ReplSetHeartbeatResponse hb;
1517 hb.setConfigVersion(1);
1518 hb.setState(MemberState::RS_SECONDARY);
1519 hb.setElectionTime(electionTime);
1520 hb.setHbMsg("READY");
1521 hb.setAppliedOpTime(oplogProgress);
1522 hb.setDurableOpTime(oplogDurable);
1523 StatusWith<ReplSetHeartbeatResponse> hbResponseGood = StatusWith<ReplSetHeartbeatResponse>(hb);
1524
1525 updateConfig(BSON("_id" << setName << "version" << 1 << "members"
1526 << BSON_ARRAY(BSON("_id" << 0 << "host"
1527 << "test0:1234")
1528 << BSON("_id" << 1 << "host"
1529 << "test1:1234")
1530 << BSON("_id" << 2 << "host"
1531 << "test2:1234")
1532 << BSON("_id" << 3 << "host"
1533 << "test3:1234"))),
1534 3,
1535 startupTime + Milliseconds(1));
1536
1537 // Now that the replica set is setup, put the members into the states we want them in.
1538 HostAndPort member = HostAndPort("test0:1234");
1539 getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(1), setName, member);
1540 getTopoCoord().processHeartbeatResponse(
1541 startupTime + Milliseconds(2), Milliseconds(1), member, hbResponseGood);
1542 getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(3), setName, member);
1543 Date_t timeoutTime =
1544 startupTime + Milliseconds(3) + ReplSetConfig::kDefaultHeartbeatTimeoutPeriod;
1545
1546 StatusWith<ReplSetHeartbeatResponse> hbResponseDown =
1547 StatusWith<ReplSetHeartbeatResponse>(Status(ErrorCodes::HostUnreachable, ""));
1548
1549 getTopoCoord().processHeartbeatResponse(
1550 timeoutTime, Milliseconds(5000), member, hbResponseDown);
1551
1552 member = HostAndPort("test1:1234");
1553 getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(2), setName, member);
1554 getTopoCoord().processHeartbeatResponse(
1555 heartbeatTime, Milliseconds(4000), member, hbResponseGood);
1556 makeSelfPrimary();
1557 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(oplogProgress, startupTime);
1558 getTopoCoord().getMyMemberData()->setLastDurableOpTime(oplogDurable, startupTime);
1559 getTopoCoord().advanceLastCommittedOpTime(lastCommittedOpTime);
1560
1561 // Now node 0 is down, node 1 is up, and for node 2 we have no heartbeat data yet.
1562 BSONObjBuilder statusBuilder;
1563 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
1564 getTopoCoord().prepareStatusResponse(
1565 TopologyCoordinator::ReplSetStatusArgs{
1566 curTime,
1567 static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)),
1568 readConcernMajorityOpTime,
1569 BSONObj()},
1570 &statusBuilder,
1571 &resultStatus);
1572 ASSERT_OK(resultStatus);
1573 BSONObj rsStatus = statusBuilder.obj();
1574
1575 // Test results for all non-self members
1576 ASSERT_EQUALS(setName, rsStatus["set"].String());
1577 ASSERT_EQUALS(curTime.asInt64(), rsStatus["date"].Date().asInt64());
1578 ASSERT_BSONOBJ_EQ(lastCommittedOpTime.toBSON(),
1579 rsStatus["optimes"]["lastCommittedOpTime"].Obj());
1580 {
1581 const auto optimes = rsStatus["optimes"].Obj();
1582 ASSERT_BSONOBJ_EQ(readConcernMajorityOpTime.toBSON(),
1583 optimes["readConcernMajorityOpTime"].Obj());
1584 ASSERT_EQUALS(oplogProgress.getTimestamp(), optimes["appliedOpTime"].timestamp());
1585 ASSERT_EQUALS((oplogDurable).getTimestamp(), optimes["durableOpTime"].timestamp());
1586 }
1587 std::vector<BSONElement> memberArray = rsStatus["members"].Array();
1588 ASSERT_EQUALS(4U, memberArray.size());
1589 BSONObj member0Status = memberArray[0].Obj();
1590 BSONObj member1Status = memberArray[1].Obj();
1591 BSONObj member2Status = memberArray[2].Obj();
1592
1593 // Test member 0, the node that's DOWN
1594 ASSERT_EQUALS(0, member0Status["_id"].numberInt());
1595 ASSERT_EQUALS("test0:1234", member0Status["name"].str());
1596 ASSERT_EQUALS(0, member0Status["health"].numberDouble());
1597 ASSERT_EQUALS(MemberState::RS_DOWN, member0Status["state"].numberInt());
1598 ASSERT_EQUALS("(not reachable/healthy)", member0Status["stateStr"].str());
1599 ASSERT_EQUALS(0, member0Status["uptime"].numberInt());
1600 ASSERT_EQUALS(Timestamp(), Timestamp(member0Status["optime"].timestampValue()));
1601 ASSERT_TRUE(member0Status.hasField("optimeDate"));
1602 ASSERT_EQUALS(Date_t::fromMillisSinceEpoch(Timestamp().getSecs() * 1000ULL),
1603 member0Status["optimeDate"].Date());
1604 ASSERT_EQUALS(timeoutTime, member0Status["lastHeartbeat"].date());
1605 ASSERT_EQUALS(Date_t(), member0Status["lastHeartbeatRecv"].date());
1606
1607 // Test member 1, the node that's SECONDARY
1608 ASSERT_EQUALS(1, member1Status["_id"].Int());
1609 ASSERT_EQUALS("test1:1234", member1Status["name"].String());
1610 ASSERT_EQUALS(1, member1Status["health"].Double());
1611 ASSERT_EQUALS(MemberState::RS_SECONDARY, member1Status["state"].numberInt());
1612 ASSERT_EQUALS(MemberState(MemberState::RS_SECONDARY).toString(),
1613 member1Status["stateStr"].String());
1614 ASSERT_EQUALS(durationCount<Seconds>(uptimeSecs), member1Status["uptime"].numberInt());
1615 ASSERT_EQUALS(oplogProgress.getTimestamp(),
1616 Timestamp(member1Status["optime"].timestampValue()));
1617 ASSERT_TRUE(member1Status.hasField("optimeDate"));
1618 ASSERT_EQUALS(Date_t::fromMillisSinceEpoch(oplogProgress.getSecs() * 1000ULL),
1619 member1Status["optimeDate"].Date());
1620 ASSERT_EQUALS(heartbeatTime, member1Status["lastHeartbeat"].date());
1621 ASSERT_EQUALS(Date_t(), member1Status["lastHeartbeatRecv"].date());
1622 ASSERT_EQUALS("READY", member1Status["lastHeartbeatMessage"].str());
1623
1624 // Test member 2, the node that's UNKNOWN
1625 ASSERT_EQUALS(2, member2Status["_id"].numberInt());
1626 ASSERT_EQUALS("test2:1234", member2Status["name"].str());
1627 ASSERT_EQUALS(-1, member2Status["health"].numberDouble());
1628 ASSERT_EQUALS(MemberState::RS_UNKNOWN, member2Status["state"].numberInt());
1629 ASSERT_EQUALS(MemberState(MemberState::RS_UNKNOWN).toString(), member2Status["stateStr"].str());
1630 ASSERT_TRUE(member2Status.hasField("uptime"));
1631 ASSERT_TRUE(member2Status.hasField("optime"));
1632 ASSERT_TRUE(member2Status.hasField("optimeDate"));
1633 ASSERT_FALSE(member2Status.hasField("lastHearbeat"));
1634 ASSERT_FALSE(member2Status.hasField("lastHearbeatRecv"));
1635
1636 // Now test results for ourself, the PRIMARY
1637 ASSERT_EQUALS(MemberState::RS_PRIMARY, rsStatus["myState"].numberInt());
1638 BSONObj selfStatus = memberArray[3].Obj();
1639 ASSERT_TRUE(selfStatus["self"].boolean());
1640 ASSERT_EQUALS(3, selfStatus["_id"].numberInt());
1641 ASSERT_EQUALS("test3:1234", selfStatus["name"].str());
1642 ASSERT_EQUALS(1, selfStatus["health"].numberDouble());
1643 ASSERT_EQUALS(MemberState::RS_PRIMARY, selfStatus["state"].numberInt());
1644 ASSERT_EQUALS(MemberState(MemberState::RS_PRIMARY).toString(), selfStatus["stateStr"].str());
1645 ASSERT_EQUALS(durationCount<Seconds>(uptimeSecs), selfStatus["uptime"].numberInt());
1646 ASSERT_EQUALS(oplogProgress.getTimestamp(), Timestamp(selfStatus["optime"].timestampValue()));
1647 ASSERT_TRUE(selfStatus.hasField("optimeDate"));
1648 ASSERT_EQUALS(Date_t::fromMillisSinceEpoch(oplogProgress.getSecs() * 1000ULL),
1649 selfStatus["optimeDate"].Date());
1650 ASSERT_FALSE(rsStatus.hasField("initialSyncStatus"));
1651
1652 ASSERT_EQUALS(2000, rsStatus["heartbeatIntervalMillis"].numberInt());
1653
1654 // TODO(spencer): Test electionTime and pingMs are set properly
1655 }
1656
TEST_F(TopoCoordTest,NodeReturnsInvalidReplicaSetConfigInResponseToGetStatusWhenAbsentFromConfig)1657 TEST_F(TopoCoordTest, NodeReturnsInvalidReplicaSetConfigInResponseToGetStatusWhenAbsentFromConfig) {
1658 // This test starts by configuring a TopologyCoordinator to NOT be a member of a 3 node
1659 // replica set. Then running prepareStatusResponse should fail.
1660 Date_t startupTime = Date_t::fromMillisSinceEpoch(100);
1661 Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000);
1662 Seconds uptimeSecs(10);
1663 Date_t curTime = heartbeatTime + uptimeSecs;
1664 OpTime oplogProgress(Timestamp(3, 4), 0);
1665 std::string setName = "mySet";
1666
1667 updateConfig(BSON("_id" << setName << "version" << 1 << "members"
1668 << BSON_ARRAY(BSON("_id" << 0 << "host"
1669 << "test0:1234")
1670 << BSON("_id" << 1 << "host"
1671 << "test1:1234")
1672 << BSON("_id" << 2 << "host"
1673 << "test2:1234"))),
1674 -1, // This one is not part of the replica set.
1675 startupTime + Milliseconds(1));
1676
1677 BSONObjBuilder statusBuilder;
1678 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
1679 getTopoCoord().prepareStatusResponse(
1680 TopologyCoordinator::ReplSetStatusArgs{
1681 curTime,
1682 static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)),
1683 OpTime(),
1684 BSONObj()},
1685 &statusBuilder,
1686 &resultStatus);
1687 ASSERT_NOT_OK(resultStatus);
1688 ASSERT_EQUALS(ErrorCodes::InvalidReplicaSetConfig, resultStatus);
1689 }
1690
TEST_F(TopoCoordTest,NodeReturnsReplicaSetNotFoundWhenFreshnessIsCheckedPriorToHavingAConfig)1691 TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenFreshnessIsCheckedPriorToHavingAConfig) {
1692 ReplicationCoordinator::ReplSetFreshArgs args;
1693 OpTime freshestOpTime(Timestamp(15, 10), 0);
1694 OpTime ourOpTime(Timestamp(10, 10), 0);
1695 OpTime staleOpTime(Timestamp(1, 1), 0);
1696 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1697
1698 // if we do not have an index in the config, we should get ErrorCodes::ReplicaSetNotFound
1699 BSONObjBuilder responseBuilder;
1700 Status status = internalErrorStatus;
1701 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1702 ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status);
1703 ASSERT_EQUALS("Cannot participate in elections because not initialized", status.reason());
1704 ASSERT_TRUE(responseBuilder.obj().isEmpty());
1705 }
1706
TEST_F(TopoCoordTest,NodeReturnsReplicaSetNotFoundWhenFreshnessIsCheckedWithTheIncorrectReplSetName)1707 TEST_F(TopoCoordTest,
1708 NodeReturnsReplicaSetNotFoundWhenFreshnessIsCheckedWithTheIncorrectReplSetName) {
1709 ReplicationCoordinator::ReplSetFreshArgs args;
1710 OpTime freshestOpTime(Timestamp(15, 10), 0);
1711 OpTime ourOpTime(Timestamp(10, 10), 0);
1712 OpTime staleOpTime(Timestamp(1, 1), 0);
1713 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1714
1715 updateConfig(BSON("_id"
1716 << "rs0"
1717 << "version"
1718 << 10
1719 << "members"
1720 << BSON_ARRAY(BSON("_id" << 10 << "host"
1721 << "hself"
1722 << "priority"
1723 << 10)
1724 << BSON("_id" << 20 << "host"
1725 << "h1")
1726 << BSON("_id" << 30 << "host"
1727 << "h2")
1728 << BSON("_id" << 40 << "host"
1729 << "h3"
1730 << "priority"
1731 << 10))),
1732 0);
1733 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1734
1735 // Test with incorrect replset name
1736 args.setName = "fakeset";
1737
1738 BSONObjBuilder responseBuilder;
1739 Status status = internalErrorStatus;
1740 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1741 ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status);
1742 ASSERT_TRUE(responseBuilder.obj().isEmpty());
1743 }
1744
TEST_F(TopoCoordTest,NodeReturnsFresherWhenFreshnessIsCheckedWithStaleConfigVersion)1745 TEST_F(TopoCoordTest, NodeReturnsFresherWhenFreshnessIsCheckedWithStaleConfigVersion) {
1746 ReplicationCoordinator::ReplSetFreshArgs args;
1747 OpTime freshestOpTime(Timestamp(15, 10), 0);
1748 OpTime ourOpTime(Timestamp(10, 10), 0);
1749 OpTime staleOpTime(Timestamp(1, 1), 0);
1750 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1751
1752 updateConfig(BSON("_id"
1753 << "rs0"
1754 << "version"
1755 << 10
1756 << "members"
1757 << BSON_ARRAY(BSON("_id" << 10 << "host"
1758 << "hself"
1759 << "priority"
1760 << 10)
1761 << BSON("_id" << 20 << "host"
1762 << "h1")
1763 << BSON("_id" << 30 << "host"
1764 << "h2")
1765 << BSON("_id" << 40 << "host"
1766 << "h3"
1767 << "priority"
1768 << 10))),
1769 0);
1770 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1771
1772 // Test with old config version
1773 args.setName = "rs0";
1774 args.cfgver = 5;
1775 args.id = 20;
1776 args.who = HostAndPort("h1");
1777 args.opTime = ourOpTime.getTimestamp();
1778
1779 BSONObjBuilder responseBuilder;
1780 Status status = internalErrorStatus;
1781 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1782 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1783 ASSERT_OK(status);
1784 BSONObj response = responseBuilder.obj();
1785 ASSERT_EQUALS("config version stale", response["info"].String());
1786 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
1787 ASSERT_TRUE(response["fresher"].Bool());
1788 ASSERT_FALSE(response["veto"].Bool());
1789 ASSERT_FALSE(response.hasField("errmsg"));
1790 }
1791
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedWithAMemberWhoIsNotInTheConfig)1792 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWithAMemberWhoIsNotInTheConfig) {
1793 ReplicationCoordinator::ReplSetFreshArgs args;
1794 OpTime freshestOpTime(Timestamp(15, 10), 0);
1795 OpTime ourOpTime(Timestamp(10, 10), 0);
1796 OpTime staleOpTime(Timestamp(1, 1), 0);
1797 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1798
1799 updateConfig(BSON("_id"
1800 << "rs0"
1801 << "version"
1802 << 10
1803 << "members"
1804 << BSON_ARRAY(BSON("_id" << 10 << "host"
1805 << "hself"
1806 << "priority"
1807 << 10)
1808 << BSON("_id" << 20 << "host"
1809 << "h1")
1810 << BSON("_id" << 30 << "host"
1811 << "h2")
1812 << BSON("_id" << 40 << "host"
1813 << "h3"
1814 << "priority"
1815 << 10))),
1816 0);
1817 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1818
1819 // Test with non-existent node.
1820 args.setName = "rs0";
1821 args.who = HostAndPort("fakenode");
1822 args.opTime = ourOpTime.getTimestamp();
1823 args.cfgver = 10;
1824 args.id = 0;
1825
1826 BSONObjBuilder responseBuilder;
1827 Status status = internalErrorStatus;
1828 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1829 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1830 ASSERT_OK(status);
1831 BSONObj response = responseBuilder.obj();
1832 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
1833 ASSERT_FALSE(response["fresher"].Bool());
1834 ASSERT_TRUE(response["veto"].Bool());
1835 ASSERT_EQUALS("replSet couldn't find member with id 0", response["errmsg"].String());
1836 }
1837
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedWhilePrimary)1838 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWhilePrimary) {
1839 ReplicationCoordinator::ReplSetFreshArgs args;
1840 OpTime freshestOpTime(Timestamp(15, 10), 0);
1841 OpTime ourOpTime(Timestamp(10, 10), 0);
1842 OpTime staleOpTime(Timestamp(1, 1), 0);
1843 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1844
1845 updateConfig(BSON("_id"
1846 << "rs0"
1847 << "version"
1848 << 10
1849 << "members"
1850 << BSON_ARRAY(BSON("_id" << 10 << "host"
1851 << "hself"
1852 << "priority"
1853 << 10)
1854 << BSON("_id" << 20 << "host"
1855 << "h1")
1856 << BSON("_id" << 30 << "host"
1857 << "h2")
1858 << BSON("_id" << 40 << "host"
1859 << "h3"
1860 << "priority"
1861 << 10))),
1862 0);
1863 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1864
1865
1866 // Test when we are primary.
1867 args.setName = "rs0";
1868 args.opTime = ourOpTime.getTimestamp();
1869 args.cfgver = 10;
1870 args.id = 20;
1871 args.who = HostAndPort("h1");
1872
1873 makeSelfPrimary();
1874
1875 BSONObjBuilder responseBuilder;
1876 Status status = internalErrorStatus;
1877
1878 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1879 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1880 ASSERT_OK(status);
1881 BSONObj response = responseBuilder.obj();
1882 ASSERT_FALSE(response.hasField("info"));
1883 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
1884 ASSERT_FALSE(response["fresher"].Bool());
1885 ASSERT_TRUE(response["veto"].Bool());
1886 ASSERT_EQUALS("I am already primary, h1:27017 can try again once I've stepped down",
1887 response["errmsg"].String());
1888 }
1889
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedWhilePrimaryExists)1890 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWhilePrimaryExists) {
1891 ReplicationCoordinator::ReplSetFreshArgs args;
1892 OpTime freshestOpTime(Timestamp(15, 10), 0);
1893 OpTime ourOpTime(Timestamp(10, 10), 0);
1894 OpTime staleOpTime(Timestamp(1, 1), 0);
1895 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1896
1897 updateConfig(BSON("_id"
1898 << "rs0"
1899 << "version"
1900 << 10
1901 << "members"
1902 << BSON_ARRAY(BSON("_id" << 10 << "host"
1903 << "hself"
1904 << "priority"
1905 << 10)
1906 << BSON("_id" << 20 << "host"
1907 << "h1")
1908 << BSON("_id" << 30 << "host"
1909 << "h2")
1910 << BSON("_id" << 40 << "host"
1911 << "h3"
1912 << "priority"
1913 << 10))),
1914 0);
1915 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1916
1917 args.setName = "rs0";
1918 args.opTime = ourOpTime.getTimestamp();
1919 args.cfgver = 10;
1920 args.id = 20;
1921 args.who = HostAndPort("h1");
1922
1923 // Test when someone else is primary.
1924 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1925 setSelfMemberState(MemberState::RS_SECONDARY);
1926 getTopoCoord()._setCurrentPrimaryForTest(2);
1927
1928 BSONObjBuilder responseBuilder;
1929 Status status = internalErrorStatus;
1930 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1931 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1932 ASSERT_OK(status);
1933 BSONObj response = responseBuilder.obj();
1934 ASSERT_FALSE(response.hasField("info"));
1935 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
1936 ASSERT_FALSE(response["fresher"].Bool());
1937 ASSERT_TRUE(response["veto"].Bool());
1938 ASSERT_EQUALS(
1939 "h1:27017 is trying to elect itself but h2:27017 is already primary and more "
1940 "up-to-date",
1941 response["errmsg"].String());
1942 }
1943
TEST_F(TopoCoordTest,NodeReturnsNotFreshestWhenFreshnessIsCheckedByALowPriorityNode)1944 TEST_F(TopoCoordTest, NodeReturnsNotFreshestWhenFreshnessIsCheckedByALowPriorityNode) {
1945 ReplicationCoordinator::ReplSetFreshArgs args;
1946 OpTime freshestOpTime(Timestamp(15, 10), 0);
1947 OpTime ourOpTime(Timestamp(10, 10), 0);
1948 OpTime staleOpTime(Timestamp(1, 1), 0);
1949 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
1950
1951 updateConfig(BSON("_id"
1952 << "rs0"
1953 << "version"
1954 << 10
1955 << "members"
1956 << BSON_ARRAY(BSON("_id" << 10 << "host"
1957 << "hself"
1958 << "priority"
1959 << 10)
1960 << BSON("_id" << 20 << "host"
1961 << "h1")
1962 << BSON("_id" << 30 << "host"
1963 << "h2")
1964 << BSON("_id" << 40 << "host"
1965 << "h3"
1966 << "priority"
1967 << 10))),
1968 0);
1969 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1970
1971 args.setName = "rs0";
1972 args.opTime = ourOpTime.getTimestamp();
1973 args.cfgver = 10;
1974 args.id = 20;
1975 args.who = HostAndPort("h1");
1976
1977 // Test trying to elect a node that is caught up but isn't the highest priority node.
1978 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1979 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, staleOpTime);
1980 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
1981
1982 BSONObjBuilder responseBuilder;
1983 Status status = internalErrorStatus;
1984 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
1985 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
1986 ASSERT_OK(status);
1987 BSONObj response = responseBuilder.obj();
1988 ASSERT_FALSE(response.hasField("info"));
1989 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
1990 ASSERT_FALSE(response["fresher"].Bool());
1991 ASSERT_TRUE(response["veto"].Bool());
1992 ASSERT(response["errmsg"].String().find("h1:27017 has lower priority of 1 than") !=
1993 std::string::npos)
1994 << response["errmsg"].String();
1995 }
1996
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedByANodeWeBelieveToBeDown)1997 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeWeBelieveToBeDown) {
1998 ReplicationCoordinator::ReplSetFreshArgs args;
1999 OpTime freshestOpTime(Timestamp(15, 10), 0);
2000 OpTime ourOpTime(Timestamp(10, 10), 0);
2001 OpTime staleOpTime(Timestamp(1, 1), 0);
2002 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2003
2004 updateConfig(BSON("_id"
2005 << "rs0"
2006 << "version"
2007 << 10
2008 << "members"
2009 << BSON_ARRAY(BSON("_id" << 10 << "host"
2010 << "hself"
2011 << "priority"
2012 << 10)
2013 << BSON("_id" << 20 << "host"
2014 << "h1")
2015 << BSON("_id" << 30 << "host"
2016 << "h2")
2017 << BSON("_id" << 40 << "host"
2018 << "h3"
2019 << "priority"
2020 << 10))),
2021 0);
2022 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2023
2024 args.setName = "rs0";
2025 args.opTime = ourOpTime.getTimestamp();
2026 args.cfgver = 10;
2027 args.id = 40;
2028 args.who = HostAndPort("h3");
2029
2030 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2031 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, staleOpTime);
2032 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2033
2034 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime());
2035
2036 BSONObjBuilder responseBuilder;
2037 Status status = internalErrorStatus;
2038 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2039 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2040 ASSERT_OK(status);
2041 BSONObj response = responseBuilder.obj();
2042 ASSERT_FALSE(response.hasField("info"));
2043 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2044 ASSERT_FALSE(response["fresher"].Bool());
2045 ASSERT_TRUE(response["veto"].Bool());
2046 ASSERT_NE(std::string::npos,
2047 response["errmsg"].String().find(
2048 "I don't think h3:27017 is electable because the member is not "
2049 "currently a secondary"))
2050 << response["errmsg"].String();
2051 }
2052
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedByANodeThatIsPrimary)2053 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsPrimary) {
2054 ReplicationCoordinator::ReplSetFreshArgs args;
2055 OpTime freshestOpTime(Timestamp(15, 10), 0);
2056 OpTime ourOpTime(Timestamp(10, 10), 0);
2057 OpTime staleOpTime(Timestamp(1, 1), 0);
2058 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2059
2060 updateConfig(BSON("_id"
2061 << "rs0"
2062 << "version"
2063 << 10
2064 << "members"
2065 << BSON_ARRAY(BSON("_id" << 10 << "host"
2066 << "hself"
2067 << "priority"
2068 << 10)
2069 << BSON("_id" << 20 << "host"
2070 << "h1")
2071 << BSON("_id" << 30 << "host"
2072 << "h2")
2073 << BSON("_id" << 40 << "host"
2074 << "h3"
2075 << "priority"
2076 << 10))),
2077 0);
2078 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2079
2080 args.setName = "rs0";
2081 args.opTime = ourOpTime.getTimestamp();
2082 args.cfgver = 10;
2083 args.id = 40;
2084 args.who = HostAndPort("h3");
2085
2086 // Test trying to elect a node that isn't electable because it's PRIMARY
2087 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
2088 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_PRIMARY, ourOpTime);
2089 ASSERT_EQUALS(3, getCurrentPrimaryIndex());
2090
2091 BSONObjBuilder responseBuilder;
2092 Status status = internalErrorStatus;
2093 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2094 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2095 ASSERT_OK(status);
2096 BSONObj response = responseBuilder.obj();
2097 ASSERT_FALSE(response.hasField("info"));
2098 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2099 ASSERT_FALSE(response["fresher"].Bool());
2100 ASSERT_TRUE(response["veto"].Bool());
2101 ASSERT_NE(
2102 std::string::npos,
2103 response["errmsg"].String().find(
2104 "I don't think h3:27017 is electable because the member is not currently a secondary"))
2105 << response["errmsg"].String();
2106 }
2107
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedByANodeThatIsInStartup)2108 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsInStartup) {
2109 ReplicationCoordinator::ReplSetFreshArgs args;
2110 OpTime freshestOpTime(Timestamp(15, 10), 0);
2111 OpTime ourOpTime(Timestamp(10, 10), 0);
2112 OpTime staleOpTime(Timestamp(1, 1), 0);
2113 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2114
2115 updateConfig(BSON("_id"
2116 << "rs0"
2117 << "version"
2118 << 10
2119 << "members"
2120 << BSON_ARRAY(BSON("_id" << 10 << "host"
2121 << "hself"
2122 << "priority"
2123 << 10)
2124 << BSON("_id" << 20 << "host"
2125 << "h1")
2126 << BSON("_id" << 30 << "host"
2127 << "h2")
2128 << BSON("_id" << 40 << "host"
2129 << "h3"
2130 << "priority"
2131 << 10))),
2132 0);
2133 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2134
2135 args.setName = "rs0";
2136 args.opTime = ourOpTime.getTimestamp();
2137 args.cfgver = 10;
2138 args.id = 40;
2139 args.who = HostAndPort("h3");
2140
2141 // Test trying to elect a node that isn't electable because it's STARTUP
2142 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_STARTUP, ourOpTime);
2143
2144 BSONObjBuilder responseBuilder;
2145 Status status = internalErrorStatus;
2146 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2147 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2148 ASSERT_OK(status);
2149 BSONObj response = responseBuilder.obj();
2150 ASSERT_FALSE(response.hasField("info"));
2151 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2152 ASSERT_FALSE(response["fresher"].Bool());
2153 ASSERT_TRUE(response["veto"].Bool());
2154 ASSERT_NE(std::string::npos,
2155 response["errmsg"].String().find(
2156 "I don't think h3:27017 is electable because the member is not "
2157 "currently a secondary"))
2158 << response["errmsg"].String();
2159 }
2160
TEST_F(TopoCoordTest,VetoWhenFreshnessIsCheckedByANodeThatIsRecovering)2161 TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsRecovering) {
2162 ReplicationCoordinator::ReplSetFreshArgs args;
2163 OpTime freshestOpTime(Timestamp(15, 10), 0);
2164 OpTime ourOpTime(Timestamp(10, 10), 0);
2165 OpTime staleOpTime(Timestamp(1, 1), 0);
2166 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2167
2168 updateConfig(BSON("_id"
2169 << "rs0"
2170 << "version"
2171 << 10
2172 << "members"
2173 << BSON_ARRAY(BSON("_id" << 10 << "host"
2174 << "hself"
2175 << "priority"
2176 << 10)
2177 << BSON("_id" << 20 << "host"
2178 << "h1")
2179 << BSON("_id" << 30 << "host"
2180 << "h2")
2181 << BSON("_id" << 40 << "host"
2182 << "h3"
2183 << "priority"
2184 << 10))),
2185 0);
2186 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2187
2188 args.setName = "rs0";
2189 args.opTime = ourOpTime.getTimestamp();
2190 args.cfgver = 10;
2191 args.id = 40;
2192 args.who = HostAndPort("h3");
2193
2194 // Test trying to elect a node that isn't electable because it's RECOVERING
2195 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_RECOVERING, ourOpTime);
2196
2197 BSONObjBuilder responseBuilder;
2198 Status status = internalErrorStatus;
2199 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2200 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2201 ASSERT_OK(status);
2202 BSONObj response = responseBuilder.obj();
2203 ASSERT_FALSE(response.hasField("info"));
2204 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2205 ASSERT_FALSE(response["fresher"].Bool());
2206 ASSERT_TRUE(response["veto"].Bool());
2207 ASSERT_NE(std::string::npos,
2208 response["errmsg"].String().find(
2209 "I don't think h3:27017 is electable because the member is not "
2210 "currently a secondary"))
2211 << response["errmsg"].String();
2212 }
2213
TEST_F(TopoCoordTest,RespondPositivelyWhenFreshnessIsCheckedByAFresherAndLowerPriorityNodeThanExistingPrimary)2214 TEST_F(TopoCoordTest,
2215 RespondPositivelyWhenFreshnessIsCheckedByAFresherAndLowerPriorityNodeThanExistingPrimary) {
2216 ReplicationCoordinator::ReplSetFreshArgs args;
2217 OpTime freshestOpTime(Timestamp(15, 10), 0);
2218 OpTime ourOpTime(Timestamp(10, 10), 0);
2219 OpTime staleOpTime(Timestamp(1, 1), 0);
2220 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2221
2222 updateConfig(BSON("_id"
2223 << "rs0"
2224 << "version"
2225 << 10
2226 << "members"
2227 << BSON_ARRAY(BSON("_id" << 10 << "host"
2228 << "hself"
2229 << "priority"
2230 << 10)
2231 << BSON("_id" << 20 << "host"
2232 << "h1")
2233 << BSON("_id" << 30 << "host"
2234 << "h2")
2235 << BSON("_id" << 40 << "host"
2236 << "h3"
2237 << "priority"
2238 << 10))),
2239 0);
2240 // Test trying to elect a node that is fresher but lower priority than the existing primary
2241 args.setName = "rs0";
2242 args.opTime = ourOpTime.getTimestamp();
2243 args.cfgver = 10;
2244 args.id = 30;
2245 args.who = HostAndPort("h2");
2246
2247 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
2248 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_PRIMARY, ourOpTime);
2249 ASSERT_EQUALS(3, getCurrentPrimaryIndex());
2250 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, freshestOpTime);
2251
2252 BSONObjBuilder responseBuilder;
2253 Status status = internalErrorStatus;
2254 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2255 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2256 ASSERT_OK(status);
2257 BSONObj response = responseBuilder.obj();
2258 ASSERT_FALSE(response.hasField("info"));
2259 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2260 ASSERT_TRUE(response["fresher"].Bool());
2261 ASSERT_FALSE(response["veto"].Bool());
2262 ASSERT_FALSE(response.hasField("errmsg"));
2263 }
2264
TEST_F(TopoCoordTest,RespondPositivelyWhenFreshnessIsCheckedByAnElectableNode)2265 TEST_F(TopoCoordTest, RespondPositivelyWhenFreshnessIsCheckedByAnElectableNode) {
2266 ReplicationCoordinator::ReplSetFreshArgs args;
2267 OpTime freshestOpTime(Timestamp(15, 10), 0);
2268 OpTime ourOpTime(Timestamp(10, 10), 0);
2269 OpTime staleOpTime(Timestamp(1, 1), 0);
2270 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2271
2272 updateConfig(BSON("_id"
2273 << "rs0"
2274 << "version"
2275 << 10
2276 << "members"
2277 << BSON_ARRAY(BSON("_id" << 10 << "host"
2278 << "hself"
2279 << "priority"
2280 << 10)
2281 << BSON("_id" << 20 << "host"
2282 << "h1")
2283 << BSON("_id" << 30 << "host"
2284 << "h2")
2285 << BSON("_id" << 40 << "host"
2286 << "h3"
2287 << "priority"
2288 << 10))),
2289 0);
2290 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2291
2292
2293 // Test trying to elect a valid node
2294 args.setName = "rs0";
2295 args.opTime = ourOpTime.getTimestamp();
2296 args.cfgver = 10;
2297 args.id = 40;
2298 args.who = HostAndPort("h3");
2299
2300 receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime());
2301 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2302
2303 BSONObjBuilder responseBuilder;
2304 Status status = internalErrorStatus;
2305 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2306 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2307 ASSERT_OK(status);
2308 BSONObj response = responseBuilder.obj();
2309 ASSERT_FALSE(response.hasField("info")) << response.toString();
2310 ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
2311 ASSERT_FALSE(response["fresher"].Bool()) << response.toString();
2312 ASSERT_FALSE(response["veto"].Bool()) << response.toString();
2313 ASSERT_FALSE(response.hasField("errmsg")) << response.toString();
2314 }
2315
TEST_F(TopoCoordTest,NodeReturnsBadValueWhenFreshnessIsCheckedByANodeWithOurID)2316 TEST_F(TopoCoordTest, NodeReturnsBadValueWhenFreshnessIsCheckedByANodeWithOurID) {
2317 ReplicationCoordinator::ReplSetFreshArgs args;
2318 OpTime freshestOpTime(Timestamp(15, 10), 0);
2319 OpTime ourOpTime(Timestamp(10, 10), 0);
2320 OpTime staleOpTime(Timestamp(1, 1), 0);
2321 Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status");
2322
2323 updateConfig(BSON("_id"
2324 << "rs0"
2325 << "version"
2326 << 10
2327 << "members"
2328 << BSON_ARRAY(BSON("_id" << 10 << "host"
2329 << "hself"
2330 << "priority"
2331 << 10)
2332 << BSON("_id" << 20 << "host"
2333 << "h1")
2334 << BSON("_id" << 30 << "host"
2335 << "h2")
2336 << BSON("_id" << 40 << "host"
2337 << "h3"
2338 << "priority"
2339 << 10))),
2340 0);
2341 heartbeatFromMember(HostAndPort("h1"), "rs0", MemberState::RS_SECONDARY, ourOpTime);
2342
2343 // Test with our id
2344 args.setName = "rs0";
2345 args.opTime = ourOpTime.getTimestamp();
2346 args.cfgver = 10;
2347 args.who = HostAndPort("h3");
2348 args.id = 10;
2349
2350 BSONObjBuilder responseBuilder;
2351 Status status = internalErrorStatus;
2352 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(ourOpTime, Date_t());
2353 getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status);
2354 ASSERT_EQUALS(ErrorCodes::BadValue, status);
2355 ASSERT_EQUALS(
2356 "Received replSetFresh command from member with the same member ID as ourself: 10",
2357 status.reason());
2358 ASSERT_TRUE(responseBuilder.obj().isEmpty());
2359 }
2360
TEST_F(TopoCoordTest,HeartbeatFrequencyShouldBeIncreasedWhenArbiter)2361 TEST_F(TopoCoordTest, HeartbeatFrequencyShouldBeIncreasedWhenArbiter) {
2362 // This tests that arbiters issue heartbeats at higher frequencies.
2363 TopoCoordTest::setUp();
2364 updateConfig(fromjson("{_id:'mySet', version:1, protocolVersion:1, members:["
2365 "{_id:1, host:'node1:12345', arbiterOnly:true}, "
2366 "{_id:2, host:'node2:12345'}], "
2367 "settings:{heartbeatIntervalMillis:3000, electionTimeoutMillis:5000}}"),
2368 0);
2369 HostAndPort target("host2", 27017);
2370 Date_t requestDate = now();
2371 std::pair<ReplSetHeartbeatArgs, Milliseconds> uppingRequest =
2372 getTopoCoord().prepareHeartbeatRequest(requestDate, "myset", target);
2373 auto action = getTopoCoord().processHeartbeatResponse(
2374 requestDate, Milliseconds(0), target, makeStatusWith<ReplSetHeartbeatResponse>());
2375 Date_t expected(now() + Milliseconds(2500));
2376 ASSERT_EQUALS(expected, action.getNextHeartbeatStartDate());
2377 }
2378
2379 class HeartbeatResponseTest : public TopoCoordTest {
2380 public:
setUp()2381 virtual void setUp() {
2382 TopoCoordTest::setUp();
2383 updateConfig(BSON("_id"
2384 << "rs0"
2385 << "version"
2386 << 5
2387 << "members"
2388 << BSON_ARRAY(BSON("_id" << 0 << "host"
2389 << "host1:27017")
2390 << BSON("_id" << 1 << "host"
2391 << "host2:27017")
2392 << BSON("_id" << 2 << "host"
2393 << "host3:27017"))
2394 << "settings"
2395 << BSON("heartbeatTimeoutSecs" << 5)),
2396 0);
2397 }
2398 };
2399
2400 class HeartbeatResponseTestOneRetry : public HeartbeatResponseTest {
2401 public:
setUp()2402 virtual void setUp() {
2403 HeartbeatResponseTest::setUp();
2404
2405 // Bring up the node we are heartbeating.
2406 _target = HostAndPort("host2", 27017);
2407 Date_t _upRequestDate = unittest::assertGet(dateFromISOString("2014-08-29T12:55Z"));
2408 std::pair<ReplSetHeartbeatArgs, Milliseconds> uppingRequest =
2409 getTopoCoord().prepareHeartbeatRequest(_upRequestDate, "rs0", _target);
2410 HeartbeatResponseAction upAction = getTopoCoord().processHeartbeatResponse(
2411 _upRequestDate,
2412 Milliseconds(0),
2413 _target,
2414 makeStatusWith<ReplSetHeartbeatResponse>()); // We've never applied anything.
2415 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, upAction.getAction());
2416 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2417
2418
2419 // Time of first request for this heartbeat period
2420 _firstRequestDate = unittest::assertGet(dateFromISOString("2014-08-29T13:00Z"));
2421
2422 // Initial heartbeat attempt prepared, at t + 0.
2423 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2424 getTopoCoord().prepareHeartbeatRequest(_firstRequestDate, "rs0", _target);
2425 // 5 seconds to successfully complete the heartbeat before the timeout expires.
2426 ASSERT_EQUALS(5000, durationCount<Milliseconds>(request.second));
2427
2428 // Initial heartbeat request fails at t + 4000ms
2429 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2430 _firstRequestDate + Seconds(4), // 4 seconds elapsed, retry allowed.
2431 Milliseconds(3990), // Spent 3.99 of the 4 seconds in the network.
2432 _target,
2433 StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"));
2434
2435 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
2436 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2437 // Because the heartbeat failed without timing out, we expect to retry immediately.
2438 ASSERT_EQUALS(_firstRequestDate + Seconds(4), action.getNextHeartbeatStartDate());
2439
2440 // First heartbeat retry prepared, at t + 4000ms.
2441 request = getTopoCoord().prepareHeartbeatRequest(
2442 _firstRequestDate + Milliseconds(4000), "rs0", _target);
2443 // One second left to complete the heartbeat.
2444 ASSERT_EQUALS(1000, durationCount<Milliseconds>(request.second));
2445
2446 // Ensure a single failed heartbeat did not cause the node to be marked down
2447 BSONObjBuilder statusBuilder;
2448 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
2449 getTopoCoord().prepareStatusResponse(
2450 TopologyCoordinator::ReplSetStatusArgs{
2451 _firstRequestDate + Milliseconds(4000), 10, OpTime(), BSONObj()},
2452 &statusBuilder,
2453 &resultStatus);
2454 ASSERT_OK(resultStatus);
2455 BSONObj rsStatus = statusBuilder.obj();
2456 std::vector<BSONElement> memberArray = rsStatus["members"].Array();
2457 BSONObj member1Status = memberArray[1].Obj();
2458
2459 ASSERT_EQUALS(1, member1Status["_id"].Int());
2460 ASSERT_EQUALS(1, member1Status["health"].Double());
2461
2462 ASSERT_EQUALS(Timestamp(0, 0),
2463 Timestamp(rsStatus["optimes"]["lastCommittedOpTime"]["ts"].timestampValue()));
2464 ASSERT_EQUALS(-1LL, rsStatus["optimes"]["lastCommittedOpTime"]["t"].numberLong());
2465 ASSERT_FALSE(rsStatus["optimes"].Obj().hasField("readConcernMajorityOpTime"));
2466 }
2467
firstRequestDate()2468 Date_t firstRequestDate() {
2469 return _firstRequestDate;
2470 }
2471
target()2472 HostAndPort target() {
2473 return _target;
2474 }
2475
2476 private:
2477 Date_t _firstRequestDate;
2478 HostAndPort _target;
2479 };
2480
2481 class HeartbeatResponseTestTwoRetries : public HeartbeatResponseTestOneRetry {
2482 public:
setUp()2483 virtual void setUp() {
2484 HeartbeatResponseTestOneRetry::setUp();
2485 // First retry fails at t + 4500ms
2486 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2487 firstRequestDate() + Milliseconds(4500), // 4.5 of the 5 seconds elapsed;
2488 // could retry.
2489 Milliseconds(400), // Spent 0.4 of the 0.5 seconds in the network.
2490 target(),
2491 StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"));
2492 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
2493 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2494 // Because the first retry failed without timing out, we expect to retry immediately.
2495 ASSERT_EQUALS(firstRequestDate() + Milliseconds(4500), action.getNextHeartbeatStartDate());
2496
2497 // Second retry prepared at t + 4500ms.
2498 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2499 getTopoCoord().prepareHeartbeatRequest(
2500 firstRequestDate() + Milliseconds(4500), "rs0", target());
2501 // 500ms left to complete the heartbeat.
2502 ASSERT_EQUALS(500, durationCount<Milliseconds>(request.second));
2503
2504 // Ensure a second failed heartbeat did not cause the node to be marked down
2505 BSONObjBuilder statusBuilder;
2506 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
2507 getTopoCoord().prepareStatusResponse(
2508 TopologyCoordinator::ReplSetStatusArgs{
2509 firstRequestDate() + Seconds(4), 10, OpTime(), BSONObj()},
2510 &statusBuilder,
2511 &resultStatus);
2512 ASSERT_OK(resultStatus);
2513 BSONObj rsStatus = statusBuilder.obj();
2514 std::vector<BSONElement> memberArray = rsStatus["members"].Array();
2515 BSONObj member1Status = memberArray[1].Obj();
2516
2517 ASSERT_EQUALS(1, member1Status["_id"].Int());
2518 ASSERT_EQUALS(1, member1Status["health"].Double());
2519 }
2520 };
2521
2522 class HeartbeatResponseHighVerbosityTest : public HeartbeatResponseTest {
2523 public:
setUp()2524 virtual void setUp() {
2525 HeartbeatResponseTest::setUp();
2526 // set verbosity as high as the highest verbosity log message we'd like to check for
2527 logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(3));
2528 }
2529
tearDown()2530 virtual void tearDown() {
2531 HeartbeatResponseTest::tearDown();
2532 logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Log());
2533 }
2534 };
2535
TEST_F(HeartbeatResponseHighVerbosityTest,LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatBelievesWeAreDown)2536 TEST_F(HeartbeatResponseHighVerbosityTest,
2537 LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatBelievesWeAreDown) {
2538 // request heartbeat
2539 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2540 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2"));
2541
2542 ReplSetHeartbeatResponse believesWeAreDownResponse;
2543 believesWeAreDownResponse.noteReplSet();
2544 believesWeAreDownResponse.setSetName("rs0");
2545 believesWeAreDownResponse.setState(MemberState::RS_SECONDARY);
2546 believesWeAreDownResponse.setElectable(true);
2547 believesWeAreDownResponse.noteStateDisagreement();
2548 startCapturingLogMessages();
2549 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2550 now()++, // Time is left.
2551 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2552 HostAndPort("host2"),
2553 StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse));
2554 stopCapturingLogMessages();
2555 ASSERT_NO_ACTION(action.getAction());
2556 ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down"));
2557 }
2558
TEST_F(HeartbeatResponseHighVerbosityTest,LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatIsNotInConfig)2559 TEST_F(HeartbeatResponseHighVerbosityTest,
2560 LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatIsNotInConfig) {
2561 // request heartbeat
2562 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2563 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host5"));
2564
2565 ReplSetHeartbeatResponse memberMissingResponse;
2566 memberMissingResponse.noteReplSet();
2567 memberMissingResponse.setSetName("rs0");
2568 memberMissingResponse.setState(MemberState::RS_SECONDARY);
2569 memberMissingResponse.setElectable(true);
2570 memberMissingResponse.noteStateDisagreement();
2571 startCapturingLogMessages();
2572 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2573 now()++, // Time is left.
2574 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2575 HostAndPort("host5"),
2576 StatusWith<ReplSetHeartbeatResponse>(memberMissingResponse));
2577 stopCapturingLogMessages();
2578 ASSERT_NO_ACTION(action.getAction());
2579 ASSERT_EQUALS(1, countLogLinesContaining("Could not find host5:27017 in current config"));
2580 }
2581
2582 // TODO(dannenberg) figure out why this test is useful
TEST_F(HeartbeatResponseHighVerbosityTest,UpdateHeartbeatDataSameConfig)2583 TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataSameConfig) {
2584 // request heartbeat
2585 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2586 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2"));
2587
2588 // construct a copy of the original config for log message checking later
2589 // see HeartbeatResponseTest for the origin of the original config
2590 ReplSetConfig originalConfig;
2591 originalConfig
2592 .initialize(BSON("_id"
2593 << "rs0"
2594 << "version"
2595 << 5
2596 << "members"
2597 << BSON_ARRAY(BSON("_id" << 0 << "host"
2598 << "host1:27017")
2599 << BSON("_id" << 1 << "host"
2600 << "host2:27017")
2601 << BSON("_id" << 2 << "host"
2602 << "host3:27017"))
2603 << "settings"
2604 << BSON("heartbeatTimeoutSecs" << 5)))
2605 .transitional_ignore();
2606
2607 ReplSetHeartbeatResponse sameConfigResponse;
2608 sameConfigResponse.noteReplSet();
2609 sameConfigResponse.setSetName("rs0");
2610 sameConfigResponse.setState(MemberState::RS_SECONDARY);
2611 sameConfigResponse.setElectable(true);
2612 sameConfigResponse.noteStateDisagreement();
2613 sameConfigResponse.setConfigVersion(2);
2614 sameConfigResponse.setConfig(originalConfig);
2615 startCapturingLogMessages();
2616 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2617 now()++, // Time is left.
2618 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2619 HostAndPort("host2"),
2620 StatusWith<ReplSetHeartbeatResponse>(sameConfigResponse));
2621 stopCapturingLogMessages();
2622 ASSERT_NO_ACTION(action.getAction());
2623 ASSERT_EQUALS(1, countLogLinesContaining("Config from heartbeat response was same as ours."));
2624 }
2625
2626 // TODO(dannenberg) change the name and functionality of this to match what this claims it is
TEST_F(HeartbeatResponseHighVerbosityTest,UpdateHeartbeatDataOldConfig)2627 TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataOldConfig) {
2628 // request heartbeat
2629 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2630 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2"));
2631
2632 ReplSetHeartbeatResponse believesWeAreDownResponse;
2633 believesWeAreDownResponse.noteReplSet();
2634 believesWeAreDownResponse.setSetName("rs0");
2635 believesWeAreDownResponse.setState(MemberState::RS_SECONDARY);
2636 believesWeAreDownResponse.setElectable(true);
2637 believesWeAreDownResponse.noteStateDisagreement();
2638 startCapturingLogMessages();
2639 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2640 now()++, // Time is left.
2641 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2642 HostAndPort("host2"),
2643 StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse));
2644 stopCapturingLogMessages();
2645 ASSERT_NO_ACTION(action.getAction());
2646 ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down"));
2647 }
2648
TEST_F(HeartbeatResponseTestOneRetry,ReconfigWhenHeartbeatResponseContainsAConfig)2649 TEST_F(HeartbeatResponseTestOneRetry, ReconfigWhenHeartbeatResponseContainsAConfig) {
2650 // Confirm that action responses can come back from retries; in this, expect a Reconfig
2651 // action.
2652 ReplSetConfig newConfig;
2653 ASSERT_OK(newConfig.initialize(BSON("_id"
2654 << "rs0"
2655 << "version"
2656 << 7
2657 << "members"
2658 << BSON_ARRAY(BSON("_id" << 0 << "host"
2659 << "host1:27017")
2660 << BSON("_id" << 1 << "host"
2661 << "host2:27017")
2662 << BSON("_id" << 2 << "host"
2663 << "host3:27017")
2664 << BSON("_id" << 3 << "host"
2665 << "host4:27017"))
2666 << "settings"
2667 << BSON("heartbeatTimeoutSecs" << 5))));
2668 ASSERT_OK(newConfig.validate());
2669
2670 ReplSetHeartbeatResponse reconfigResponse;
2671 reconfigResponse.noteReplSet();
2672 reconfigResponse.setSetName("rs0");
2673 reconfigResponse.setState(MemberState::RS_SECONDARY);
2674 reconfigResponse.setElectable(true);
2675 reconfigResponse.setConfigVersion(1);
2676 reconfigResponse.setConfig(newConfig);
2677 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2678 firstRequestDate() + Milliseconds(4500), // Time is left.
2679 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2680 target(),
2681 StatusWith<ReplSetHeartbeatResponse>(reconfigResponse));
2682 ASSERT_EQUALS(HeartbeatResponseAction::Reconfig, action.getAction());
2683 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2684 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
2685 }
2686
TEST_F(HeartbeatResponseTestOneRetry,StepDownRemotePrimaryWhenWeWereElectedMoreRecently)2687 TEST_F(HeartbeatResponseTestOneRetry, StepDownRemotePrimaryWhenWeWereElectedMoreRecently) {
2688 // Confirm that action responses can come back from retries; in this, expect a
2689 // StepDownRemotePrimary action.
2690
2691 // make self primary
2692 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
2693 makeSelfPrimary(Timestamp(5, 0));
2694 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
2695
2696 ReplSetHeartbeatResponse electedMoreRecentlyResponse;
2697 electedMoreRecentlyResponse.noteReplSet();
2698 electedMoreRecentlyResponse.setSetName("rs0");
2699 electedMoreRecentlyResponse.setState(MemberState::RS_PRIMARY);
2700 electedMoreRecentlyResponse.setElectable(true);
2701 electedMoreRecentlyResponse.setElectionTime(Timestamp(3, 0));
2702 electedMoreRecentlyResponse.setConfigVersion(5);
2703 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2704 firstRequestDate() + Milliseconds(4500), // Time is left.
2705 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2706 target(),
2707 StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse));
2708 ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, action.getAction());
2709 ASSERT_EQUALS(1, action.getPrimaryConfigIndex());
2710 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
2711 }
2712
TEST_F(HeartbeatResponseTestOneRetry,StepDownSelfWhenRemoteNodeWasElectedMoreRecently)2713 TEST_F(HeartbeatResponseTestOneRetry, StepDownSelfWhenRemoteNodeWasElectedMoreRecently) {
2714 // Confirm that action responses can come back from retries; in this, expect a StepDownSelf
2715 // action.
2716
2717 // acknowledge the other member so that we see a majority
2718 HeartbeatResponseAction action =
2719 receiveDownHeartbeat(HostAndPort("host3"), "rs0", OpTime(Timestamp(100, 0), 0));
2720 ASSERT_NO_ACTION(action.getAction());
2721
2722 // make us PRIMARY
2723 makeSelfPrimary();
2724
2725 ReplSetHeartbeatResponse electedMoreRecentlyResponse;
2726 electedMoreRecentlyResponse.noteReplSet();
2727 electedMoreRecentlyResponse.setSetName("rs0");
2728 electedMoreRecentlyResponse.setState(MemberState::RS_PRIMARY);
2729 electedMoreRecentlyResponse.setElectable(false);
2730 electedMoreRecentlyResponse.setElectionTime(Timestamp(10, 0));
2731 electedMoreRecentlyResponse.setConfigVersion(5);
2732 action = getTopoCoord().processHeartbeatResponse(
2733 firstRequestDate() + Milliseconds(4500), // Time is left.
2734 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2735 target(),
2736 StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse));
2737 ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, action.getAction());
2738 ASSERT_EQUALS(0, action.getPrimaryConfigIndex());
2739 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
2740 // Doesn't actually do the stepdown until stepDownIfPending is called
2741 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
2742 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
2743
2744 getTopoCoord().prepareForUnconditionalStepDown();
2745 getTopoCoord().finishUnconditionalStepDown();
2746 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2747 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
2748 }
2749
TEST_F(HeartbeatResponseTestOneRetry,StartElectionAfterReceivingAHeartbeatWhileNoPrimaryExistsAndWeAreElectable)2750 TEST_F(HeartbeatResponseTestOneRetry,
2751 StartElectionAfterReceivingAHeartbeatWhileNoPrimaryExistsAndWeAreElectable) {
2752 // Confirm that action responses can come back from retries; in this, expect a StartElection
2753 // action.
2754
2755 // acknowledge the other member so that we see a majority
2756 OpTime election = OpTime(Timestamp(400, 0), 0);
2757 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
2758 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
2759 HeartbeatResponseAction action = receiveUpHeartbeat(
2760 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
2761 ASSERT_NO_ACTION(action.getAction());
2762
2763 // make sure we are electable
2764 setSelfMemberState(MemberState::RS_SECONDARY);
2765 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(election, Date_t());
2766
2767 ReplSetHeartbeatResponse startElectionResponse;
2768 startElectionResponse.noteReplSet();
2769 startElectionResponse.setSetName("rs0");
2770 startElectionResponse.setState(MemberState::RS_SECONDARY);
2771 startElectionResponse.setElectable(true);
2772 startElectionResponse.setConfigVersion(5);
2773 action = getTopoCoord().processHeartbeatResponse(
2774 firstRequestDate() + Milliseconds(4500), // Time is left.
2775 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2776 target(),
2777 StatusWith<ReplSetHeartbeatResponse>(startElectionResponse));
2778 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, action.getAction());
2779 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
2780 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
2781 }
2782
TEST_F(HeartbeatResponseTestTwoRetries,NodeDoesNotRetryHeartbeatsAfterFailingTwiceInARow)2783 TEST_F(HeartbeatResponseTestTwoRetries, NodeDoesNotRetryHeartbeatsAfterFailingTwiceInARow) {
2784 // Confirm that the topology coordinator attempts to retry a failed heartbeat two times
2785 // after initial failure, assuming that the heartbeat timeout (set to 5 seconds in the
2786 // fixture) has not expired.
2787 //
2788 // Failed heartbeats propose taking no action, other than scheduling the next heartbeat. We
2789 // can detect a retry vs the next regularly scheduled heartbeat because retries are
2790 // scheduled immediately, while subsequent heartbeats are scheduled after the hard-coded
2791 // heartbeat interval of 2 seconds.
2792
2793 // Second retry fails at t + 4800ms
2794 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2795 firstRequestDate() + Milliseconds(4800), // 4.8 of the 5 seconds elapsed;
2796 // could still retry.
2797 Milliseconds(100), // Spent 0.1 of the 0.3 seconds in the network.
2798 target(),
2799 StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"));
2800 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
2801 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2802 // Because this is the second retry, rather than retry again, we expect to wait for the
2803 // heartbeat interval of 2 seconds to elapse.
2804 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6800), action.getNextHeartbeatStartDate());
2805
2806 // Ensure a third failed heartbeat caused the node to be marked down
2807 BSONObjBuilder statusBuilder;
2808 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
2809 getTopoCoord().prepareStatusResponse(
2810 TopologyCoordinator::ReplSetStatusArgs{
2811 firstRequestDate() + Milliseconds(4900), 10, OpTime(), BSONObj()},
2812 &statusBuilder,
2813 &resultStatus);
2814 ASSERT_OK(resultStatus);
2815 BSONObj rsStatus = statusBuilder.obj();
2816 std::vector<BSONElement> memberArray = rsStatus["members"].Array();
2817 BSONObj member1Status = memberArray[1].Obj();
2818
2819 ASSERT_EQUALS(1, member1Status["_id"].Int());
2820 ASSERT_EQUALS(0, member1Status["health"].Double());
2821 }
2822
TEST_F(HeartbeatResponseTestTwoRetries,ReconfigWhenHeartbeatResponseContainsAConfig)2823 TEST_F(HeartbeatResponseTestTwoRetries, ReconfigWhenHeartbeatResponseContainsAConfig) {
2824 // Confirm that action responses can come back from retries; in this, expect a Reconfig
2825 // action.
2826 ReplSetConfig newConfig;
2827 ASSERT_OK(newConfig.initialize(BSON("_id"
2828 << "rs0"
2829 << "version"
2830 << 7
2831 << "members"
2832 << BSON_ARRAY(BSON("_id" << 0 << "host"
2833 << "host1:27017")
2834 << BSON("_id" << 1 << "host"
2835 << "host2:27017")
2836 << BSON("_id" << 2 << "host"
2837 << "host3:27017")
2838 << BSON("_id" << 3 << "host"
2839 << "host4:27017"))
2840 << "settings"
2841 << BSON("heartbeatTimeoutSecs" << 5))));
2842 ASSERT_OK(newConfig.validate());
2843
2844 ReplSetHeartbeatResponse reconfigResponse;
2845 reconfigResponse.noteReplSet();
2846 reconfigResponse.setSetName("rs0");
2847 reconfigResponse.setState(MemberState::RS_SECONDARY);
2848 reconfigResponse.setElectable(true);
2849 reconfigResponse.setConfigVersion(1);
2850 reconfigResponse.setConfig(newConfig);
2851 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2852 firstRequestDate() + Milliseconds(4500), // Time is left.
2853 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2854 target(),
2855 StatusWith<ReplSetHeartbeatResponse>(reconfigResponse));
2856 ASSERT_EQUALS(HeartbeatResponseAction::Reconfig, action.getAction());
2857 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2858 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
2859 }
2860
TEST_F(HeartbeatResponseTestTwoRetries,StepDownRemotePrimaryWhenWeWereElectedMoreRecently)2861 TEST_F(HeartbeatResponseTestTwoRetries, StepDownRemotePrimaryWhenWeWereElectedMoreRecently) {
2862 // Confirm that action responses can come back from retries; in this, expect a
2863 // StepDownRemotePrimary action.
2864
2865 // make self primary
2866 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
2867 makeSelfPrimary(Timestamp(5, 0));
2868 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
2869
2870 ReplSetHeartbeatResponse electedMoreRecentlyResponse;
2871 electedMoreRecentlyResponse.noteReplSet();
2872 electedMoreRecentlyResponse.setSetName("rs0");
2873 electedMoreRecentlyResponse.setState(MemberState::RS_PRIMARY);
2874 electedMoreRecentlyResponse.setElectable(true);
2875 electedMoreRecentlyResponse.setElectionTime(Timestamp(3, 0));
2876 electedMoreRecentlyResponse.setConfigVersion(5);
2877 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2878 firstRequestDate() + Milliseconds(5000), // Time is left.
2879 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2880 target(),
2881 StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse));
2882 ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, action.getAction());
2883 ASSERT_EQUALS(1, action.getPrimaryConfigIndex());
2884 ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate());
2885 }
2886
TEST_F(HeartbeatResponseTestTwoRetries,StepDownSelfWhenRemoteNodeWasElectedMoreRecently)2887 TEST_F(HeartbeatResponseTestTwoRetries, StepDownSelfWhenRemoteNodeWasElectedMoreRecently) {
2888 // Confirm that action responses can come back from retries; in this, expect a StepDownSelf
2889 // action.
2890
2891 // acknowledge the other member so that we see a majority
2892 HeartbeatResponseAction action =
2893 receiveDownHeartbeat(HostAndPort("host3"), "rs0", OpTime(Timestamp(100, 0), 0));
2894 ASSERT_NO_ACTION(action.getAction());
2895
2896 // make us PRIMARY
2897 makeSelfPrimary();
2898
2899 ReplSetHeartbeatResponse electedMoreRecentlyResponse;
2900 electedMoreRecentlyResponse.noteReplSet();
2901 electedMoreRecentlyResponse.setSetName("rs0");
2902 electedMoreRecentlyResponse.setState(MemberState::RS_PRIMARY);
2903 electedMoreRecentlyResponse.setElectable(false);
2904 electedMoreRecentlyResponse.setElectionTime(Timestamp(10, 0));
2905 electedMoreRecentlyResponse.setConfigVersion(5);
2906 action = getTopoCoord().processHeartbeatResponse(
2907 firstRequestDate() + Milliseconds(5000), // Time is left.
2908 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2909 target(),
2910 StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse));
2911 ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, action.getAction());
2912 ASSERT_EQUALS(0, action.getPrimaryConfigIndex());
2913 ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate());
2914 // Doesn't actually do the stepdown until stepDownIfPending is called
2915 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
2916 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
2917
2918 getTopoCoord().prepareForUnconditionalStepDown();
2919 getTopoCoord().finishUnconditionalStepDown();
2920 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2921 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
2922 }
2923
TEST_F(HeartbeatResponseTestTwoRetries,StartElectionAfterReceivingAHeartbeatWhileNoPrimaryExistsAndWeAreElectable)2924 TEST_F(HeartbeatResponseTestTwoRetries,
2925 StartElectionAfterReceivingAHeartbeatWhileNoPrimaryExistsAndWeAreElectable) {
2926 // Confirm that action responses can come back from retries; in this, expect a StartElection
2927 // action.
2928
2929 // acknowledge the other member so that we see a majority
2930 OpTime election = OpTime(Timestamp(400, 0), 0);
2931 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
2932 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
2933 HeartbeatResponseAction action = receiveUpHeartbeat(
2934 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
2935 ASSERT_NO_ACTION(action.getAction());
2936
2937 // make sure we are electable
2938 setSelfMemberState(MemberState::RS_SECONDARY);
2939 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(election, Date_t());
2940
2941 ReplSetHeartbeatResponse startElectionResponse;
2942 startElectionResponse.noteReplSet();
2943 startElectionResponse.setSetName("rs0");
2944 startElectionResponse.setState(MemberState::RS_SECONDARY);
2945 startElectionResponse.setElectable(true);
2946 startElectionResponse.setConfigVersion(5);
2947 action = getTopoCoord().processHeartbeatResponse(
2948 firstRequestDate() + Milliseconds(5000), // Time is left.
2949 Milliseconds(400), // Spent 0.4 of the 0.5 second in the network.
2950 target(),
2951 StatusWith<ReplSetHeartbeatResponse>(startElectionResponse));
2952 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, action.getAction());
2953 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
2954 ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate());
2955 }
2956
TEST_F(HeartbeatResponseTest,NodeDoesNotRetryHeartbeatIfTheFirstFailureTakesTheFullTime)2957 TEST_F(HeartbeatResponseTest, NodeDoesNotRetryHeartbeatIfTheFirstFailureTakesTheFullTime) {
2958 // Confirm that the topology coordinator does not schedule an immediate heartbeat retry if
2959 // the heartbeat timeout period expired before the initial request completed.
2960
2961 HostAndPort target("host2", 27017);
2962 Date_t firstRequestDate = unittest::assertGet(dateFromISOString("2014-08-29T13:00Z"));
2963
2964 // Initial heartbeat request prepared, at t + 0.
2965 std::pair<ReplSetHeartbeatArgs, Milliseconds> request =
2966 getTopoCoord().prepareHeartbeatRequest(firstRequestDate, "rs0", target);
2967 // 5 seconds to successfully complete the heartbeat before the timeout expires.
2968 ASSERT_EQUALS(5000, durationCount<Milliseconds>(request.second));
2969
2970 // Initial heartbeat request fails at t + 5000ms
2971 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2972 firstRequestDate + Milliseconds(5000), // Entire heartbeat period elapsed;
2973 // no retry allowed.
2974 Milliseconds(4990), // Spent 4.99 of the 5 seconds in the network.
2975 target,
2976 StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"));
2977
2978 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
2979 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2980 // Because the heartbeat timed out, we'll retry in 2 seconds.
2981 ASSERT_EQUALS(firstRequestDate + Milliseconds(7000), action.getNextHeartbeatStartDate());
2982 }
2983
TEST_F(HeartbeatResponseTestOneRetry,NodeDoesNotRetryHeartbeatIfTheFirstAndSecondFailuresExhaustTheFullTime)2984 TEST_F(HeartbeatResponseTestOneRetry,
2985 NodeDoesNotRetryHeartbeatIfTheFirstAndSecondFailuresExhaustTheFullTime) {
2986 // Confirm that the topology coordinator does not schedule a second heartbeat retry if
2987 // the heartbeat timeout period expired before the first retry completed.
2988 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
2989 firstRequestDate() + Milliseconds(5010), // Entire heartbeat period elapsed;
2990 // no retry allowed.
2991 Milliseconds(1000), // Spent 1 of the 1.01 seconds in the network.
2992 target(),
2993 StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"));
2994
2995 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
2996 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
2997 // Because the heartbeat timed out, we'll retry in 2 seconds.
2998 ASSERT_EQUALS(firstRequestDate() + Milliseconds(7010), action.getNextHeartbeatStartDate());
2999 }
3000
TEST_F(HeartbeatResponseTestTwoRetries,NodeDoesNotMarkANodeAsDownAfterThreeNonConsecutiveFailedHeartbeats)3001 TEST_F(HeartbeatResponseTestTwoRetries,
3002 NodeDoesNotMarkANodeAsDownAfterThreeNonConsecutiveFailedHeartbeats) {
3003 // Confirm that the topology coordinator does not mark a node down on three
3004 // nonconsecutive heartbeat failures.
3005 ReplSetHeartbeatResponse response;
3006 response.noteReplSet();
3007 response.setSetName("rs0");
3008 response.setState(MemberState::RS_SECONDARY);
3009 response.setElectable(true);
3010 response.setConfigVersion(5);
3011
3012 // successful response (third response due to the two failures in setUp())
3013 HeartbeatResponseAction action =
3014 getTopoCoord().processHeartbeatResponse(firstRequestDate() + Milliseconds(4500),
3015 Milliseconds(400),
3016 target(),
3017 StatusWith<ReplSetHeartbeatResponse>(response));
3018
3019 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
3020 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3021 // Because the heartbeat succeeded, we'll retry in 2 seconds.
3022 ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate());
3023
3024 // request next heartbeat
3025 getTopoCoord().prepareHeartbeatRequest(
3026 firstRequestDate() + Milliseconds(6500), "rs0", target());
3027 // third failed response
3028 action = getTopoCoord().processHeartbeatResponse(
3029 firstRequestDate() + Milliseconds(7100),
3030 Milliseconds(400),
3031 target(),
3032 StatusWith<ReplSetHeartbeatResponse>(Status{ErrorCodes::HostUnreachable, ""}));
3033
3034 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction());
3035 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3036
3037 // Ensure a third nonconsecutive heartbeat failure did not cause the node to be marked down
3038 BSONObjBuilder statusBuilder;
3039 Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result");
3040 getTopoCoord().prepareStatusResponse(
3041 TopologyCoordinator::ReplSetStatusArgs{
3042 firstRequestDate() + Milliseconds(7000), 600, OpTime(), BSONObj()},
3043 &statusBuilder,
3044 &resultStatus);
3045 ASSERT_OK(resultStatus);
3046 BSONObj rsStatus = statusBuilder.obj();
3047 std::vector<BSONElement> memberArray = rsStatus["members"].Array();
3048 BSONObj member1Status = memberArray[1].Obj();
3049
3050 ASSERT_EQUALS(1, member1Status["_id"].Int());
3051 ASSERT_EQUALS(1, member1Status["health"].Double());
3052 }
3053
TEST_F(HeartbeatResponseTest,UpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfANewPrimary)3054 TEST_F(HeartbeatResponseTest, UpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfANewPrimary) {
3055 OpTime election = OpTime(Timestamp(5, 0), 0);
3056 OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3057
3058 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3059 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3060 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3061 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3062 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3063 ASSERT_NO_ACTION(nextAction.getAction());
3064 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3065 }
3066
TEST_F(HeartbeatResponseTest,NodeDoesNotUpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfAnOlderPrimary)3067 TEST_F(HeartbeatResponseTest,
3068 NodeDoesNotUpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfAnOlderPrimary) {
3069 OpTime election = OpTime(Timestamp(5, 0), 0);
3070 OpTime election2 = OpTime(Timestamp(4, 0), 0);
3071 OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3072
3073 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3074 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3075 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3076 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3077 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3078 ASSERT_NO_ACTION(nextAction.getAction());
3079
3080 nextAction = receiveUpHeartbeat(
3081 HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election);
3082 // second primary does not change primary index
3083 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3084 ASSERT_NO_ACTION(nextAction.getAction());
3085 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3086 }
3087
TEST_F(HeartbeatResponseTest,NodeDoesNotUpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfANewerPrimary)3088 TEST_F(HeartbeatResponseTest,
3089 NodeDoesNotUpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfANewerPrimary) {
3090 OpTime election = OpTime(Timestamp(4, 0), 0);
3091 OpTime election2 = OpTime(Timestamp(5, 0), 0);
3092 OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3093
3094 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3095 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3096 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3097 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3098 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3099 ASSERT_NO_ACTION(nextAction.getAction());
3100
3101 nextAction = receiveUpHeartbeat(
3102 HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election);
3103 // second primary does not change primary index
3104 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3105 ASSERT_NO_ACTION(nextAction.getAction());
3106 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3107 }
3108
TEST_F(HeartbeatResponseTest,StepDownRemotePrimaryWhenAHeartbeatMakesNodeAwareOfAnOlderPrimaryWhilePrimary)3109 TEST_F(HeartbeatResponseTest,
3110 StepDownRemotePrimaryWhenAHeartbeatMakesNodeAwareOfAnOlderPrimaryWhilePrimary) {
3111 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3112 makeSelfPrimary(Timestamp(5, 0));
3113
3114 OpTime election = OpTime(Timestamp(4, 0), 0);
3115 OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3116
3117 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3118 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3119 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3120 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3121 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3122 ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, nextAction.getAction());
3123 ASSERT_EQUALS(1, nextAction.getPrimaryConfigIndex());
3124 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3125 }
3126
3127 // TODO(dannenberg) figure out what this is about...
TEST_F(HeartbeatResponseTest,UpdateHeartbeatDataStepDownPrimaryForHighPriorityFreshNode)3128 TEST_F(HeartbeatResponseTest, UpdateHeartbeatDataStepDownPrimaryForHighPriorityFreshNode) {
3129 // In this test, the Topology coordinator sees a PRIMARY ("host2") and then sees a higher
3130 // priority and similarly fresh node ("host3"). However, since the coordinator's node
3131 // (host1) is not the higher priority node, it takes no action.
3132 updateConfig(BSON("_id"
3133 << "rs0"
3134 << "version"
3135 << 6
3136 << "members"
3137 << BSON_ARRAY(BSON("_id" << 0 << "host"
3138 << "host1:27017")
3139 << BSON("_id" << 1 << "host"
3140 << "host2:27017")
3141 << BSON("_id" << 2 << "host"
3142 << "host3:27017"
3143 << "priority"
3144 << 3))
3145 << "settings"
3146 << BSON("heartbeatTimeoutSecs" << 5)),
3147 0);
3148 setSelfMemberState(MemberState::RS_SECONDARY);
3149
3150 OpTime election = OpTime();
3151 OpTime lastOpTimeApplied = OpTime(Timestamp(13, 0), 0);
3152 OpTime slightlyLessFreshLastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3153
3154 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3155 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3156 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3157 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3158
3159 nextAction = receiveUpHeartbeat(HostAndPort("host3"),
3160 "rs0",
3161 MemberState::RS_SECONDARY,
3162 election,
3163 slightlyLessFreshLastOpTimeApplied);
3164 ASSERT_EQUALS(HeartbeatResponseAction::NoAction, nextAction.getAction());
3165 }
3166
TEST_F(HeartbeatResponseTest,StepDownSelfButRemainElectableWhenHeartbeatResponseContainsAnEquallyFreshHigherPriorityNode)3167 TEST_F(
3168 HeartbeatResponseTest,
3169 StepDownSelfButRemainElectableWhenHeartbeatResponseContainsAnEquallyFreshHigherPriorityNode) {
3170 // In this test, the Topology coordinator becomes PRIMARY and then sees a higher priority
3171 // and equally fresh node ("host3"). As a result it responds with a StepDownSelf action.
3172 //
3173 // Despite having stepped down, we should remain electable, in order to dissuade lower
3174 // priority nodes from standing for election.
3175 updateConfig(BSON("_id"
3176 << "rs0"
3177 << "version"
3178 << 6
3179 << "members"
3180 << BSON_ARRAY(BSON("_id" << 0 << "host"
3181 << "host1:27017")
3182 << BSON("_id" << 1 << "host"
3183 << "host2:27017")
3184 << BSON("_id" << 2 << "host"
3185 << "host3:27017"
3186 << "priority"
3187 << 3))
3188 << "settings"
3189 << BSON("heartbeatTimeoutSecs" << 5)),
3190 0);
3191 OpTime election = OpTime(Timestamp(1000, 0), 0);
3192
3193 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
3194 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3195 makeSelfPrimary(election.getTimestamp());
3196 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3197
3198 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3199 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3200 ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, nextAction.getAction());
3201 ASSERT_EQUALS(0, nextAction.getPrimaryConfigIndex());
3202
3203 // Process a heartbeat response to confirm that this node, which is no longer primary,
3204 // still tells other nodes that it is electable. This will stop lower priority nodes
3205 // from standing for election.
3206 ReplSetHeartbeatArgs hbArgs;
3207 hbArgs.setSetName("rs0");
3208 hbArgs.setProtocolVersion(1);
3209 hbArgs.setConfigVersion(6);
3210 hbArgs.setSenderId(1);
3211 hbArgs.setSenderHost(HostAndPort("host3", 27017));
3212 ReplSetHeartbeatResponse hbResp;
3213 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(election, Date_t());
3214 getTopoCoord().getMyMemberData()->setLastDurableOpTime(election, Date_t());
3215 ASSERT_OK(getTopoCoord().prepareHeartbeatResponse(now(), hbArgs, "rs0", &hbResp));
3216 ASSERT(!hbResp.hasIsElectable() || hbResp.isElectable()) << hbResp.toString();
3217 }
3218
TEST_F(HeartbeatResponseTest,NodeDoesNotStepDownSelfWhenHeartbeatResponseContainsALessFreshHigherPriorityNode)3219 TEST_F(HeartbeatResponseTest,
3220 NodeDoesNotStepDownSelfWhenHeartbeatResponseContainsALessFreshHigherPriorityNode) {
3221 // In this test, the Topology coordinator becomes PRIMARY and then sees a higher priority
3222 // and stale node ("host3"). As a result it responds with NoAction.
3223 updateConfig(BSON("_id"
3224 << "rs0"
3225 << "version"
3226 << 6
3227 << "members"
3228 << BSON_ARRAY(BSON("_id" << 0 << "host"
3229 << "host1:27017")
3230 << BSON("_id" << 1 << "host"
3231 << "host2:27017")
3232 << BSON("_id" << 2 << "host"
3233 << "host3:27017"
3234 << "priority"
3235 << 3))
3236 << "settings"
3237 << BSON("heartbeatTimeoutSecs" << 5)),
3238 0);
3239 OpTime election = OpTime(Timestamp(1000, 0), 0);
3240 OpTime staleTime = OpTime();
3241
3242 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3243 makeSelfPrimary(election.getTimestamp());
3244 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3245
3246 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(election, Date_t());
3247 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3248 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, staleTime);
3249 ASSERT_NO_ACTION(nextAction.getAction());
3250 }
3251
TEST_F(HeartbeatResponseTest,NodeDoesNotStepDownRemoteWhenHeartbeatResponseContainsALessFreshHigherPriorityNode)3252 TEST_F(HeartbeatResponseTest,
3253 NodeDoesNotStepDownRemoteWhenHeartbeatResponseContainsALessFreshHigherPriorityNode) {
3254 // In this test, the Topology coordinator sees a PRIMARY ("host2") and then sees a higher
3255 // priority and stale node ("host3"). As a result it responds with NoAction.
3256 updateConfig(BSON("_id"
3257 << "rs0"
3258 << "version"
3259 << 6
3260 << "members"
3261 << BSON_ARRAY(BSON("_id" << 0 << "host"
3262 << "host1:27017")
3263 << BSON("_id" << 1 << "host"
3264 << "host2:27017")
3265 << BSON("_id" << 2 << "host"
3266 << "host3:27017"
3267 << "priority"
3268 << 3))
3269 << "settings"
3270 << BSON("heartbeatTimeoutSecs" << 5)),
3271 0);
3272 setSelfMemberState(MemberState::RS_SECONDARY);
3273
3274 OpTime election = OpTime(Timestamp(1000, 0), 0);
3275 OpTime stale = OpTime();
3276
3277 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3278 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(election, Date_t());
3279 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3280 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3281 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3282
3283 nextAction =
3284 receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, stale);
3285 ASSERT_NO_ACTION(nextAction.getAction());
3286 }
3287
TEST_F(HeartbeatResponseTest,StepDownSelfWhenRemoteNodeWasElectedMoreRecently)3288 TEST_F(HeartbeatResponseTest, StepDownSelfWhenRemoteNodeWasElectedMoreRecently) {
3289 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3290 makeSelfPrimary(Timestamp(2, 0));
3291
3292 OpTime election = OpTime(Timestamp(4, 0), 0);
3293 OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0);
3294
3295 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3296 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3297 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3298 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3299 ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, nextAction.getAction());
3300 ASSERT_EQUALS(0, nextAction.getPrimaryConfigIndex());
3301 // Doesn't actually do the stepdown until stepDownIfPending is called
3302 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3303 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3304
3305 getTopoCoord().prepareForUnconditionalStepDown();
3306 getTopoCoord().finishUnconditionalStepDown();
3307 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3308 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3309 }
3310
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeCannotSeeMajority)3311 TEST_F(HeartbeatResponseTest,
3312 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeCannotSeeMajority) {
3313 setSelfMemberState(MemberState::RS_SECONDARY);
3314
3315 OpTime election = OpTime(Timestamp(400, 0), 0);
3316 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3317
3318 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3319 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3320 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3321 ASSERT_NO_ACTION(nextAction.getAction());
3322 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3323
3324 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3325 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3326 ASSERT_NO_ACTION(nextAction.getAction());
3327 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3328 }
3329
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeHaveZeroPriority)3330 TEST_F(HeartbeatResponseTest,
3331 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeHaveZeroPriority) {
3332 setSelfMemberState(MemberState::RS_SECONDARY);
3333
3334 updateConfig(BSON("_id"
3335 << "rs0"
3336 << "version"
3337 << 5
3338 << "members"
3339 << BSON_ARRAY(BSON("_id" << 0 << "host"
3340 << "host1:27017"
3341 << "priority"
3342 << 0)
3343 << BSON("_id" << 1 << "host"
3344 << "host2:27017")
3345 << BSON("_id" << 2 << "host"
3346 << "host3:27017"))),
3347 0);
3348
3349 OpTime election = OpTime(Timestamp(400, 0), 0);
3350 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3351
3352 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3353 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3354 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3355 ASSERT_NO_ACTION(nextAction.getAction());
3356 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3357
3358 nextAction = receiveUpHeartbeat(
3359 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3360 ASSERT_NO_ACTION(nextAction.getAction());
3361 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3362
3363 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3364 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3365 ASSERT_NO_ACTION(nextAction.getAction());
3366 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3367 }
3368
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreInStartup)3369 TEST_F(HeartbeatResponseTest,
3370 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreInStartup) {
3371 setSelfMemberState(MemberState::RS_STARTUP);
3372
3373 OpTime election = OpTime(Timestamp(400, 0), 0);
3374 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3375
3376 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3377 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3378 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3379 ASSERT_NO_ACTION(nextAction.getAction());
3380 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3381
3382 nextAction = receiveUpHeartbeat(
3383 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3384 ASSERT_NO_ACTION(nextAction.getAction());
3385
3386 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3387 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3388 ASSERT_NO_ACTION(nextAction.getAction());
3389 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3390 }
3391
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreInRecovering)3392 TEST_F(HeartbeatResponseTest,
3393 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreInRecovering) {
3394 setSelfMemberState(MemberState::RS_RECOVERING);
3395
3396 OpTime election = OpTime(Timestamp(400, 0), 0);
3397 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3398
3399 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3400 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3401 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3402 ASSERT_NO_ACTION(nextAction.getAction());
3403 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3404
3405 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3406 ASSERT_NO_ACTION(nextAction.getAction());
3407 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3408 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3409 }
3410
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeHaveStepdownWait)3411 TEST_F(HeartbeatResponseTest,
3412 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeHaveStepdownWait) {
3413 setSelfMemberState(MemberState::RS_SECONDARY);
3414
3415 OpTime election = OpTime(Timestamp(400, 0), 0);
3416 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3417
3418 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3419 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3420 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3421 ASSERT_NO_ACTION(nextAction.getAction());
3422 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3423
3424 nextAction = receiveUpHeartbeat(
3425 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3426 ASSERT_NO_ACTION(nextAction.getAction());
3427
3428 // freeze node to set stepdown wait
3429 BSONObjBuilder response;
3430 ASSERT_EQUALS(
3431 TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
3432 unittest::assertGet(getTopoCoord().prepareFreezeResponse(now()++, 20, &response)));
3433
3434 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3435 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3436 ASSERT_NO_ACTION(nextAction.getAction());
3437 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3438 }
3439
TEST_F(HeartbeatResponseTest,NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreAnArbiter)3440 TEST_F(HeartbeatResponseTest,
3441 NodeDoesNotStandForElectionWhenPrimaryIsMarkedDownViaHeartbeatButWeAreAnArbiter) {
3442 updateConfig(BSON("_id"
3443 << "rs0"
3444 << "version"
3445 << 5
3446 << "members"
3447 << BSON_ARRAY(BSON("_id" << 0 << "host"
3448 << "host1:27017"
3449 << "arbiterOnly"
3450 << true)
3451 << BSON("_id" << 1 << "host"
3452 << "host2:27017")
3453 << BSON("_id" << 2 << "host"
3454 << "host3:27017"))),
3455 0);
3456
3457 OpTime election = OpTime(Timestamp(400, 0), 0);
3458 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3459
3460 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3461 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3462 ASSERT_NO_ACTION(nextAction.getAction());
3463 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3464
3465 nextAction = receiveUpHeartbeat(
3466 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3467 ASSERT_NO_ACTION(nextAction.getAction());
3468 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3469
3470 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3471 ASSERT_NO_ACTION(nextAction.getAction());
3472 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3473 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3474 }
3475
TEST_F(HeartbeatResponseTest,StartElectionWhenPrimaryIsMarkedDownAndWeAreElectable)3476 TEST_F(HeartbeatResponseTest, StartElectionWhenPrimaryIsMarkedDownAndWeAreElectable) {
3477 setSelfMemberState(MemberState::RS_SECONDARY);
3478
3479 OpTime election = OpTime(Timestamp(400, 0), 0);
3480 OpTime lastOpTimeApplied = OpTime(Timestamp(399, 0), 0);
3481
3482 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3483 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3484 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3485 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3486 ASSERT_NO_ACTION(nextAction.getAction());
3487 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3488
3489 nextAction = receiveUpHeartbeat(
3490 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election);
3491 ASSERT_NO_ACTION(nextAction.getAction());
3492
3493 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3494 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3495 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3496 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3497 }
3498
TEST_F(HeartbeatResponseTest,NodeDoesNotStartElectionWhileAlreadyCandidate)3499 TEST_F(HeartbeatResponseTest, NodeDoesNotStartElectionWhileAlreadyCandidate) {
3500 // In this test, the TopologyCoordinator goes through the steps of a successful election,
3501 // during which it receives a heartbeat that would normally trigger it to become a candidate
3502 // and respond with a StartElection HeartbeatResponseAction. However, since it is already in
3503 // candidate state, it responds with a NoAction HeartbeatResponseAction. Then finishes by
3504 // being winning the election.
3505
3506 // 1. All nodes heartbeat to indicate that they are up and that "host2" is PRIMARY.
3507 // 2. "host2" goes down, triggering an election.
3508 // 3. "host2" comes back, which would normally trigger election, but since the
3509 // TopologyCoordinator is already in candidate mode, does not.
3510 // 4. TopologyCoordinator concludes its freshness round successfully and wins the election.
3511
3512 setSelfMemberState(MemberState::RS_SECONDARY);
3513 now() += Seconds(30); // we need to be more than LastVote::leaseTime from the start of time or
3514 // else some Date_t math goes horribly awry
3515
3516 OpTime election = OpTime();
3517 OpTime lastOpTimeApplied = OpTime(Timestamp(130, 0), 0);
3518 OID round = OID::gen();
3519
3520 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3521 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3522 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3523 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3524 ASSERT_NO_ACTION(nextAction.getAction());
3525 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3526
3527 nextAction = receiveUpHeartbeat(
3528 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3529 ASSERT_NO_ACTION(nextAction.getAction());
3530
3531 // candidate time!
3532 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3533 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3534 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3535 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3536
3537 // see the downed node as SECONDARY and decide to take no action, but are still a candidate
3538 nextAction = receiveUpHeartbeat(
3539 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3540 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3541
3542 // normally this would trigger StartElection, but we are already a candidate
3543 ASSERT_NO_ACTION(nextAction.getAction());
3544 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3545
3546 // now voteForSelf as though we received all our fresh responses
3547 ASSERT_TRUE(getTopoCoord().voteForMyself(now()++));
3548
3549 // now win election and ensure _electionId and _electionTime are set properly
3550 getTopoCoord().processWinElection(round, election.getTimestamp());
3551 ASSERT_EQUALS(round, getTopoCoord().getElectionId());
3552 ASSERT_EQUALS(election.getTimestamp(), getTopoCoord().getElectionTime());
3553 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3554 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3555 }
3556
TEST_F(HeartbeatResponseTest,LoseElectionWhenVotingForAnotherNodeWhileRunningTheFreshnessChecker)3557 TEST_F(HeartbeatResponseTest, LoseElectionWhenVotingForAnotherNodeWhileRunningTheFreshnessChecker) {
3558 // In this test, the TopologyCoordinator goes through the steps of an election. However,
3559 // before its freshness round ends, it receives a fresh command followed by an elect command
3560 // from another node, both of which it responds positively to. The TopologyCoordinator's
3561 // freshness round then concludes successfully, but it fails to vote for itself, since it
3562 // recently voted for another node.
3563
3564 // 1. All nodes heartbeat to indicate that they are up and that "host2" is PRIMARY.
3565 // 2. "host2" goes down, triggering an election.
3566 // 3. "host3" sends a fresh command, which the TopologyCoordinator responds to positively.
3567 // 4. "host3" sends an elect command, which the TopologyCoordinator responds to positively.
3568 // 5. The TopologyCoordinator's concludes its freshness round successfully.
3569 // 6. The TopologyCoordinator loses the election.
3570
3571 setSelfMemberState(MemberState::RS_SECONDARY);
3572 now() += Seconds(30); // we need to be more than LastVote::leaseTime from the start of time or
3573 // else some Date_t math goes horribly awry
3574
3575 OpTime election = OpTime();
3576 OpTime lastOpTimeApplied = OpTime(Timestamp(100, 0), 0);
3577 OpTime fresherOpApplied = OpTime(Timestamp(200, 0), 0);
3578
3579 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3580 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3581 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3582 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3583 ASSERT_NO_ACTION(nextAction.getAction());
3584 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3585
3586 nextAction = receiveUpHeartbeat(
3587 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3588 ASSERT_NO_ACTION(nextAction.getAction());
3589
3590 // candidate time!
3591 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3592 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3593 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3594 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3595
3596 Timestamp originalElectionTime = getTopoCoord().getElectionTime();
3597 OID originalElectionId = getTopoCoord().getElectionId();
3598 // prepare an incoming fresh command
3599 ReplicationCoordinator::ReplSetFreshArgs freshArgs;
3600 freshArgs.setName = "rs0";
3601 freshArgs.cfgver = 5;
3602 freshArgs.id = 2;
3603 freshArgs.who = HostAndPort("host3");
3604 freshArgs.opTime = fresherOpApplied.getTimestamp();
3605
3606 BSONObjBuilder freshResponseBuilder;
3607 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3608 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3609 getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result);
3610 BSONObj response = freshResponseBuilder.obj();
3611 ASSERT_OK(result);
3612 ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
3613 ASSERT_FALSE(response["fresher"].trueValue());
3614 ASSERT_FALSE(response["veto"].trueValue());
3615 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3616 // make sure incoming fresh commands do not change electionTime and electionId
3617 ASSERT_EQUALS(originalElectionTime, getTopoCoord().getElectionTime());
3618 ASSERT_EQUALS(originalElectionId, getTopoCoord().getElectionId());
3619
3620 // an elect command comes in
3621 ReplicationCoordinator::ReplSetElectArgs electArgs;
3622 OID round = OID::gen();
3623 electArgs.set = "rs0";
3624 electArgs.round = round;
3625 electArgs.cfgver = 5;
3626 electArgs.whoid = 2;
3627
3628 BSONObjBuilder electResponseBuilder;
3629 result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3630 startCapturingLogMessages();
3631 getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result);
3632 stopCapturingLogMessages();
3633 response = electResponseBuilder.obj();
3634 ASSERT_OK(result);
3635 ASSERT_EQUALS(1, response["vote"].Int());
3636 ASSERT_EQUALS(round, response["round"].OID());
3637 ASSERT_EQUALS(1, countLogLinesContaining("voting yea for host3:27017 (2)"));
3638 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3639 // make sure incoming elect commands do not change electionTime and electionId
3640 ASSERT_EQUALS(originalElectionTime, getTopoCoord().getElectionTime());
3641 ASSERT_EQUALS(originalElectionId, getTopoCoord().getElectionId());
3642
3643 // now voteForSelf as though we received all our fresh responses
3644 ASSERT_FALSE(getTopoCoord().voteForMyself(now()++));
3645
3646 // receive a heartbeat indicating the other node was elected
3647 nextAction = receiveUpHeartbeat(
3648 HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3649 ASSERT_NO_ACTION(nextAction.getAction());
3650 ASSERT_EQUALS(2, getCurrentPrimaryIndex());
3651 // make sure seeing a new primary does not change electionTime and electionId
3652 ASSERT_EQUALS(originalElectionTime, getTopoCoord().getElectionTime());
3653 ASSERT_EQUALS(originalElectionId, getTopoCoord().getElectionId());
3654
3655 // now lose election and ensure _electionTime and _electionId are 0'd out
3656 getTopoCoord().processLoseElection();
3657 ASSERT_EQUALS(OID(), getTopoCoord().getElectionId());
3658 ASSERT_EQUALS(Timestamp(), getTopoCoord().getElectionTime());
3659 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3660 ASSERT_EQUALS(2, getCurrentPrimaryIndex());
3661 }
3662
TEST_F(HeartbeatResponseTest,NodeDoesNotLoseElectionWhenRespondingToAFreshnessCheckWhileRunningItsOwnFreshnessCheck)3663 TEST_F(HeartbeatResponseTest,
3664 NodeDoesNotLoseElectionWhenRespondingToAFreshnessCheckWhileRunningItsOwnFreshnessCheck) {
3665 // In this test, the TopologyCoordinator goes through the steps of an election. However,
3666 // before its freshness round ends, the TopologyCoordinator receives a fresh command from
3667 // another node, which it responds positively to. Its freshness then ends successfully and
3668 // it wins the election. The other node's elect command then comes in and is responded to
3669 // negatively, maintaining the TopologyCoordinator's PRIMARY state.
3670
3671 // 1. All nodes heartbeat to indicate that they are up and that "host2" is PRIMARY.
3672 // 2. "host2" goes down, triggering an election.
3673 // 3. "host3" sends a fresh command, which the TopologyCoordinator responds to positively.
3674 // 4. The TopologyCoordinator concludes its freshness round successfully and wins
3675 // the election.
3676 // 5. "host3" sends an elect command, which the TopologyCoordinator responds to negatively.
3677
3678 setSelfMemberState(MemberState::RS_SECONDARY);
3679 now() += Seconds(30); // we need to be more than LastVote::leaseTime from the start of time or
3680 // else some Date_t math goes horribly awry
3681
3682 OpTime election = OpTime();
3683 OpTime lastOpTimeApplied = OpTime(Timestamp(100, 0), 0);
3684 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(200, 0), 0);
3685 OID round = OID::gen();
3686 OID remoteRound = OID::gen();
3687
3688 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3689 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3690 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3691 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3692 ASSERT_NO_ACTION(nextAction.getAction());
3693 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3694
3695 nextAction = receiveUpHeartbeat(
3696 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3697 ASSERT_NO_ACTION(nextAction.getAction());
3698
3699 // candidate time!
3700 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3701 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3702 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3703 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3704
3705 // prepare an incoming fresh command
3706 ReplicationCoordinator::ReplSetFreshArgs freshArgs;
3707 freshArgs.setName = "rs0";
3708 freshArgs.cfgver = 5;
3709 freshArgs.id = 2;
3710 freshArgs.who = HostAndPort("host3");
3711 freshArgs.opTime = fresherLastOpTimeApplied.getTimestamp();
3712
3713 BSONObjBuilder freshResponseBuilder;
3714 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3715 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3716 getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result);
3717 BSONObj response = freshResponseBuilder.obj();
3718 ASSERT_OK(result);
3719 ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
3720 ASSERT_FALSE(response["fresher"].trueValue());
3721 ASSERT_FALSE(response["veto"].trueValue());
3722 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3723
3724 // now voteForSelf as though we received all our fresh responses
3725 ASSERT_TRUE(getTopoCoord().voteForMyself(now()++));
3726 // now win election and ensure _electionId and _electionTime are set properly
3727 getTopoCoord().processWinElection(round, election.getTimestamp());
3728 ASSERT_EQUALS(round, getTopoCoord().getElectionId());
3729 ASSERT_EQUALS(election.getTimestamp(), getTopoCoord().getElectionTime());
3730 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3731 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3732
3733 // an elect command comes in
3734 ReplicationCoordinator::ReplSetElectArgs electArgs;
3735 electArgs.set = "rs0";
3736 electArgs.round = remoteRound;
3737 electArgs.cfgver = 5;
3738 electArgs.whoid = 2;
3739
3740 BSONObjBuilder electResponseBuilder;
3741 result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3742 startCapturingLogMessages();
3743 getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result);
3744 stopCapturingLogMessages();
3745 response = electResponseBuilder.obj();
3746 ASSERT_OK(result);
3747 ASSERT_EQUALS(-10000, response["vote"].Int());
3748 ASSERT_EQUALS(remoteRound, response["round"].OID());
3749 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3750 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3751 }
3752
TEST_F(HeartbeatResponseTest,RespondPositivelyToFreshnessButNegativelyToElectCommandAfterBecomingPrimary)3753 TEST_F(HeartbeatResponseTest,
3754 RespondPositivelyToFreshnessButNegativelyToElectCommandAfterBecomingPrimary) {
3755 // In this test, the TopologyCoordinator goes through the steps of an election. After
3756 // being successfully elected, a fresher node sends a fresh command, which the
3757 // TopologyCoordinator responds positively to. The fresher node then sends an elect command,
3758 // which the Topology coordinator negatively to since the TopologyCoordinator just elected
3759 // itself.
3760
3761 // 1. All nodes heartbeat to indicate that they are up and that "host2" is PRIMARY.
3762 // 2. "host2" goes down, triggering an election.
3763 // 3. The TopologyCoordinator concludes its freshness round successfully and wins
3764 // the election.
3765 // 4. "host3" sends a fresh command, which the TopologyCoordinator responds to positively.
3766 // 5. "host3" sends an elect command, which the TopologyCoordinator responds to negatively.
3767
3768 setSelfMemberState(MemberState::RS_SECONDARY);
3769 now() += Seconds(30); // we need to be more than LastVote::leaseTime from the start of time or
3770 // else some Date_t math goes horribly awry
3771
3772 OpTime election = OpTime();
3773 OpTime lastOpTimeApplied = OpTime(Timestamp(100, 0), 0);
3774 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(200, 0), 0);
3775 OID round = OID::gen();
3776 OID remoteRound = OID::gen();
3777
3778 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3779 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3780 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3781 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
3782 ASSERT_NO_ACTION(nextAction.getAction());
3783 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3784
3785 nextAction = receiveUpHeartbeat(
3786 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3787 ASSERT_NO_ACTION(nextAction.getAction());
3788
3789 // candidate time!
3790 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3791 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3792 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3793 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3794
3795 // now voteForSelf as though we received all our fresh responses
3796 ASSERT_TRUE(getTopoCoord().voteForMyself(now()++));
3797 // now win election
3798 getTopoCoord().processWinElection(round, election.getTimestamp());
3799 ASSERT_EQUALS(0, getTopoCoord().getCurrentPrimaryIndex());
3800 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3801
3802 // prepare an incoming fresh command
3803 ReplicationCoordinator::ReplSetFreshArgs freshArgs;
3804 freshArgs.setName = "rs0";
3805 freshArgs.cfgver = 5;
3806 freshArgs.id = 2;
3807 freshArgs.who = HostAndPort("host3");
3808 freshArgs.opTime = fresherLastOpTimeApplied.getTimestamp();
3809
3810 BSONObjBuilder freshResponseBuilder;
3811 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3812 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3813 getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result);
3814 BSONObj response = freshResponseBuilder.obj();
3815 ASSERT_OK(result);
3816 ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue()));
3817 ASSERT_FALSE(response["fresher"].trueValue());
3818 ASSERT_TRUE(response["veto"].trueValue()) << response["errmsg"];
3819 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3820 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3821
3822 // an elect command comes in
3823 ReplicationCoordinator::ReplSetElectArgs electArgs;
3824 electArgs.set = "rs0";
3825 electArgs.round = remoteRound;
3826 electArgs.cfgver = 5;
3827 electArgs.whoid = 2;
3828
3829 BSONObjBuilder electResponseBuilder;
3830 result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3831 startCapturingLogMessages();
3832 getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result);
3833 stopCapturingLogMessages();
3834 response = electResponseBuilder.obj();
3835 ASSERT_OK(result);
3836 ASSERT_EQUALS(-10000, response["vote"].Int());
3837 ASSERT_EQUALS(remoteRound, response["round"].OID());
3838 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3839 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3840 }
3841
TEST_F(HeartbeatResponseTest,StartElectionIfAMajorityOfVotersIsVisibleEvenThoughATrueMajorityIsNot)3842 TEST_F(HeartbeatResponseTest,
3843 StartElectionIfAMajorityOfVotersIsVisibleEvenThoughATrueMajorityIsNot) {
3844 updateConfig(BSON("_id"
3845 << "rs0"
3846 << "version"
3847 << 5
3848 << "members"
3849 << BSON_ARRAY(BSON("_id" << 0 << "host"
3850 << "host1:27017")
3851 << BSON("_id" << 1 << "host"
3852 << "host2:27017")
3853 << BSON("_id" << 2 << "host"
3854 << "host3:27017"
3855 << "votes"
3856 << 0
3857 << "priority"
3858 << 0)
3859 << BSON("_id" << 3 << "host"
3860 << "host4:27017"
3861 << "votes"
3862 << 0
3863 << "priority"
3864 << 0)
3865 << BSON("_id" << 4 << "host"
3866 << "host5:27017"
3867 << "votes"
3868 << 0
3869 << "priority"
3870 << 0)
3871 << BSON("_id" << 5 << "host"
3872 << "host6:27017"
3873 << "votes"
3874 << 0
3875 << "priority"
3876 << 0)
3877 << BSON("_id" << 6 << "host"
3878 << "host7:27017"))
3879 << "settings"
3880 << BSON("heartbeatTimeoutSecs" << 5)),
3881 0);
3882
3883 setSelfMemberState(MemberState::RS_SECONDARY);
3884
3885 OpTime election = OpTime(Timestamp(400, 0), 0);
3886 OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0);
3887
3888 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3889 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t());
3890 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
3891 HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election);
3892 ASSERT_NO_ACTION(nextAction.getAction());
3893 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
3894
3895 // make sure all non-voting nodes are down, that way we do not have a majority of nodes
3896 // but do have a majority of votes since one of two voting members is up and so are we
3897 nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", lastOpTimeApplied);
3898 ASSERT_NO_ACTION(nextAction.getAction());
3899 nextAction = receiveDownHeartbeat(HostAndPort("host4"), "rs0", lastOpTimeApplied);
3900 ASSERT_NO_ACTION(nextAction.getAction());
3901 nextAction = receiveDownHeartbeat(HostAndPort("host5"), "rs0", lastOpTimeApplied);
3902 ASSERT_NO_ACTION(nextAction.getAction());
3903 nextAction = receiveDownHeartbeat(HostAndPort("host6"), "rs0", lastOpTimeApplied);
3904 ASSERT_NO_ACTION(nextAction.getAction());
3905 nextAction = receiveUpHeartbeat(
3906 HostAndPort("host7"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
3907 ASSERT_NO_ACTION(nextAction.getAction());
3908
3909 nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied);
3910 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3911 ASSERT_EQUALS(HeartbeatResponseAction::StartElection, nextAction.getAction());
3912 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
3913 }
3914
TEST_F(HeartbeatResponseTest,RelinquishPrimaryWhenMajorityOfVotersIsNoLongerVisible)3915 TEST_F(HeartbeatResponseTest, RelinquishPrimaryWhenMajorityOfVotersIsNoLongerVisible) {
3916 // become PRIMARY
3917 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3918 makeSelfPrimary(Timestamp(2, 0));
3919 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3920
3921 // become aware of other nodes
3922 heartbeatFromMember(
3923 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
3924 heartbeatFromMember(
3925 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
3926 heartbeatFromMember(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, OpTime());
3927 heartbeatFromMember(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, OpTime());
3928
3929 // lose that awareness and be sure we are going to stepdown
3930 HeartbeatResponseAction nextAction =
3931 receiveDownHeartbeat(HostAndPort("host2"), "rs0", OpTime(Timestamp(100, 0), 0));
3932 ASSERT_NO_ACTION(nextAction.getAction());
3933 nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", OpTime(Timestamp(100, 0), 0));
3934 ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, nextAction.getAction());
3935 ASSERT_EQUALS(0, nextAction.getPrimaryConfigIndex());
3936 // Doesn't actually do the stepdown until stepDownIfPending is called
3937 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
3938 ASSERT_EQUALS(0, getCurrentPrimaryIndex());
3939
3940 getTopoCoord().prepareForUnconditionalStepDown();
3941 getTopoCoord().finishUnconditionalStepDown();
3942 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
3943 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
3944 }
3945
3946 class PrepareElectResponseTest : public TopoCoordTest {
3947 public:
PrepareElectResponseTest()3948 PrepareElectResponseTest()
3949 : round(OID::gen()), cbData(NULL, executor::TaskExecutor::CallbackHandle(), Status::OK()) {}
3950
setUp()3951 virtual void setUp() {
3952 TopoCoordTest::setUp();
3953 updateConfig(BSON("_id"
3954 << "rs0"
3955 << "version"
3956 << 10
3957 << "members"
3958 << BSON_ARRAY(BSON("_id" << 0 << "host"
3959 << "hself")
3960 << BSON("_id" << 1 << "host"
3961 << "h1")
3962 << BSON("_id" << 2 << "host"
3963 << "h2"
3964 << "priority"
3965 << 10)
3966 << BSON("_id" << 3 << "host"
3967 << "h3"
3968 << "priority"
3969 << 10))),
3970 0);
3971 }
3972
3973 protected:
3974 Date_t now;
3975 OID round;
3976 executor::TaskExecutor::CallbackArgs cbData;
3977 };
3978
TEST_F(PrepareElectResponseTest,RespondNegativelyWhenElectCommandHasTheWrongReplSetName)3979 TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasTheWrongReplSetName) {
3980 // Test with incorrect replset name
3981 ReplicationCoordinator::ReplSetElectArgs args;
3982 args.set = "fakeset";
3983 args.round = round;
3984 args.cfgver = 10;
3985 args.whoid = 1;
3986
3987 BSONObjBuilder responseBuilder;
3988 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
3989 startCapturingLogMessages();
3990 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
3991 stopCapturingLogMessages();
3992 BSONObj response = responseBuilder.obj();
3993 ASSERT_OK(result);
3994 ASSERT_EQUALS(0, response["vote"].Int());
3995 ASSERT_EQUALS(round, response["round"].OID());
3996 ASSERT_EQUALS(1,
3997 countLogLinesContaining(
3998 "received an elect request for 'fakeset' but our set name is 'rs0'"));
3999
4000 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4001 args.set = "rs0";
4002 BSONObjBuilder responseBuilder2;
4003 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4004 BSONObj response2 = responseBuilder2.obj();
4005 ASSERT_EQUALS(1, response2["vote"].Int());
4006 ASSERT_EQUALS(round, response2["round"].OID());
4007 }
4008
TEST_F(PrepareElectResponseTest,RespondNegativelyWhenElectCommandHasANewerConfig)4009 TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasANewerConfig) {
4010 // Test with us having a stale config version
4011 ReplicationCoordinator::ReplSetElectArgs args;
4012 args.set = "rs0";
4013 args.round = round;
4014 args.cfgver = 20;
4015 args.whoid = 1;
4016
4017 BSONObjBuilder responseBuilder;
4018 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4019 startCapturingLogMessages();
4020 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4021 stopCapturingLogMessages();
4022 BSONObj response = responseBuilder.obj();
4023 ASSERT_OK(result);
4024 ASSERT_EQUALS(0, response["vote"].Int());
4025 ASSERT_EQUALS(round, response["round"].OID());
4026 ASSERT_EQUALS(1, countLogLinesContaining("not voting because our config version is stale"));
4027
4028 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4029 args.cfgver = 10;
4030 BSONObjBuilder responseBuilder2;
4031 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4032 BSONObj response2 = responseBuilder2.obj();
4033 ASSERT_EQUALS(1, response2["vote"].Int());
4034 ASSERT_EQUALS(round, response2["round"].OID());
4035 }
4036
TEST_F(PrepareElectResponseTest,RespondWithAVetoWhenElectCommandHasAnOlderConfig)4037 TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasAnOlderConfig) {
4038 // Test with them having a stale config version
4039 ReplicationCoordinator::ReplSetElectArgs args;
4040 args.set = "rs0";
4041 args.round = round;
4042 args.cfgver = 5;
4043 args.whoid = 1;
4044
4045 BSONObjBuilder responseBuilder;
4046 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4047 startCapturingLogMessages();
4048 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4049 stopCapturingLogMessages();
4050 BSONObj response = responseBuilder.obj();
4051 ASSERT_OK(result);
4052 ASSERT_EQUALS(-10000, response["vote"].Int());
4053 ASSERT_EQUALS(round, response["round"].OID());
4054 ASSERT_EQUALS(1, countLogLinesContaining("received stale config version # during election"));
4055
4056 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4057 args.cfgver = 10;
4058 BSONObjBuilder responseBuilder2;
4059 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4060 BSONObj response2 = responseBuilder2.obj();
4061 ASSERT_EQUALS(1, response2["vote"].Int());
4062 ASSERT_EQUALS(round, response2["round"].OID());
4063 }
4064
TEST_F(PrepareElectResponseTest,RespondWithAVetoWhenElectCommandHasANonExistentMember)4065 TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasANonExistentMember) {
4066 // Test with a non-existent node
4067 ReplicationCoordinator::ReplSetElectArgs args;
4068 args.set = "rs0";
4069 args.round = round;
4070 args.cfgver = 10;
4071 args.whoid = 99;
4072
4073 BSONObjBuilder responseBuilder;
4074 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4075 startCapturingLogMessages();
4076 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4077 stopCapturingLogMessages();
4078 BSONObj response = responseBuilder.obj();
4079 ASSERT_OK(result);
4080 ASSERT_EQUALS(-10000, response["vote"].Int());
4081 ASSERT_EQUALS(round, response["round"].OID());
4082 ASSERT_EQUALS(1, countLogLinesContaining("couldn't find member with id 99"));
4083
4084 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4085 args.whoid = 1;
4086 BSONObjBuilder responseBuilder2;
4087 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4088 BSONObj response2 = responseBuilder2.obj();
4089 ASSERT_EQUALS(1, response2["vote"].Int());
4090 ASSERT_EQUALS(round, response2["round"].OID());
4091 }
4092
TEST_F(PrepareElectResponseTest,RespondWithAVetoWhenElectCommandIsReceivedByPrimary)4093 TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedByPrimary) {
4094 // Test when we are already primary
4095 ReplicationCoordinator::ReplSetElectArgs args;
4096 args.set = "rs0";
4097 args.round = round;
4098 args.cfgver = 10;
4099 args.whoid = 1;
4100
4101 getTopoCoord()._setCurrentPrimaryForTest(0);
4102
4103 BSONObjBuilder responseBuilder;
4104 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4105 startCapturingLogMessages();
4106 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4107 stopCapturingLogMessages();
4108 BSONObj response = responseBuilder.obj();
4109 ASSERT_OK(result);
4110 ASSERT_EQUALS(-10000, response["vote"].Int());
4111 ASSERT_EQUALS(round, response["round"].OID());
4112 ASSERT_EQUALS(1, countLogLinesContaining("I am already primary"));
4113
4114 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4115 getTopoCoord()._setCurrentPrimaryForTest(-1);
4116 BSONObjBuilder responseBuilder2;
4117 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4118 BSONObj response2 = responseBuilder2.obj();
4119 ASSERT_EQUALS(1, response2["vote"].Int());
4120 ASSERT_EQUALS(round, response2["round"].OID());
4121 }
4122
TEST_F(PrepareElectResponseTest,RespondWithAVetoWhenElectCommandIsReceivedWhileAPrimaryExists)4123 TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedWhileAPrimaryExists) {
4124 // Test when someone else is already primary
4125 ReplicationCoordinator::ReplSetElectArgs args;
4126 args.set = "rs0";
4127 args.round = round;
4128 args.cfgver = 10;
4129 args.whoid = 1;
4130 getTopoCoord()._setCurrentPrimaryForTest(2);
4131
4132 BSONObjBuilder responseBuilder;
4133 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4134 startCapturingLogMessages();
4135 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4136 stopCapturingLogMessages();
4137 BSONObj response = responseBuilder.obj();
4138 ASSERT_OK(result);
4139 ASSERT_EQUALS(-10000, response["vote"].Int());
4140 ASSERT_EQUALS(round, response["round"].OID());
4141 ASSERT_EQUALS(1, countLogLinesContaining("h2:27017 is already primary"));
4142
4143 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4144 getTopoCoord()._setCurrentPrimaryForTest(-1);
4145 BSONObjBuilder responseBuilder2;
4146 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4147 BSONObj response2 = responseBuilder2.obj();
4148 ASSERT_EQUALS(1, response2["vote"].Int());
4149 ASSERT_EQUALS(round, response2["round"].OID());
4150 }
4151
TEST_F(PrepareElectResponseTest,RespondWithAVetoWhenAHigherPriorityNodeExistsDuringElectCommand)4152 TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenAHigherPriorityNodeExistsDuringElectCommand) {
4153 // Test trying to elect someone who isn't the highest priority node
4154 ReplicationCoordinator::ReplSetElectArgs args;
4155 args.set = "rs0";
4156 args.round = round;
4157 args.cfgver = 10;
4158 args.whoid = 1;
4159
4160 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime());
4161
4162 BSONObjBuilder responseBuilder;
4163 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4164 startCapturingLogMessages();
4165 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4166 stopCapturingLogMessages();
4167 BSONObj response = responseBuilder.obj();
4168 ASSERT_OK(result);
4169 ASSERT_EQUALS(-10000, response["vote"].Int());
4170 ASSERT_EQUALS(round, response["round"].OID());
4171 ASSERT_EQUALS(1, countLogLinesContaining("h1:27017 has lower priority than h3:27017"));
4172
4173 // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would)
4174 args.whoid = 3;
4175 BSONObjBuilder responseBuilder2;
4176 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result);
4177 BSONObj response2 = responseBuilder2.obj();
4178 ASSERT_EQUALS(1, response2["vote"].Int());
4179 ASSERT_EQUALS(round, response2["round"].OID());
4180 }
4181
TEST_F(PrepareElectResponseTest,RespondPositivelyWhenElectCommandComesFromHighestPriorityNode)4182 TEST_F(PrepareElectResponseTest, RespondPositivelyWhenElectCommandComesFromHighestPriorityNode) {
4183 // Test trying to elect someone who isn't the highest priority node, but all higher nodes
4184 // are down
4185 ReplicationCoordinator::ReplSetElectArgs args;
4186 args.set = "rs0";
4187 args.round = round;
4188 args.cfgver = 10;
4189 args.whoid = 1;
4190
4191 receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime());
4192 receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime());
4193
4194 BSONObjBuilder responseBuilder;
4195 Status result = Status::OK();
4196 startCapturingLogMessages();
4197 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result);
4198 stopCapturingLogMessages();
4199 BSONObj response = responseBuilder.obj();
4200 ASSERT_EQUALS(1, response["vote"].Int());
4201 ASSERT_EQUALS(round, response["round"].OID());
4202 }
4203
TEST_F(PrepareElectResponseTest,RespondNegativelyToElectCommandsWhenAPositiveResponseWasGivenInTheVoteLeasePeriod)4204 TEST_F(PrepareElectResponseTest,
4205 RespondNegativelyToElectCommandsWhenAPositiveResponseWasGivenInTheVoteLeasePeriod) {
4206 // Test a valid vote
4207 ReplicationCoordinator::ReplSetElectArgs args;
4208 args.set = "rs0";
4209 args.round = round;
4210 args.cfgver = 10;
4211 args.whoid = 2;
4212 now = Date_t::fromMillisSinceEpoch(100);
4213
4214 BSONObjBuilder responseBuilder1;
4215 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4216 startCapturingLogMessages();
4217 getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder1, &result);
4218 stopCapturingLogMessages();
4219 BSONObj response1 = responseBuilder1.obj();
4220 ASSERT_OK(result);
4221 ASSERT_EQUALS(1, response1["vote"].Int());
4222 ASSERT_EQUALS(round, response1["round"].OID());
4223 ASSERT_EQUALS(1, countLogLinesContaining("voting yea for h2:27017 (2)"));
4224
4225 // Test what would be a valid vote except that we already voted too recently
4226 args.whoid = 3;
4227
4228 BSONObjBuilder responseBuilder2;
4229 startCapturingLogMessages();
4230 getTopoCoord().prepareElectResponse(args, now, &responseBuilder2, &result);
4231 stopCapturingLogMessages();
4232 BSONObj response2 = responseBuilder2.obj();
4233 ASSERT_OK(result);
4234 ASSERT_EQUALS(0, response2["vote"].Int());
4235 ASSERT_EQUALS(round, response2["round"].OID());
4236 ASSERT_EQUALS(1,
4237 countLogLinesContaining("voting no for h3:27017; "
4238 "voted for h2:27017 0 secs ago"));
4239
4240 // Test that after enough time passes the same vote can proceed
4241 now += Seconds(30) + Milliseconds(1); // just over 30 seconds later
4242
4243 BSONObjBuilder responseBuilder3;
4244 startCapturingLogMessages();
4245 getTopoCoord().prepareElectResponse(args, now++, &responseBuilder3, &result);
4246 stopCapturingLogMessages();
4247 BSONObj response3 = responseBuilder3.obj();
4248 ASSERT_OK(result);
4249 ASSERT_EQUALS(1, response3["vote"].Int());
4250 ASSERT_EQUALS(round, response3["round"].OID());
4251 ASSERT_EQUALS(1, countLogLinesContaining("voting yea for h3:27017 (3)"));
4252 }
4253
TEST_F(TopoCoordTest,NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhileRemoved)4254 TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhileRemoved) {
4255 updateConfig(BSON("_id"
4256 << "rs0"
4257 << "version"
4258 << 5
4259 << "members"
4260 << BSON_ARRAY(BSON("_id" << 0 << "host"
4261 << "host1:27017")
4262 << BSON("_id" << 1 << "host"
4263 << "host2:27017"))),
4264 0);
4265 // Reconfig to remove self.
4266 updateConfig(BSON("_id"
4267 << "rs0"
4268 << "version"
4269 << 2
4270 << "members"
4271 << BSON_ARRAY(BSON("_id" << 1 << "host"
4272 << "host2:27017")
4273 << BSON("_id" << 2 << "host"
4274 << "host3:27017"))),
4275 -1);
4276 ASSERT_EQUALS(MemberState::RS_REMOVED, getTopoCoord().getMemberState().s);
4277
4278 ReplicationCoordinator::ReplSetElectArgs args;
4279 BSONObjBuilder response;
4280 Status status = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4281 getTopoCoord().prepareElectResponse(args, now(), &response, &status);
4282 ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status);
4283 ASSERT_EQUALS("Cannot participate in election because not initialized", status.reason());
4284 }
4285
TEST_F(TopoCoordTest,NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhileNotInitialized)4286 TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhileNotInitialized) {
4287 ReplicationCoordinator::ReplSetElectArgs args;
4288 BSONObjBuilder response;
4289 Status status = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
4290 getTopoCoord().prepareElectResponse(args, now(), &response, &status);
4291 ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status);
4292 ASSERT_EQUALS("Cannot participate in election because not initialized", status.reason());
4293 }
4294
4295 class PrepareFreezeResponseTest : public TopoCoordTest {
4296 public:
setUp()4297 virtual void setUp() {
4298 TopoCoordTest::setUp();
4299 updateConfig(BSON("_id"
4300 << "rs0"
4301 << "version"
4302 << 5
4303 << "members"
4304 << BSON_ARRAY(BSON("_id" << 0 << "host"
4305 << "host1:27017")
4306 << BSON("_id" << 1 << "host"
4307 << "host2:27017"))),
4308 0);
4309 }
4310
4311 std::pair<StatusWith<TopologyCoordinator::PrepareFreezeResponseResult>, BSONObj>
prepareFreezeResponse(int duration)4312 prepareFreezeResponse(int duration) {
4313 BSONObjBuilder response;
4314 startCapturingLogMessages();
4315 auto result = getTopoCoord().prepareFreezeResponse(now()++, duration, &response);
4316 stopCapturingLogMessages();
4317 return std::make_pair(result, response.obj());
4318 }
4319 };
4320
TEST_F(PrepareFreezeResponseTest,FreezeForOneSecondWhenToldToFreezeForZeroSeconds)4321 TEST_F(PrepareFreezeResponseTest, FreezeForOneSecondWhenToldToFreezeForZeroSeconds) {
4322 auto result = prepareFreezeResponse(0);
4323 ASSERT_EQUALS(TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4324 unittest::assertGet(result.first));
4325 const auto& response = result.second;
4326 ASSERT_EQUALS("unfreezing", response["info"].String());
4327 ASSERT_EQUALS(1, countLogLinesContaining("'unfreezing'"));
4328 // 1 instead of 0 because it assigns to "now" in this case
4329 ASSERT_EQUALS(1LL, getTopoCoord().getStepDownTime().asInt64());
4330 }
4331
TEST_F(PrepareFreezeResponseTest,LogAMessageAndFreezeForOneSecondWhenToldToFreezeForOneSecond)4332 TEST_F(PrepareFreezeResponseTest, LogAMessageAndFreezeForOneSecondWhenToldToFreezeForOneSecond) {
4333 auto result = prepareFreezeResponse(1);
4334 ASSERT_EQUALS(TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4335 unittest::assertGet(result.first));
4336 const auto& response = result.second;
4337 ASSERT_EQUALS("you really want to freeze for only 1 second?", response["warning"].String());
4338 ASSERT_EQUALS(1, countLogLinesContaining("'freezing' for 1 seconds"));
4339 // 1001 because "now" was incremented once during initialization + 1000 ms wait
4340 ASSERT_EQUALS(1001LL, getTopoCoord().getStepDownTime().asInt64());
4341 }
4342
TEST_F(PrepareFreezeResponseTest,FreezeForTheSpecifiedDurationWhenToldToFreeze)4343 TEST_F(PrepareFreezeResponseTest, FreezeForTheSpecifiedDurationWhenToldToFreeze) {
4344 auto result = prepareFreezeResponse(20);
4345 ASSERT_EQUALS(TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4346 unittest::assertGet(result.first));
4347 const auto& response = result.second;
4348 ASSERT_TRUE(response.isEmpty());
4349 ASSERT_EQUALS(1, countLogLinesContaining("'freezing' for 20 seconds"));
4350 // 20001 because "now" was incremented once during initialization + 20000 ms wait
4351 ASSERT_EQUALS(20001LL, getTopoCoord().getStepDownTime().asInt64());
4352 }
4353
TEST_F(PrepareFreezeResponseTest,FreezeForOneSecondWhenToldToFreezeForZeroSecondsWhilePrimary)4354 TEST_F(PrepareFreezeResponseTest, FreezeForOneSecondWhenToldToFreezeForZeroSecondsWhilePrimary) {
4355 makeSelfPrimary();
4356 auto result = prepareFreezeResponse(0);
4357 ASSERT_EQUALS(ErrorCodes::NotSecondary, result.first);
4358 const auto& response = result.second;
4359 ASSERT_TRUE(response.isEmpty());
4360 ASSERT_EQUALS(1,
4361 countLogLinesContaining(
4362 "cannot freeze node when primary or running for election. state: Primary"));
4363 ASSERT_EQUALS(0LL, getTopoCoord().getStepDownTime().asInt64());
4364 }
4365
TEST_F(PrepareFreezeResponseTest,NodeDoesNotFreezeWhenToldToFreezeForOneSecondWhilePrimary)4366 TEST_F(PrepareFreezeResponseTest, NodeDoesNotFreezeWhenToldToFreezeForOneSecondWhilePrimary) {
4367 makeSelfPrimary();
4368 auto result = prepareFreezeResponse(1);
4369 ASSERT_EQUALS(ErrorCodes::NotSecondary, result.first);
4370 const auto& response = result.second;
4371 ASSERT_TRUE(response.isEmpty());
4372 ASSERT_EQUALS(1,
4373 countLogLinesContaining(
4374 "cannot freeze node when primary or running for election. state: Primary"));
4375 ASSERT_EQUALS(0LL, getTopoCoord().getStepDownTime().asInt64());
4376 }
4377
TEST_F(PrepareFreezeResponseTest,NodeDoesNotFreezeWhenToldToFreezeForSeveralSecondsWhilePrimary)4378 TEST_F(PrepareFreezeResponseTest, NodeDoesNotFreezeWhenToldToFreezeForSeveralSecondsWhilePrimary) {
4379 makeSelfPrimary();
4380 auto result = prepareFreezeResponse(20);
4381 ASSERT_EQUALS(ErrorCodes::NotSecondary, result.first);
4382 const auto& response = result.second;
4383 ASSERT_TRUE(response.isEmpty());
4384 ASSERT_EQUALS(1,
4385 countLogLinesContaining(
4386 "cannot freeze node when primary or running for election. state: Primary"));
4387 ASSERT_EQUALS(0LL, getTopoCoord().getStepDownTime().asInt64());
4388 }
4389
TEST_F(TopoCoordTest,UnfreezeImmediatelyWhenToldToFreezeForZeroSecondsAfterBeingToldToFreezeForLonger)4390 TEST_F(TopoCoordTest,
4391 UnfreezeImmediatelyWhenToldToFreezeForZeroSecondsAfterBeingToldToFreezeForLonger) {
4392 updateConfig(BSON("_id"
4393 << "rs0"
4394 << "version"
4395 << 5
4396 << "members"
4397 << BSON_ARRAY(BSON("_id" << 0 << "host"
4398 << "host1:27017"))),
4399 0);
4400 setSelfMemberState(MemberState::RS_SECONDARY);
4401
4402 BSONObjBuilder response;
4403 ASSERT_EQUALS(
4404 TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4405 unittest::assertGet(getTopoCoord().prepareFreezeResponse(now()++, 20, &response)));
4406 ASSERT(response.obj().isEmpty());
4407 BSONObjBuilder response2;
4408 ASSERT_EQUALS(
4409 TopologyCoordinator::PrepareFreezeResponseResult::kSingleNodeSelfElect,
4410 unittest::assertGet(getTopoCoord().prepareFreezeResponse(now()++, 0, &response2)));
4411 ASSERT_EQUALS("unfreezing", response2.obj()["info"].String());
4412 ASSERT(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4413
4414 // prepareFreezeResult returns error if we are running for election.
4415 auto result = getTopoCoord().prepareFreezeResponse(now()++, 20, &response);
4416 auto status = result.getStatus();
4417 ASSERT_EQUALS(ErrorCodes::NotSecondary, status);
4418 ASSERT_STRING_CONTAINS(
4419 status.reason(),
4420 "cannot freeze node when primary or running for election. state: Running-Election");
4421 }
4422
TEST_F(TopoCoordTest,DoNotBecomeCandidateOnUnfreezingInMaintenanceMode)4423 TEST_F(TopoCoordTest, DoNotBecomeCandidateOnUnfreezingInMaintenanceMode) {
4424 updateConfig(BSON("_id"
4425 << "rs0"
4426 << "version"
4427 << 5
4428 << "members"
4429 << BSON_ARRAY(BSON("_id" << 0 << "host"
4430 << "host1:27017"))),
4431 0);
4432 setSelfMemberState(MemberState::RS_SECONDARY);
4433
4434 BSONObjBuilder response;
4435 ASSERT_EQUALS(
4436 TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4437 unittest::assertGet(getTopoCoord().prepareFreezeResponse(now()++, 20, &response)));
4438 ASSERT(response.obj().isEmpty());
4439 BSONObjBuilder response2;
4440
4441 // We should not transition to Role::kCandidate if we are in maintenance upon unfreezing.
4442 getTopoCoord().adjustMaintenanceCountBy(1);
4443
4444 ASSERT_EQUALS(
4445 TopologyCoordinator::PrepareFreezeResponseResult::kNoAction,
4446 unittest::assertGet(getTopoCoord().prepareFreezeResponse(now()++, 0, &response2)));
4447 ASSERT_EQUALS("unfreezing", response2.obj()["info"].String());
4448 ASSERT(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4449 }
4450
4451 class PrepareHeartbeatResponseTest : public TopoCoordTest {
4452 public:
setUp()4453 virtual void setUp() {
4454 TopoCoordTest::setUp();
4455 updateConfig(BSON("_id"
4456 << "rs0"
4457 << "version"
4458 << 1
4459 << "members"
4460 << BSON_ARRAY(BSON("_id" << 10 << "host"
4461 << "hself")
4462 << BSON("_id" << 20 << "host"
4463 << "h2")
4464 << BSON("_id" << 30 << "host"
4465 << "h3"))),
4466 0);
4467 setSelfMemberState(MemberState::RS_SECONDARY);
4468 }
4469
prepareHeartbeatResponse(const ReplSetHeartbeatArgs & args,OpTime lastOpApplied,ReplSetHeartbeatResponse * response,Status * result)4470 void prepareHeartbeatResponse(const ReplSetHeartbeatArgs& args,
4471 OpTime lastOpApplied,
4472 ReplSetHeartbeatResponse* response,
4473 Status* result) {
4474 getTopoCoord().getMyMemberData()->setLastAppliedOpTime(lastOpApplied, Date_t());
4475 getTopoCoord().getMyMemberData()->setLastDurableOpTime(lastOpApplied, Date_t());
4476 *result = getTopoCoord().prepareHeartbeatResponse(now()++, args, "rs0", response);
4477 }
4478 };
4479
TEST_F(PrepareHeartbeatResponseTest,NodeReturnsBadValueWhenAHeartbeatRequestHasAnInvalidProtocolVersion)4480 TEST_F(PrepareHeartbeatResponseTest,
4481 NodeReturnsBadValueWhenAHeartbeatRequestHasAnInvalidProtocolVersion) {
4482 // set up args with bad protocol version
4483 ReplSetHeartbeatArgs args;
4484 args.setProtocolVersion(3);
4485 ReplSetHeartbeatResponse response;
4486 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4487
4488 // prepare response and check the results
4489 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4490 ASSERT_EQUALS(ErrorCodes::BadValue, result);
4491 ASSERT_EQUALS("replset: incompatible replset protocol version: 3", result.reason());
4492 ASSERT_EQUALS("", response.getHbMsg());
4493 }
4494
TEST_F(PrepareHeartbeatResponseTest,NodeReturnsBadValueWhenAHeartbeatRequestIsFromSelf)4495 TEST_F(PrepareHeartbeatResponseTest, NodeReturnsBadValueWhenAHeartbeatRequestIsFromSelf) {
4496 // set up args with incorrect replset name
4497 ReplSetHeartbeatArgs args;
4498 args.setProtocolVersion(1);
4499 args.setSetName("rs0");
4500 args.setSenderId(10);
4501 ReplSetHeartbeatResponse response;
4502 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4503 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4504 ASSERT_EQUALS(ErrorCodes::BadValue, result);
4505 ASSERT(result.reason().find("from member with the same member ID as our self"))
4506 << "Actual string was \"" << result.reason() << '"';
4507 ASSERT_EQUALS("", response.getHbMsg());
4508 }
4509
TEST_F(PrepareHeartbeatResponseTest,NodeReturnsInconsistentReplicaSetNamesWhenAHeartbeatRequestHasADifferentReplicaSetName)4510 TEST_F(PrepareHeartbeatResponseTest,
4511 NodeReturnsInconsistentReplicaSetNamesWhenAHeartbeatRequestHasADifferentReplicaSetName) {
4512 // set up args with incorrect replset name
4513 ReplSetHeartbeatArgs args;
4514 args.setProtocolVersion(1);
4515 args.setSetName("rs1");
4516 ReplSetHeartbeatResponse response;
4517 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4518
4519 startCapturingLogMessages();
4520 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4521 stopCapturingLogMessages();
4522 ASSERT_EQUALS(ErrorCodes::InconsistentReplicaSetNames, result);
4523 ASSERT(result.reason().find("repl set names do not match")) << "Actual string was \""
4524 << result.reason() << '"';
4525 ASSERT_EQUALS(1,
4526 countLogLinesContaining("replSet set names do not match, ours: rs0; remote "
4527 "node's: rs1"));
4528 ASSERT_TRUE(response.isMismatched());
4529 ASSERT_EQUALS("", response.getHbMsg());
4530 }
4531
TEST_F(PrepareHeartbeatResponseTest,PopulateFullHeartbeatResponseEvenWhenHeartbeatRequestLacksASenderID)4532 TEST_F(PrepareHeartbeatResponseTest,
4533 PopulateFullHeartbeatResponseEvenWhenHeartbeatRequestLacksASenderID) {
4534 // set up args without a senderID
4535 ReplSetHeartbeatArgs args;
4536 args.setProtocolVersion(1);
4537 args.setSetName("rs0");
4538 args.setConfigVersion(1);
4539 ReplSetHeartbeatResponse response;
4540 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4541
4542 // prepare response and check the results
4543 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4544 ASSERT_OK(result);
4545 ASSERT_FALSE(response.isElectable());
4546 ASSERT_TRUE(response.isReplSet());
4547 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4548 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4549 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4550 ASSERT_EQUALS("", response.getHbMsg());
4551 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4552 ASSERT_EQUALS(1, response.getConfigVersion());
4553 }
4554
TEST_F(PrepareHeartbeatResponseTest,PopulateFullHeartbeatResponseEvenWhenHeartbeatRequestHasAnInvalidSenderID)4555 TEST_F(PrepareHeartbeatResponseTest,
4556 PopulateFullHeartbeatResponseEvenWhenHeartbeatRequestHasAnInvalidSenderID) {
4557 // set up args with a senderID which is not present in our config
4558 ReplSetHeartbeatArgs args;
4559 args.setProtocolVersion(1);
4560 args.setSetName("rs0");
4561 args.setConfigVersion(1);
4562 args.setSenderId(2);
4563 ReplSetHeartbeatResponse response;
4564 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4565
4566 // prepare response and check the results
4567 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4568 ASSERT_OK(result);
4569 ASSERT_FALSE(response.isElectable());
4570 ASSERT_TRUE(response.isReplSet());
4571 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4572 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4573 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4574 ASSERT_EQUALS("", response.getHbMsg());
4575 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4576 ASSERT_EQUALS(1, response.getConfigVersion());
4577 }
4578
TEST_F(PrepareHeartbeatResponseTest,PopulateHeartbeatResponseWithFullConfigWhenHeartbeatRequestHasAnOldConfigVersion)4579 TEST_F(PrepareHeartbeatResponseTest,
4580 PopulateHeartbeatResponseWithFullConfigWhenHeartbeatRequestHasAnOldConfigVersion) {
4581 // set up args with a config version lower than ours
4582 ReplSetHeartbeatArgs args;
4583 args.setProtocolVersion(1);
4584 args.setConfigVersion(0);
4585 args.setSetName("rs0");
4586 args.setSenderId(20);
4587 ReplSetHeartbeatResponse response;
4588 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4589
4590 // prepare response and check the results
4591 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4592 ASSERT_OK(result);
4593 ASSERT_TRUE(response.hasConfig());
4594 ASSERT_FALSE(response.isElectable());
4595 ASSERT_TRUE(response.isReplSet());
4596 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4597 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4598 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4599 ASSERT_EQUALS("", response.getHbMsg());
4600 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4601 ASSERT_EQUALS(1, response.getConfigVersion());
4602 }
4603
TEST_F(PrepareHeartbeatResponseTest,PopulateFullHeartbeatResponseWhenHeartbeatRequestHasANewerConfigVersion)4604 TEST_F(PrepareHeartbeatResponseTest,
4605 PopulateFullHeartbeatResponseWhenHeartbeatRequestHasANewerConfigVersion) {
4606 // set up args with a config version higher than ours
4607 ReplSetHeartbeatArgs args;
4608 args.setProtocolVersion(1);
4609 args.setConfigVersion(10);
4610 args.setSetName("rs0");
4611 args.setSenderId(20);
4612 ReplSetHeartbeatResponse response;
4613 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4614
4615 // prepare response and check the results
4616 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4617 ASSERT_OK(result);
4618 ASSERT_FALSE(response.hasConfig());
4619 ASSERT_FALSE(response.isElectable());
4620 ASSERT_TRUE(response.isReplSet());
4621 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4622 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4623 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4624 ASSERT_EQUALS("", response.getHbMsg());
4625 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4626 ASSERT_EQUALS(1, response.getConfigVersion());
4627 }
4628
TEST_F(PrepareHeartbeatResponseTest,SetStateDisagreementInHeartbeatResponseWhenHeartbeatRequestIsFromADownNode)4629 TEST_F(PrepareHeartbeatResponseTest,
4630 SetStateDisagreementInHeartbeatResponseWhenHeartbeatRequestIsFromADownNode) {
4631 // set up args with sender down from our perspective
4632 ReplSetHeartbeatArgs args;
4633 args.setProtocolVersion(1);
4634 args.setConfigVersion(1);
4635 args.setSetName("rs0");
4636 args.setSenderId(20);
4637 ReplSetHeartbeatResponse response;
4638 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4639
4640 // prepare response and check the results
4641 prepareHeartbeatResponse(args, OpTime(), &response, &result);
4642 ASSERT_OK(result);
4643 ASSERT_FALSE(response.isElectable());
4644 ASSERT_TRUE(response.isReplSet());
4645 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4646 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4647 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4648 ASSERT_EQUALS("", response.getHbMsg());
4649 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4650 ASSERT_EQUALS(1, response.getConfigVersion());
4651 ASSERT_TRUE(response.isStateDisagreement());
4652 }
4653
TEST_F(PrepareHeartbeatResponseTest,SetElectableInHeartbeatResponseWhenWeCanSeeAMajorityOfTheNodes)4654 TEST_F(PrepareHeartbeatResponseTest,
4655 SetElectableInHeartbeatResponseWhenWeCanSeeAMajorityOfTheNodes) {
4656 // set up args and acknowledge sender
4657 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime());
4658 ReplSetHeartbeatArgs args;
4659 args.setProtocolVersion(1);
4660 args.setConfigVersion(1);
4661 args.setSetName("rs0");
4662 args.setSenderId(20);
4663 ReplSetHeartbeatResponse response;
4664 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4665
4666 // prepare response and check the results
4667 prepareHeartbeatResponse(args, OpTime(Timestamp(100, 0), 0), &response, &result);
4668 ASSERT_OK(result);
4669 // this change to true because we can now see a majority, unlike in the previous cases
4670 ASSERT_TRUE(response.isElectable());
4671 ASSERT_TRUE(response.isReplSet());
4672 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4673 ASSERT_EQUALS(OpTime(Timestamp(100, 0), 0), response.getDurableOpTime());
4674 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4675 ASSERT_EQUALS("", response.getHbMsg());
4676 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4677 ASSERT_EQUALS(1, response.getConfigVersion());
4678 }
4679
TEST_F(TopoCoordTest,SetConfigVersionToNegativeTwoInHeartbeatResponseWhenNoConfigHasBeenReceived)4680 TEST_F(TopoCoordTest, SetConfigVersionToNegativeTwoInHeartbeatResponseWhenNoConfigHasBeenReceived) {
4681 // set up args and acknowledge sender
4682 ReplSetHeartbeatArgs args;
4683 args.setProtocolVersion(1);
4684 args.setConfigVersion(1);
4685 args.setSetName("rs0");
4686 args.setSenderId(20);
4687 ReplSetHeartbeatResponse response;
4688 // prepare response and check the results
4689 Status result = getTopoCoord().prepareHeartbeatResponse(now()++, args, "rs0", &response);
4690 ASSERT_OK(result);
4691 ASSERT_FALSE(response.isElectable());
4692 ASSERT_TRUE(response.isReplSet());
4693 ASSERT_EQUALS(MemberState::RS_STARTUP, response.getState().s);
4694 ASSERT_EQUALS(OpTime(), response.getDurableOpTime());
4695 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4696 ASSERT_EQUALS("", response.getHbMsg());
4697 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4698 ASSERT_EQUALS(-2, response.getConfigVersion());
4699 }
4700
TEST_F(PrepareHeartbeatResponseTest,SetElectableTrueAndStatePrimaryInHeartbeatResponseWhenPrimary)4701 TEST_F(PrepareHeartbeatResponseTest,
4702 SetElectableTrueAndStatePrimaryInHeartbeatResponseWhenPrimary) {
4703 makeSelfPrimary(Timestamp(10, 0));
4704 heartbeatFromMember(HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime());
4705
4706 ReplSetHeartbeatArgs args;
4707 args.setProtocolVersion(1);
4708 args.setConfigVersion(1);
4709 args.setSetName("rs0");
4710 args.setSenderId(20);
4711 ReplSetHeartbeatResponse response;
4712 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4713
4714 // prepare response and check the results
4715 prepareHeartbeatResponse(args, OpTime(Timestamp(11, 0), 0), &response, &result);
4716 ASSERT_OK(result);
4717 // electable because we are already primary
4718 ASSERT_TRUE(response.isElectable());
4719 ASSERT_TRUE(response.isReplSet());
4720 ASSERT_EQUALS(MemberState::RS_PRIMARY, response.getState().s);
4721 ASSERT_EQUALS(OpTime(Timestamp(11, 0), 0), response.getDurableOpTime());
4722 ASSERT_EQUALS(Timestamp(10, 0), response.getElectionTime());
4723 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4724 ASSERT_EQUALS("", response.getHbMsg());
4725 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4726 ASSERT_EQUALS(1, response.getConfigVersion());
4727 }
4728
TEST_F(PrepareHeartbeatResponseTest,IncludeSyncingFromMessageInHeartbeatResponseWhenThereIsASyncSource)4729 TEST_F(PrepareHeartbeatResponseTest,
4730 IncludeSyncingFromMessageInHeartbeatResponseWhenThereIsASyncSource) {
4731 // get a sync source
4732 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime());
4733 heartbeatFromMember(HostAndPort("h3"), "rs0", MemberState::RS_SECONDARY, OpTime());
4734 heartbeatFromMember(
4735 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
4736 heartbeatFromMember(
4737 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(1, 0), 0));
4738 getTopoCoord().chooseNewSyncSource(
4739 now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration);
4740
4741 // set up args
4742 ReplSetHeartbeatArgs args;
4743 args.setProtocolVersion(1);
4744 args.setConfigVersion(1);
4745 args.setSetName("rs0");
4746 args.setSenderId(20);
4747 ReplSetHeartbeatResponse response;
4748 Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result");
4749
4750 // prepare response and check the results
4751 prepareHeartbeatResponse(args, OpTime(Timestamp(100, 0), 0), &response, &result);
4752 ASSERT_OK(result);
4753 ASSERT_TRUE(response.isElectable());
4754 ASSERT_TRUE(response.isReplSet());
4755 ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s);
4756 ASSERT_EQUALS(OpTime(Timestamp(100, 0), 0), response.getDurableOpTime());
4757 ASSERT_EQUALS(0, durationCount<Seconds>(response.getTime()));
4758 // changed to a syncing message because our sync source changed recently
4759 ASSERT_EQUALS("syncing from: h2:27017", response.getHbMsg());
4760 ASSERT_EQUALS("rs0", response.getReplicaSetName());
4761 ASSERT_EQUALS(1, response.getConfigVersion());
4762 ASSERT_EQUALS(HostAndPort("h2"), response.getSyncingTo());
4763 }
4764
TEST_F(TopoCoordTest,BecomeCandidateWhenBecomingSecondaryInSingleNodeSet)4765 TEST_F(TopoCoordTest, BecomeCandidateWhenBecomingSecondaryInSingleNodeSet) {
4766 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4767 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4768 updateConfig(BSON("_id"
4769 << "rs0"
4770 << "version"
4771 << 1
4772 << "members"
4773 << BSON_ARRAY(BSON("_id" << 1 << "host"
4774 << "hself"))),
4775 0);
4776 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4777
4778 // if we are the only node, we should become a candidate when we transition to SECONDARY
4779 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4780 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
4781 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4782 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
4783 }
4784
TEST_F(TopoCoordTest,BecomeCandidateWhenReconfigToBeElectableInSingleNodeSet)4785 TEST_F(TopoCoordTest, BecomeCandidateWhenReconfigToBeElectableInSingleNodeSet) {
4786 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4787 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4788 ReplSetConfig cfg;
4789 cfg.initialize(BSON("_id"
4790 << "rs0"
4791 << "version"
4792 << 1
4793 << "members"
4794 << BSON_ARRAY(BSON("_id" << 1 << "host"
4795 << "hself"
4796 << "priority"
4797 << 0))))
4798 .transitional_ignore();
4799 getTopoCoord().updateConfig(cfg, 0, now()++);
4800 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4801
4802 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4803 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
4804 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4805 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
4806
4807 // we should become a candidate when we reconfig to become electable
4808
4809 updateConfig(BSON("_id"
4810 << "rs0"
4811 << "version"
4812 << 1
4813 << "members"
4814 << BSON_ARRAY(BSON("_id" << 1 << "host"
4815 << "hself"))),
4816 0);
4817 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4818 }
4819
TEST_F(TopoCoordTest,NodeDoesNotBecomeCandidateWhenBecomingSecondaryInSingleNodeSetIfUnelectable)4820 TEST_F(TopoCoordTest, NodeDoesNotBecomeCandidateWhenBecomingSecondaryInSingleNodeSetIfUnelectable) {
4821 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4822 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4823 ReplSetConfig cfg;
4824 cfg.initialize(BSON("_id"
4825 << "rs0"
4826 << "version"
4827 << 1
4828 << "members"
4829 << BSON_ARRAY(BSON("_id" << 1 << "host"
4830 << "hself"
4831 << "priority"
4832 << 0))))
4833 .transitional_ignore();
4834
4835 getTopoCoord().updateConfig(cfg, 0, now()++);
4836 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4837
4838 // despite being the only node, we are unelectable, so we should not become a candidate
4839 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4840 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
4841 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4842 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
4843 }
4844
TEST_F(TopoCoordTest,NodeTransitionsFromRemovedToStartup2WhenAddedToConfig)4845 TEST_F(TopoCoordTest, NodeTransitionsFromRemovedToStartup2WhenAddedToConfig) {
4846 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4847 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4848 // config to be absent from the set
4849 updateConfig(BSON("_id"
4850 << "rs0"
4851 << "version"
4852 << 1
4853 << "members"
4854 << BSON_ARRAY(BSON("_id" << 1 << "host"
4855 << "host2:27017")
4856 << BSON("_id" << 2 << "host"
4857 << "host3:27017"))),
4858 -1);
4859 // should become removed since we are not in the set
4860 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4861 ASSERT_EQUALS(MemberState::RS_REMOVED, getTopoCoord().getMemberState().s);
4862
4863 // reconfig to add to set
4864 updateConfig(BSON("_id"
4865 << "rs0"
4866 << "version"
4867 << 2
4868 << "members"
4869 << BSON_ARRAY(BSON("_id" << 0 << "host"
4870 << "host1:27017")
4871 << BSON("_id" << 1 << "host"
4872 << "host2:27017")
4873 << BSON("_id" << 2 << "host"
4874 << "host3:27017"))),
4875 0);
4876 // having been added to the config, we should no longer be REMOVED and should enter STARTUP2
4877 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4878 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4879 }
4880
TEST_F(TopoCoordTest,NodeTransitionsToRemovedWhenRemovedFromConfig)4881 TEST_F(TopoCoordTest, NodeTransitionsToRemovedWhenRemovedFromConfig) {
4882 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4883 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4884 updateConfig(BSON("_id"
4885 << "rs0"
4886 << "version"
4887 << 1
4888 << "members"
4889 << BSON_ARRAY(BSON("_id" << 0 << "host"
4890 << "host1:27017")
4891 << BSON("_id" << 1 << "host"
4892 << "host2:27017")
4893 << BSON("_id" << 2 << "host"
4894 << "host3:27017"))),
4895 0);
4896 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4897 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4898
4899 // reconfig to remove self
4900 updateConfig(BSON("_id"
4901 << "rs0"
4902 << "version"
4903 << 2
4904 << "members"
4905 << BSON_ARRAY(BSON("_id" << 1 << "host"
4906 << "host2:27017")
4907 << BSON("_id" << 2 << "host"
4908 << "host3:27017"))),
4909 -1);
4910 // should become removed since we are no longer in the set
4911 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4912 ASSERT_EQUALS(MemberState::RS_REMOVED, getTopoCoord().getMemberState().s);
4913 }
4914
TEST_F(TopoCoordTest,NodeTransitionsToRemovedWhenRemovedFromConfigEvenWhenPrimary)4915 TEST_F(TopoCoordTest, NodeTransitionsToRemovedWhenRemovedFromConfigEvenWhenPrimary) {
4916 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4917 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4918 updateConfig(BSON("_id"
4919 << "rs0"
4920 << "version"
4921 << 1
4922 << "members"
4923 << BSON_ARRAY(BSON("_id" << 0 << "host"
4924 << "host1:27017"))),
4925 0);
4926 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4927 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4928 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
4929 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4930
4931 // win election and primary
4932 getTopoCoord().processWinElection(OID::gen(), Timestamp());
4933 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
4934 ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s);
4935
4936 // reconfig to remove self
4937 updateConfig(BSON("_id"
4938 << "rs0"
4939 << "version"
4940 << 2
4941 << "members"
4942 << BSON_ARRAY(BSON("_id" << 1 << "host"
4943 << "host2:27017")
4944 << BSON("_id" << 2 << "host"
4945 << "host3:27017"))),
4946 -1);
4947 // should become removed since we are no longer in the set even though we were primary
4948 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4949 ASSERT_EQUALS(MemberState::RS_REMOVED, getTopoCoord().getMemberState().s);
4950 }
4951
TEST_F(TopoCoordTest,NodeTransitionsToSecondaryWhenReconfiggingToBeUnelectable)4952 TEST_F(TopoCoordTest, NodeTransitionsToSecondaryWhenReconfiggingToBeUnelectable) {
4953 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4954 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4955 updateConfig(BSON("_id"
4956 << "rs0"
4957 << "version"
4958 << 1
4959 << "members"
4960 << BSON_ARRAY(BSON("_id" << 0 << "host"
4961 << "host1:27017"))),
4962 0);
4963 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4964 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
4965 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
4966 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
4967
4968 // win election and primary
4969 getTopoCoord().processWinElection(OID::gen(), Timestamp());
4970 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
4971 ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s);
4972
4973 // now lose primary due to loss of electability
4974 updateConfig(BSON("_id"
4975 << "rs0"
4976 << "version"
4977 << 2
4978 << "members"
4979 << BSON_ARRAY(BSON("_id" << 0 << "host"
4980 << "host1:27017"
4981 << "priority"
4982 << 0)
4983 << BSON("_id" << 1 << "host"
4984 << "host2:27017")
4985 << BSON("_id" << 2 << "host"
4986 << "host3:27017"))),
4987 0);
4988 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4989 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
4990 }
4991
TEST_F(TopoCoordTest,NodeMaintainsPrimaryStateAcrossReconfigIfNodeRemainsElectable)4992 TEST_F(TopoCoordTest, NodeMaintainsPrimaryStateAcrossReconfigIfNodeRemainsElectable) {
4993 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
4994 ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s);
4995 updateConfig(BSON("_id"
4996 << "rs0"
4997 << "version"
4998 << 1
4999 << "members"
5000 << BSON_ARRAY(BSON("_id" << 0 << "host"
5001 << "host1:27017"))),
5002 0);
5003
5004 ASSERT_FALSE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
5005 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
5006 getTopoCoord().setFollowerMode(MemberState::RS_SECONDARY);
5007 ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
5008
5009 // win election and primary
5010 getTopoCoord().processWinElection(OID::gen(), Timestamp());
5011 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
5012 ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s);
5013
5014 // Now reconfig in ways that leave us electable and ensure we are still the primary.
5015 // Add hosts
5016 updateConfig(BSON("_id"
5017 << "rs0"
5018 << "version"
5019 << 2
5020 << "members"
5021 << BSON_ARRAY(BSON("_id" << 0 << "host"
5022 << "host1:27017")
5023 << BSON("_id" << 1 << "host"
5024 << "host2:27017")
5025 << BSON("_id" << 2 << "host"
5026 << "host3:27017"))),
5027 0,
5028 Date_t::fromMillisSinceEpoch(-1),
5029 OpTime(Timestamp(10, 0), 0));
5030 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
5031 ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s);
5032
5033 // Change priorities and tags
5034 updateConfig(BSON("_id"
5035 << "rs0"
5036 << "version"
5037 << 2
5038 << "members"
5039 << BSON_ARRAY(BSON("_id" << 0 << "host"
5040 << "host1:27017"
5041 << "priority"
5042 << 10)
5043 << BSON("_id" << 1 << "host"
5044 << "host2:27017"
5045 << "priority"
5046 << 5
5047 << "tags"
5048 << BSON("dc"
5049 << "NA"
5050 << "rack"
5051 << "rack1")))),
5052 0,
5053 Date_t::fromMillisSinceEpoch(-1),
5054 OpTime(Timestamp(10, 0), 0));
5055 ASSERT_TRUE(TopologyCoordinator::Role::kLeader == getTopoCoord().getRole());
5056 ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s);
5057 }
5058
TEST_F(TopoCoordTest,NodeMaintainsSecondaryStateAcrossReconfig)5059 TEST_F(TopoCoordTest, NodeMaintainsSecondaryStateAcrossReconfig) {
5060 updateConfig(BSON("_id"
5061 << "rs0"
5062 << "version"
5063 << 1
5064 << "members"
5065 << BSON_ARRAY(BSON("_id" << 1 << "host"
5066 << "host1:27017")
5067 << BSON("_id" << 2 << "host"
5068 << "host2:27017"))),
5069 0);
5070 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
5071 ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s);
5072 setSelfMemberState(MemberState::RS_SECONDARY);
5073 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
5074
5075 // reconfig and stay secondary
5076 updateConfig(BSON("_id"
5077 << "rs0"
5078 << "version"
5079 << 2
5080 << "members"
5081 << BSON_ARRAY(BSON("_id" << 0 << "host"
5082 << "host1:27017")
5083 << BSON("_id" << 1 << "host"
5084 << "host2:27017")
5085 << BSON("_id" << 2 << "host"
5086 << "host3:27017"))),
5087 0);
5088 ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
5089 ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s);
5090 }
5091
5092 // TODO(dannenberg) figure out what this is trying to test..
TEST_F(HeartbeatResponseTest,ReconfigBetweenHeartbeatRequestAndRepsonse)5093 TEST_F(HeartbeatResponseTest, ReconfigBetweenHeartbeatRequestAndRepsonse) {
5094 OpTime election = OpTime(Timestamp(14, 0), 0);
5095 OpTime lastOpTimeApplied = OpTime(Timestamp(13, 0), 0);
5096
5097 // all three members up and secondaries
5098 setSelfMemberState(MemberState::RS_SECONDARY);
5099
5100 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5101 HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
5102 ASSERT_NO_ACTION(nextAction.getAction());
5103
5104 nextAction = receiveUpHeartbeat(
5105 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5106 ASSERT_NO_ACTION(nextAction.getAction());
5107
5108 // now request from host3 and receive after host2 has been removed via reconfig
5109 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host3"));
5110
5111 updateConfig(BSON("_id"
5112 << "rs0"
5113 << "version"
5114 << 2
5115 << "members"
5116 << BSON_ARRAY(BSON("_id" << 0 << "host"
5117 << "host1:27017")
5118 << BSON("_id" << 2 << "host"
5119 << "host3:27017"))),
5120 0);
5121
5122 ReplSetHeartbeatResponse hb;
5123 hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_PRIMARY), 0)
5124 .transitional_ignore();
5125 hb.setDurableOpTime(lastOpTimeApplied);
5126 hb.setElectionTime(election.getTimestamp());
5127 StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb);
5128 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
5129 now()++, Milliseconds(0), HostAndPort("host3"), hbResponse);
5130
5131 // now primary should be host3, index 1, and we should perform NoAction in response
5132 ASSERT_EQUALS(1, getCurrentPrimaryIndex());
5133 ASSERT_NO_ACTION(action.getAction());
5134 }
5135
5136 // TODO(dannenberg) figure out what this is trying to test..
TEST_F(HeartbeatResponseTest,ReconfigNodeRemovedBetweenHeartbeatRequestAndRepsonse)5137 TEST_F(HeartbeatResponseTest, ReconfigNodeRemovedBetweenHeartbeatRequestAndRepsonse) {
5138 OpTime election = OpTime(Timestamp(14, 0), 0);
5139 OpTime lastOpTimeApplied = OpTime(Timestamp(13, 0), 0);
5140
5141 // all three members up and secondaries
5142 setSelfMemberState(MemberState::RS_SECONDARY);
5143
5144 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5145 HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied);
5146 ASSERT_NO_ACTION(nextAction.getAction());
5147
5148 nextAction = receiveUpHeartbeat(
5149 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5150 ASSERT_NO_ACTION(nextAction.getAction());
5151
5152 // now request from host3 and receive after host2 has been removed via reconfig
5153 getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host3"));
5154
5155 updateConfig(BSON("_id"
5156 << "rs0"
5157 << "version"
5158 << 2
5159 << "members"
5160 << BSON_ARRAY(BSON("_id" << 0 << "host"
5161 << "host1:27017")
5162 << BSON("_id" << 1 << "host"
5163 << "host2:27017"))),
5164 0);
5165
5166 ReplSetHeartbeatResponse hb;
5167 hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_PRIMARY), 0)
5168 .transitional_ignore();
5169 hb.setDurableOpTime(lastOpTimeApplied);
5170 hb.setElectionTime(election.getTimestamp());
5171 StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb);
5172 HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse(
5173 now()++, Milliseconds(0), HostAndPort("host3"), hbResponse);
5174
5175 // primary should not be set and we should perform NoAction in response
5176 ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
5177 ASSERT_NO_ACTION(action.getAction());
5178 }
5179
TEST_F(HeartbeatResponseTest,ShouldChangeSyncSourceWhenMemberNotInConfig)5180 TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenMemberNotInConfig) {
5181 // In this test, the TopologyCoordinator should tell us to change sync sources away from
5182 // "host4" since "host4" is absent from the config of version 10.
5183 ReplSetMetadata metadata(0, OpTime(), OpTime(), 10, OID(), -1, -1);
5184 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
5185 HostAndPort("host4"), metadata, makeOplogQueryMetadata(), now()));
5186 }
5187
TEST_F(HeartbeatResponseTest,ShouldChangeSyncSourceWhenMemberHasYetToHeartbeatUs)5188 TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenMemberHasYetToHeartbeatUs) {
5189 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5190 // "host2" since we do not yet have a heartbeat (and as a result do not yet have an optime)
5191 // for "host2"
5192 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5193 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5194 }
5195
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhenNodeIsFreshByHeartbeatButNotMetadata)5196 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsFreshByHeartbeatButNotMetadata) {
5197 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5198 // "host2" and to "host3" since "host2" is only more than maxSyncSourceLagSecs(30) behind
5199 // "host3" according to metadata, not heartbeat data.
5200 OpTime election = OpTime();
5201 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5202 // ahead by more than maxSyncSourceLagSecs (30)
5203 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5204
5205 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5206 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5207 ASSERT_NO_ACTION(nextAction.getAction());
5208
5209 nextAction = receiveUpHeartbeat(
5210 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5211 ASSERT_NO_ACTION(nextAction.getAction());
5212
5213 // set up complete, time for actual check
5214 startCapturingLogMessages();
5215 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"),
5216 makeReplSetMetadata(),
5217 makeOplogQueryMetadata(lastOpTimeApplied),
5218 now()));
5219 stopCapturingLogMessages();
5220 ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source"));
5221
5222 // set up complete, time for actual check
5223 startCapturingLogMessages();
5224 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5225 HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now()));
5226 stopCapturingLogMessages();
5227 ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source"));
5228 }
5229
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhenNodeIsStaleByHeartbeatButNotMetadata)5230 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsStaleByHeartbeatButNotMetadata) {
5231 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5232 // "host2" and to "host3" since "host2" is only more than maxSyncSourceLagSecs(30) behind
5233 // "host3" according to heartbeat data, not metadata.
5234 OpTime election = OpTime();
5235 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5236 // ahead by more than maxSyncSourceLagSecs (30)
5237 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5238
5239 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5240 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5241 ASSERT_NO_ACTION(nextAction.getAction());
5242
5243 nextAction = receiveUpHeartbeat(
5244 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5245 ASSERT_NO_ACTION(nextAction.getAction());
5246
5247 // set up complete, time for actual check
5248 startCapturingLogMessages();
5249 ASSERT_FALSE(
5250 getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"),
5251 makeReplSetMetadata(),
5252 makeOplogQueryMetadata(fresherLastOpTimeApplied),
5253 now()));
5254 stopCapturingLogMessages();
5255 ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source"));
5256
5257 // set up complete, time for actual check
5258 startCapturingLogMessages();
5259 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5260 HostAndPort("host2"), makeReplSetMetadata(fresherLastOpTimeApplied), boost::none, now()));
5261 stopCapturingLogMessages();
5262 ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source"));
5263 }
5264
TEST_F(HeartbeatResponseTest,ShouldChangeSyncSourceWhenFresherMemberExists)5265 TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenFresherMemberExists) {
5266 // In this test, the TopologyCoordinator should tell us to change sync sources away from
5267 // "host2" and to "host3" since "host2" is more than maxSyncSourceLagSecs(30) behind "host3"
5268 OpTime election = OpTime();
5269 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5270 // ahead by more than maxSyncSourceLagSecs (30)
5271 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5272
5273 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5274 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5275 ASSERT_NO_ACTION(nextAction.getAction());
5276
5277 nextAction = receiveUpHeartbeat(
5278 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5279 ASSERT_NO_ACTION(nextAction.getAction());
5280
5281 // set up complete, time for actual check
5282 startCapturingLogMessages();
5283 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
5284 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5285 stopCapturingLogMessages();
5286 ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source"));
5287 }
5288
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhileFresherMemberIsBlackListed)5289 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhileFresherMemberIsBlackListed) {
5290 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5291 // "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind
5292 // "host3", since "host3" is blacklisted
5293 // Then, confirm that unblacklisting only works if time has passed the blacklist time.
5294 OpTime election = OpTime();
5295 OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0);
5296 // ahead by more than maxSyncSourceLagSecs (30)
5297 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5298
5299 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5300 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5301 ASSERT_NO_ACTION(nextAction.getAction());
5302
5303 nextAction = receiveUpHeartbeat(
5304 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5305 ASSERT_NO_ACTION(nextAction.getAction());
5306 getTopoCoord().blacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100));
5307
5308 // set up complete, time for actual check
5309 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5310 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5311
5312 // unblacklist with too early a time (node should remained blacklisted)
5313 getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(90));
5314 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5315 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5316
5317 // unblacklist and it should succeed
5318 getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100));
5319 startCapturingLogMessages();
5320 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
5321 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5322 stopCapturingLogMessages();
5323 ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source"));
5324 }
5325
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhenFresherMemberIsDown)5326 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberIsDown) {
5327 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5328 // "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind
5329 // "host3", since "host3" is down
5330 OpTime election = OpTime();
5331 OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0);
5332 // ahead by more than maxSyncSourceLagSecs (30)
5333 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5334
5335 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5336 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5337 ASSERT_NO_ACTION(nextAction.getAction());
5338
5339 nextAction = receiveUpHeartbeat(
5340 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5341 ASSERT_NO_ACTION(nextAction.getAction());
5342
5343 // set up complete, time for actual check
5344 nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", lastOpTimeApplied);
5345 ASSERT_NO_ACTION(nextAction.getAction());
5346 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5347 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5348 }
5349
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhenFresherMemberIsNotReadable)5350 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberIsNotReadable) {
5351 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5352 // "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind
5353 // "host3", since "host3" is in a non-readable mode (RS_ROLLBACK)
5354 OpTime election = OpTime();
5355 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5356 // ahead by more than maxSyncSourceLagSecs (30)
5357 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5358
5359 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5360 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5361 ASSERT_NO_ACTION(nextAction.getAction());
5362
5363 nextAction = receiveUpHeartbeat(
5364 HostAndPort("host3"), "rs0", MemberState::RS_ROLLBACK, election, fresherLastOpTimeApplied);
5365 ASSERT_NO_ACTION(nextAction.getAction());
5366
5367 // set up complete, time for actual check
5368 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5369 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5370 }
5371
TEST_F(HeartbeatResponseTest,ShouldNotChangeSyncSourceWhenFresherMemberDoesNotBuildIndexes)5372 TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberDoesNotBuildIndexes) {
5373 // In this test, the TopologyCoordinator should not tell us to change sync sources away from
5374 // "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind
5375 // "host3", since "host3" does not build indexes
5376 OpTime election = OpTime();
5377 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5378 // ahead by more than maxSyncSourceLagSecs (30)
5379 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5380
5381 updateConfig(BSON("_id"
5382 << "rs0"
5383 << "version"
5384 << 6
5385 << "members"
5386 << BSON_ARRAY(BSON("_id" << 0 << "host"
5387 << "hself")
5388 << BSON("_id" << 1 << "host"
5389 << "host2")
5390 << BSON("_id" << 2 << "host"
5391 << "host3"
5392 << "buildIndexes"
5393 << false
5394 << "priority"
5395 << 0))),
5396 0);
5397 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5398 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5399 ASSERT_NO_ACTION(nextAction.getAction());
5400 nextAction = receiveUpHeartbeat(
5401 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5402 ASSERT_NO_ACTION(nextAction.getAction());
5403
5404 // set up complete, time for actual check
5405 ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(
5406 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5407 }
5408
TEST_F(HeartbeatResponseTest,ShouldChangeSyncSourceWhenFresherMemberDoesNotBuildIndexesAndNeitherDoWe)5409 TEST_F(HeartbeatResponseTest,
5410 ShouldChangeSyncSourceWhenFresherMemberDoesNotBuildIndexesAndNeitherDoWe) {
5411 // In this test, the TopologyCoordinator should tell us to change sync sources away from
5412 // "host2" and to "host3" despite "host3" not building indexes because we do not build
5413 // indexes either and "host2" is more than maxSyncSourceLagSecs(30) behind "host3"
5414 OpTime election = OpTime();
5415 OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0);
5416 // ahead by more than maxSyncSourceLagSecs (30)
5417 OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0);
5418
5419 updateConfig(BSON("_id"
5420 << "rs0"
5421 << "version"
5422 << 7
5423 << "members"
5424 << BSON_ARRAY(BSON("_id" << 0 << "host"
5425 << "hself"
5426 << "buildIndexes"
5427 << false
5428 << "priority"
5429 << 0)
5430 << BSON("_id" << 1 << "host"
5431 << "host2")
5432 << BSON("_id" << 2 << "host"
5433 << "host3"
5434 << "buildIndexes"
5435 << false
5436 << "priority"
5437 << 0))),
5438 0);
5439 HeartbeatResponseAction nextAction = receiveUpHeartbeat(
5440 HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied);
5441 ASSERT_NO_ACTION(nextAction.getAction());
5442 nextAction = receiveUpHeartbeat(
5443 HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied);
5444 ASSERT_NO_ACTION(nextAction.getAction());
5445
5446 // set up complete, time for actual check
5447 startCapturingLogMessages();
5448 ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(
5449 HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now()));
5450 stopCapturingLogMessages();
5451 ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source"));
5452 }
5453
TEST_F(TopoCoordTest,ShouldNotStandForElectionWhileAwareOfPrimary)5454 TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileAwareOfPrimary) {
5455 updateConfig(BSON("_id"
5456 << "rs0"
5457 << "version"
5458 << 1
5459 << "members"
5460 << BSON_ARRAY(BSON("_id" << 10 << "host"
5461 << "hself")
5462 << BSON("_id" << 20 << "host"
5463 << "h2")
5464 << BSON("_id" << 30 << "host"
5465 << "h3"))),
5466 0);
5467 setSelfMemberState(MemberState::RS_SECONDARY);
5468
5469 heartbeatFromMember(
5470 HostAndPort("h2"), "rs0", MemberState::RS_PRIMARY, OpTime(Timestamp(1, 0), 0));
5471 const auto status = getTopoCoord().checkShouldStandForElection(now()++);
5472 ASSERT_EQ(ErrorCodes::NodeNotElectable, status);
5473 ASSERT_STRING_CONTAINS(status.reason(), "there is a Primary");
5474 }
5475
TEST_F(TopoCoordTest,ShouldNotStandForElectionWhileTooStale)5476 TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileTooStale) {
5477 updateConfig(BSON("_id"
5478 << "rs0"
5479 << "version"
5480 << 1
5481 << "members"
5482 << BSON_ARRAY(BSON("_id" << 10 << "host"
5483 << "hself")
5484 << BSON("_id" << 20 << "host"
5485 << "h2")
5486 << BSON("_id" << 30 << "host"
5487 << "h3"))),
5488 0);
5489 setSelfMemberState(MemberState::RS_SECONDARY);
5490
5491 heartbeatFromMember(
5492 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(10000, 0), 0));
5493 const auto status = getTopoCoord().checkShouldStandForElection(now()++);
5494 ASSERT_EQ(ErrorCodes::NodeNotElectable, status);
5495 ASSERT_STRING_CONTAINS(status.reason(), "my last optime is");
5496 }
5497
TEST_F(TopoCoordTest,VoteForMyselfFailsWhileNotCandidate)5498 TEST_F(TopoCoordTest, VoteForMyselfFailsWhileNotCandidate) {
5499 updateConfig(BSON("_id"
5500 << "rs0"
5501 << "version"
5502 << 1
5503 << "members"
5504 << BSON_ARRAY(BSON("_id" << 10 << "host"
5505 << "hself")
5506 << BSON("_id" << 20 << "host"
5507 << "h2")
5508 << BSON("_id" << 30 << "host"
5509 << "h3"))),
5510 0);
5511 setSelfMemberState(MemberState::RS_SECONDARY);
5512 ASSERT_FALSE(getTopoCoord().voteForMyself(now()++));
5513 }
5514
TEST_F(TopoCoordTest,NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter)5515 TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) {
5516 updateConfig(BSON("_id"
5517 << "rs0"
5518 << "version"
5519 << 1
5520 << "members"
5521 << BSON_ARRAY(BSON("_id" << 10 << "host"
5522 << "hself"
5523 << "arbiterOnly"
5524 << true)
5525 << BSON("_id" << 20 << "host"
5526 << "h2")
5527 << BSON("_id" << 30 << "host"
5528 << "h3"))),
5529 0);
5530 ASSERT_EQUALS(MemberState::RS_ARBITER, getTopoCoord().getMemberState().s);
5531 }
5532
TEST_F(TopoCoordTest,ShouldNotStandForElectionWhileRemovedFromTheConfig)5533 TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) {
5534 const auto status = getTopoCoord().checkShouldStandForElection(now()++);
5535 ASSERT_EQ(ErrorCodes::NodeNotElectable, status);
5536 ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config");
5537 }
5538
TEST_F(TopoCoordTest,ShouldNotStandForElectionWhenAPositiveResponseWasGivenInTheVoteLeasePeriod)5539 TEST_F(TopoCoordTest, ShouldNotStandForElectionWhenAPositiveResponseWasGivenInTheVoteLeasePeriod) {
5540 updateConfig(BSON("_id"
5541 << "rs0"
5542 << "version"
5543 << 1
5544 << "members"
5545 << BSON_ARRAY(BSON("_id" << 10 << "host"
5546 << "hself")
5547 << BSON("_id" << 20 << "host"
5548 << "h2")
5549 << BSON("_id" << 30 << "host"
5550 << "h3"))),
5551 0);
5552 setSelfMemberState(MemberState::RS_SECONDARY);
5553 heartbeatFromMember(
5554 HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(100, 0), 0));
5555
5556 // vote for another node
5557 OID remoteRound = OID::gen();
5558 ReplicationCoordinator::ReplSetElectArgs electArgs;
5559 electArgs.set = "rs0";
5560 electArgs.round = remoteRound;
5561 electArgs.cfgver = 1;
5562 electArgs.whoid = 20;
5563
5564 // need to be 30 secs beyond the start of time to pass last vote lease
5565 now() += Seconds(30);
5566 BSONObjBuilder electResponseBuilder;
5567 Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse");
5568 getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result);
5569 BSONObj response = electResponseBuilder.obj();
5570 ASSERT_OK(result);
5571 std::cout << response;
5572 ASSERT_EQUALS(1, response["vote"].Int());
5573 ASSERT_EQUALS(remoteRound, response["round"].OID());
5574
5575 const auto status = getTopoCoord().checkShouldStandForElection(now()++);
5576 ASSERT_EQ(ErrorCodes::NodeNotElectable, status);
5577 ASSERT_STRING_CONTAINS(status.reason(), "I recently voted for ");
5578 }
5579
TEST_F(TopoCoordTest,NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm)5580 TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) {
5581 updateConfig(BSON("_id"
5582 << "rs0"
5583 << "version"
5584 << 1
5585 << "members"
5586 << BSON_ARRAY(BSON("_id" << 10 << "host"
5587 << "hself")
5588 << BSON("_id" << 20 << "host"
5589 << "h2")
5590 << BSON("_id" << 30 << "host"
5591 << "h3"))),
5592 0);
5593 setSelfMemberState(MemberState::RS_SECONDARY);
5594
5595 ReplSetRequestVotesArgs args;
5596 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5597 << "rs0"
5598 << "term"
5599 << 1LL
5600 << "candidateIndex"
5601 << 0LL
5602 << "configVersion"
5603 << 1LL
5604 << "lastCommittedOp"
5605 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5606 .transitional_ignore();
5607 ReplSetRequestVotesResponse response;
5608
5609 getTopoCoord().processReplSetRequestVotes(args, &response);
5610 ASSERT_EQUALS("", response.getReason());
5611 ASSERT_TRUE(response.getVoteGranted());
5612
5613 ReplSetRequestVotesArgs args2;
5614 args2
5615 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
5616 << "rs0"
5617 << "term"
5618 << 1LL
5619 << "candidateIndex"
5620 << 1LL
5621 << "configVersion"
5622 << 1LL
5623 << "lastCommittedOp"
5624 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5625 .transitional_ignore();
5626 ReplSetRequestVotesResponse response2;
5627
5628 // different candidate same term, should be a problem
5629 getTopoCoord().processReplSetRequestVotes(args2, &response2);
5630 ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)",
5631 response2.getReason());
5632 ASSERT_FALSE(response2.getVoteGranted());
5633 }
5634
TEST_F(TopoCoordTest,DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatTerm)5635 TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatTerm) {
5636 updateConfig(BSON("_id"
5637 << "rs0"
5638 << "version"
5639 << 1
5640 << "members"
5641 << BSON_ARRAY(BSON("_id" << 10 << "host"
5642 << "hself")
5643 << BSON("_id" << 20 << "host"
5644 << "h2")
5645 << BSON("_id" << 30 << "host"
5646 << "h3"))),
5647 0);
5648 setSelfMemberState(MemberState::RS_SECONDARY);
5649
5650 // dry run
5651 ReplSetRequestVotesArgs args;
5652 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5653 << "rs0"
5654 << "dryRun"
5655 << true
5656 << "term"
5657 << 1LL
5658 << "candidateIndex"
5659 << 0LL
5660 << "configVersion"
5661 << 1LL
5662 << "lastCommittedOp"
5663 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5664 .transitional_ignore();
5665 ReplSetRequestVotesResponse response;
5666
5667 getTopoCoord().processReplSetRequestVotes(args, &response);
5668 ASSERT_EQUALS("", response.getReason());
5669 ASSERT_TRUE(response.getVoteGranted());
5670
5671 // second dry run fine
5672 ReplSetRequestVotesArgs args2;
5673 args2
5674 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
5675 << "rs0"
5676 << "dryRun"
5677 << true
5678 << "term"
5679 << 1LL
5680 << "candidateIndex"
5681 << 0LL
5682 << "configVersion"
5683 << 1LL
5684 << "lastCommittedOp"
5685 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5686 .transitional_ignore();
5687 ReplSetRequestVotesResponse response2;
5688
5689 getTopoCoord().processReplSetRequestVotes(args2, &response2);
5690 ASSERT_EQUALS("", response2.getReason());
5691 ASSERT_TRUE(response2.getVoteGranted());
5692 }
5693
TEST_F(TopoCoordTest,VoteRequestShouldNotPreventDryRunsForThatTerm)5694 TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) {
5695 updateConfig(BSON("_id"
5696 << "rs0"
5697 << "version"
5698 << 1
5699 << "members"
5700 << BSON_ARRAY(BSON("_id" << 10 << "host"
5701 << "hself")
5702 << BSON("_id" << 20 << "host"
5703 << "h2")
5704 << BSON("_id" << 30 << "host"
5705 << "h3"))),
5706 0);
5707 setSelfMemberState(MemberState::RS_SECONDARY);
5708
5709 // real request fine
5710 ReplSetRequestVotesArgs args;
5711 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5712 << "rs0"
5713 << "dryRun"
5714 << false
5715 << "term"
5716 << 1LL
5717 << "candidateIndex"
5718 << 0LL
5719 << "configVersion"
5720 << 1LL
5721 << "lastCommittedOp"
5722 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5723 .transitional_ignore();
5724 ReplSetRequestVotesResponse response;
5725
5726 getTopoCoord().processReplSetRequestVotes(args, &response);
5727 ASSERT_EQUALS("", response.getReason());
5728 ASSERT_TRUE(response.getVoteGranted());
5729
5730 // dry post real, fails
5731 ReplSetRequestVotesArgs args2;
5732 args2
5733 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
5734 << "rs0"
5735 << "dryRun"
5736 << false
5737 << "term"
5738 << 1LL
5739 << "candidateIndex"
5740 << 0LL
5741 << "configVersion"
5742 << 1LL
5743 << "lastCommittedOp"
5744 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5745 .transitional_ignore();
5746 ReplSetRequestVotesResponse response2;
5747
5748 getTopoCoord().processReplSetRequestVotes(args2, &response2);
5749 ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)",
5750 response2.getReason());
5751 ASSERT_FALSE(response2.getVoteGranted());
5752 }
5753
TEST_F(TopoCoordTest,NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch)5754 TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch) {
5755 updateConfig(BSON("_id"
5756 << "rs0"
5757 << "version"
5758 << 1
5759 << "members"
5760 << BSON_ARRAY(BSON("_id" << 10 << "host"
5761 << "hself")
5762 << BSON("_id" << 20 << "host"
5763 << "h2")
5764 << BSON("_id" << 30 << "host"
5765 << "h3"))),
5766 0);
5767 setSelfMemberState(MemberState::RS_SECONDARY);
5768
5769 // mismatched setName
5770 ReplSetRequestVotesArgs args;
5771 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5772 << "wrongName"
5773 << "term"
5774 << 1LL
5775 << "candidateIndex"
5776 << 0LL
5777 << "configVersion"
5778 << 1LL
5779 << "lastCommittedOp"
5780 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5781 .transitional_ignore();
5782 ReplSetRequestVotesResponse response;
5783
5784 getTopoCoord().processReplSetRequestVotes(args, &response);
5785 ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason());
5786 ASSERT_FALSE(response.getVoteGranted());
5787 }
5788
TEST_F(TopoCoordTest,NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch)5789 TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) {
5790 updateConfig(BSON("_id"
5791 << "rs0"
5792 << "version"
5793 << 1
5794 << "members"
5795 << BSON_ARRAY(BSON("_id" << 10 << "host"
5796 << "hself")
5797 << BSON("_id" << 20 << "host"
5798 << "h2")
5799 << BSON("_id" << 30 << "host"
5800 << "h3"))),
5801 0);
5802 setSelfMemberState(MemberState::RS_SECONDARY);
5803
5804 // mismatched configVersion
5805 ReplSetRequestVotesArgs args;
5806 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5807 << "rs0"
5808 << "term"
5809 << 1LL
5810 << "candidateIndex"
5811 << 1LL
5812 << "configVersion"
5813 << 0LL
5814 << "lastCommittedOp"
5815 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5816 .transitional_ignore();
5817 ReplSetRequestVotesResponse response;
5818
5819 getTopoCoord().processReplSetRequestVotes(args, &response);
5820 ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason());
5821 ASSERT_FALSE(response.getVoteGranted());
5822 }
5823
TEST_F(TopoCoordTest,ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualOrGreaterPriority)5824 TEST_F(TopoCoordTest, ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualOrGreaterPriority) {
5825 updateConfig(BSON("_id"
5826 << "rs0"
5827 << "version"
5828 << 1
5829 << "members"
5830 << BSON_ARRAY(BSON("_id" << 10 << "host"
5831 << "hself"
5832 << "arbiterOnly"
5833 << true)
5834 << BSON("_id" << 20 << "host"
5835 << "h2"
5836 << "priority"
5837 << 5)
5838 << BSON("_id" << 30 << "host"
5839 << "h3"))),
5840 0);
5841 heartbeatFromMember(HostAndPort("h2"),
5842 "rs0",
5843 MemberState::RS_PRIMARY,
5844 OpTime(Timestamp(0, 0), 0),
5845 Milliseconds(300));
5846 heartbeatFromMember(HostAndPort("h3"),
5847 "rs0",
5848 MemberState::RS_SECONDARY,
5849 OpTime(Timestamp(0, 0), 0),
5850 Milliseconds(300));
5851
5852 ReplSetRequestVotesArgs args;
5853 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5854 << "rs0"
5855 << "term"
5856 << 1LL
5857 << "candidateIndex"
5858 << 2LL
5859 << "configVersion"
5860 << 1LL
5861 << "lastCommittedOp"
5862 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5863 .transitional_ignore();
5864 ReplSetRequestVotesResponse response;
5865
5866 getTopoCoord().processReplSetRequestVotes(args, &response);
5867 ASSERT_EQUALS("can see a healthy primary (h2:27017) of equal or greater priority",
5868 response.getReason());
5869 ASSERT_FALSE(response.getVoteGranted());
5870 }
5871
TEST_F(TopoCoordTest,NodeDoesNotGrantVoteWhenTermIsStale)5872 TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) {
5873 updateConfig(BSON("_id"
5874 << "rs0"
5875 << "version"
5876 << 1
5877 << "members"
5878 << BSON_ARRAY(BSON("_id" << 10 << "host"
5879 << "hself")
5880 << BSON("_id" << 20 << "host"
5881 << "h2")
5882 << BSON("_id" << 30 << "host"
5883 << "h3"))),
5884 0);
5885 setSelfMemberState(MemberState::RS_SECONDARY);
5886
5887 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
5888 getTopoCoord().updateTerm(2, now()));
5889 ASSERT_EQUALS(2, getTopoCoord().getTerm());
5890
5891 // stale term
5892 ReplSetRequestVotesArgs args;
5893 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5894 << "rs0"
5895 << "term"
5896 << 1LL
5897 << "candidateIndex"
5898 << 1LL
5899 << "configVersion"
5900 << 1LL
5901 << "lastCommittedOp"
5902 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5903 .transitional_ignore();
5904 ReplSetRequestVotesResponse response;
5905
5906 getTopoCoord().processReplSetRequestVotes(args, &response);
5907 ASSERT_EQUALS("candidate's term (1) is lower than mine (2)", response.getReason());
5908 ASSERT_EQUALS(2, response.getTerm());
5909 ASSERT_FALSE(response.getVoteGranted());
5910 }
5911
TEST_F(TopoCoordTest,NodeDoesNotGrantVoteWhenOpTimeIsStale)5912 TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenOpTimeIsStale) {
5913 updateConfig(BSON("_id"
5914 << "rs0"
5915 << "version"
5916 << 1
5917 << "members"
5918 << BSON_ARRAY(BSON("_id" << 10 << "host"
5919 << "hself")
5920 << BSON("_id" << 20 << "host"
5921 << "h2")
5922 << BSON("_id" << 30 << "host"
5923 << "h3"))),
5924 0);
5925 setSelfMemberState(MemberState::RS_SECONDARY);
5926
5927
5928 // stale OpTime
5929 ReplSetRequestVotesArgs args;
5930 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5931 << "rs0"
5932 << "term"
5933 << 3LL
5934 << "candidateIndex"
5935 << 1LL
5936 << "configVersion"
5937 << 1LL
5938 << "lastCommittedOp"
5939 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5940 .transitional_ignore();
5941 ReplSetRequestVotesResponse response;
5942
5943 getTopoCoord().getMyMemberData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, Date_t());
5944 getTopoCoord().processReplSetRequestVotes(args, &response);
5945 ASSERT_EQUALS(
5946 str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: "
5947 << OpTime().toString()
5948 << ", my last applied OpTime: "
5949 << OpTime(Timestamp(20, 0), 0).toString(),
5950 response.getReason());
5951 ASSERT_FALSE(response.getVoteGranted());
5952 }
5953
TEST_F(TopoCoordTest,NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch)5954 TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) {
5955 updateConfig(BSON("_id"
5956 << "rs0"
5957 << "version"
5958 << 1
5959 << "members"
5960 << BSON_ARRAY(BSON("_id" << 10 << "host"
5961 << "hself")
5962 << BSON("_id" << 20 << "host"
5963 << "h2")
5964 << BSON("_id" << 30 << "host"
5965 << "h3"))),
5966 0);
5967 setSelfMemberState(MemberState::RS_SECONDARY);
5968 // set term to 1
5969 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
5970 getTopoCoord().updateTerm(1, now()));
5971 // and make sure we voted in term 1
5972 ReplSetRequestVotesArgs argsForRealVote;
5973 argsForRealVote
5974 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
5975 << "rs0"
5976 << "term"
5977 << 1LL
5978 << "candidateIndex"
5979 << 0LL
5980 << "configVersion"
5981 << 1LL
5982 << "lastCommittedOp"
5983 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
5984 .transitional_ignore();
5985 ReplSetRequestVotesResponse responseForRealVote;
5986
5987 getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote);
5988 ASSERT_EQUALS("", responseForRealVote.getReason());
5989 ASSERT_TRUE(responseForRealVote.getVoteGranted());
5990
5991
5992 // mismatched setName
5993 ReplSetRequestVotesArgs args;
5994 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
5995 << "wrongName"
5996 << "dryRun"
5997 << true
5998 << "term"
5999 << 2LL
6000 << "candidateIndex"
6001 << 0LL
6002 << "configVersion"
6003 << 1LL
6004 << "lastCommittedOp"
6005 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6006 .transitional_ignore();
6007 ReplSetRequestVotesResponse response;
6008
6009 getTopoCoord().processReplSetRequestVotes(args, &response);
6010 ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason());
6011 ASSERT_EQUALS(1, response.getTerm());
6012 ASSERT_FALSE(response.getVoteGranted());
6013 }
6014
TEST_F(TopoCoordTest,NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch)6015 TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) {
6016 updateConfig(BSON("_id"
6017 << "rs0"
6018 << "version"
6019 << 1
6020 << "members"
6021 << BSON_ARRAY(BSON("_id" << 10 << "host"
6022 << "hself")
6023 << BSON("_id" << 20 << "host"
6024 << "h2")
6025 << BSON("_id" << 30 << "host"
6026 << "h3"))),
6027 0);
6028 setSelfMemberState(MemberState::RS_SECONDARY);
6029 // set term to 1
6030 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
6031 getTopoCoord().updateTerm(1, now()));
6032 // and make sure we voted in term 1
6033 ReplSetRequestVotesArgs argsForRealVote;
6034 argsForRealVote
6035 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
6036 << "rs0"
6037 << "term"
6038 << 1LL
6039 << "candidateIndex"
6040 << 0LL
6041 << "configVersion"
6042 << 1LL
6043 << "lastCommittedOp"
6044 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6045 .transitional_ignore();
6046 ReplSetRequestVotesResponse responseForRealVote;
6047
6048 getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote);
6049 ASSERT_EQUALS("", responseForRealVote.getReason());
6050 ASSERT_TRUE(responseForRealVote.getVoteGranted());
6051
6052
6053 // mismatched configVersion
6054 ReplSetRequestVotesArgs args;
6055 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
6056 << "rs0"
6057 << "dryRun"
6058 << true
6059 << "term"
6060 << 2LL
6061 << "candidateIndex"
6062 << 1LL
6063 << "configVersion"
6064 << 0LL
6065 << "lastCommittedOp"
6066 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6067 .transitional_ignore();
6068 ReplSetRequestVotesResponse response;
6069
6070 getTopoCoord().processReplSetRequestVotes(args, &response);
6071 ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason());
6072 ASSERT_EQUALS(1, response.getTerm());
6073 ASSERT_FALSE(response.getVoteGranted());
6074 }
6075
TEST_F(TopoCoordTest,NodeDoesNotGrantDryRunVoteWhenTermIsStale)6076 TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) {
6077 updateConfig(BSON("_id"
6078 << "rs0"
6079 << "version"
6080 << 1
6081 << "members"
6082 << BSON_ARRAY(BSON("_id" << 10 << "host"
6083 << "hself")
6084 << BSON("_id" << 20 << "host"
6085 << "h2")
6086 << BSON("_id" << 30 << "host"
6087 << "h3"))),
6088 0);
6089 setSelfMemberState(MemberState::RS_SECONDARY);
6090 // set term to 1
6091 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
6092 getTopoCoord().updateTerm(1, now()));
6093 // and make sure we voted in term 1
6094 ReplSetRequestVotesArgs argsForRealVote;
6095 argsForRealVote
6096 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
6097 << "rs0"
6098 << "term"
6099 << 1LL
6100 << "candidateIndex"
6101 << 0LL
6102 << "configVersion"
6103 << 1LL
6104 << "lastCommittedOp"
6105 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6106 .transitional_ignore();
6107 ReplSetRequestVotesResponse responseForRealVote;
6108
6109 getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote);
6110 ASSERT_EQUALS("", responseForRealVote.getReason());
6111 ASSERT_TRUE(responseForRealVote.getVoteGranted());
6112
6113 // stale term
6114 ReplSetRequestVotesArgs args;
6115 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
6116 << "rs0"
6117 << "dryRun"
6118 << true
6119 << "term"
6120 << 0LL
6121 << "candidateIndex"
6122 << 1LL
6123 << "configVersion"
6124 << 1LL
6125 << "lastCommittedOp"
6126 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6127 .transitional_ignore();
6128 ReplSetRequestVotesResponse response;
6129
6130 getTopoCoord().processReplSetRequestVotes(args, &response);
6131 ASSERT_EQUALS("candidate's term (0) is lower than mine (1)", response.getReason());
6132 ASSERT_EQUALS(1, response.getTerm());
6133 ASSERT_FALSE(response.getVoteGranted());
6134 }
6135
TEST_F(TopoCoordTest,GrantDryRunVoteEvenWhenTermHasBeenSeen)6136 TEST_F(TopoCoordTest, GrantDryRunVoteEvenWhenTermHasBeenSeen) {
6137 updateConfig(BSON("_id"
6138 << "rs0"
6139 << "version"
6140 << 1
6141 << "members"
6142 << BSON_ARRAY(BSON("_id" << 10 << "host"
6143 << "hself")
6144 << BSON("_id" << 20 << "host"
6145 << "h2")
6146 << BSON("_id" << 30 << "host"
6147 << "h3"))),
6148 0);
6149 setSelfMemberState(MemberState::RS_SECONDARY);
6150 // set term to 1
6151 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
6152 getTopoCoord().updateTerm(1, now()));
6153 // and make sure we voted in term 1
6154 ReplSetRequestVotesArgs argsForRealVote;
6155 argsForRealVote
6156 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
6157 << "rs0"
6158 << "term"
6159 << 1LL
6160 << "candidateIndex"
6161 << 0LL
6162 << "configVersion"
6163 << 1LL
6164 << "lastCommittedOp"
6165 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6166 .transitional_ignore();
6167 ReplSetRequestVotesResponse responseForRealVote;
6168
6169 getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote);
6170 ASSERT_EQUALS("", responseForRealVote.getReason());
6171 ASSERT_TRUE(responseForRealVote.getVoteGranted());
6172
6173
6174 // repeat term
6175 ReplSetRequestVotesArgs args;
6176 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
6177 << "rs0"
6178 << "dryRun"
6179 << true
6180 << "term"
6181 << 1LL
6182 << "candidateIndex"
6183 << 1LL
6184 << "configVersion"
6185 << 1LL
6186 << "lastCommittedOp"
6187 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6188 .transitional_ignore();
6189 ReplSetRequestVotesResponse response;
6190
6191 getTopoCoord().processReplSetRequestVotes(args, &response);
6192 ASSERT_EQUALS("", response.getReason());
6193 ASSERT_EQUALS(1, response.getTerm());
6194 ASSERT_TRUE(response.getVoteGranted());
6195 }
6196
TEST_F(TopoCoordTest,DoNotGrantDryRunVoteWhenOpTimeIsStale)6197 TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) {
6198 updateConfig(BSON("_id"
6199 << "rs0"
6200 << "version"
6201 << 1
6202 << "members"
6203 << BSON_ARRAY(BSON("_id" << 10 << "host"
6204 << "hself")
6205 << BSON("_id" << 20 << "host"
6206 << "h2")
6207 << BSON("_id" << 30 << "host"
6208 << "h3"))),
6209 0);
6210 setSelfMemberState(MemberState::RS_SECONDARY);
6211 // set term to 1
6212 ASSERT(TopologyCoordinator::UpdateTermResult::kUpdatedTerm ==
6213 getTopoCoord().updateTerm(1, now()));
6214 // and make sure we voted in term 1
6215 ReplSetRequestVotesArgs argsForRealVote;
6216 argsForRealVote
6217 .initialize(BSON("replSetRequestVotes" << 1 << "setName"
6218 << "rs0"
6219 << "term"
6220 << 1LL
6221 << "candidateIndex"
6222 << 0LL
6223 << "configVersion"
6224 << 1LL
6225 << "lastCommittedOp"
6226 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6227 .transitional_ignore();
6228 ReplSetRequestVotesResponse responseForRealVote;
6229
6230 getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote);
6231 ASSERT_EQUALS("", responseForRealVote.getReason());
6232 ASSERT_TRUE(responseForRealVote.getVoteGranted());
6233
6234
6235 // stale OpTime
6236 ReplSetRequestVotesArgs args;
6237 args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
6238 << "rs0"
6239 << "dryRun"
6240 << true
6241 << "term"
6242 << 3LL
6243 << "candidateIndex"
6244 << 1LL
6245 << "configVersion"
6246 << 1LL
6247 << "lastCommittedOp"
6248 << BSON("ts" << Timestamp(10, 0) << "term" << 0LL)))
6249 .transitional_ignore();
6250 ReplSetRequestVotesResponse response;
6251
6252 getTopoCoord().getMyMemberData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, Date_t());
6253 getTopoCoord().processReplSetRequestVotes(args, &response);
6254 ASSERT_EQUALS(
6255 str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: "
6256 << OpTime().toString()
6257 << ", my last applied OpTime: "
6258 << OpTime(Timestamp(20, 0), 0).toString(),
6259 response.getReason());
6260 ASSERT_EQUALS(1, response.getTerm());
6261 ASSERT_FALSE(response.getVoteGranted());
6262 }
6263
TEST_F(TopoCoordTest,CSRSConfigServerRejectsPV0Config)6264 TEST_F(TopoCoordTest, CSRSConfigServerRejectsPV0Config) {
6265 ON_BLOCK_EXIT([]() { serverGlobalParams.clusterRole = ClusterRole::None; });
6266 serverGlobalParams.clusterRole = ClusterRole::ConfigServer;
6267 TopologyCoordinator::Options options;
6268 options.clusterRole = ClusterRole::ConfigServer;
6269 setOptions(options);
6270 getTopoCoord().setStorageEngineSupportsReadCommitted(false);
6271
6272 auto configObj = BSON("_id"
6273 << "rs0"
6274 << "version"
6275 << 1
6276 << "configsvr"
6277 << true
6278 << "members"
6279 << BSON_ARRAY(BSON("_id" << 10 << "host"
6280 << "hself")
6281 << BSON("_id" << 20 << "host"
6282 << "h2")
6283 << BSON("_id" << 30 << "host"
6284 << "h3")));
6285 ReplSetConfig config;
6286 ASSERT_OK(config.initialize(configObj, false));
6287 ASSERT_EQ(ErrorCodes::BadValue, config.validate());
6288 }
6289
6290 } // namespace
6291 } // namespace repl
6292 } // namespace mongo
6293