1 /*
2 * Copyright (C) 2009-2020 Codership Oy <info@codership.com>
3 */
4
5 /*!
6 * @file Unit tests for refactored EVS
7 */
8
9
10
11 #include "evs_proto.hpp"
12 #include "evs_input_map2.hpp"
13 #include "evs_message2.hpp"
14 #include "evs_seqno.hpp"
15
16 #include "check_gcomm.hpp"
17 #include "check_templ.hpp"
18 #include "check_trace.hpp"
19
20 #include "gcomm/conf.hpp"
21
22 #include "gu_asio.hpp" // gu::ssl_register_params()
23
24 #include <stdexcept>
25 #include <vector>
26 #include <set>
27
28 #include "check.h"
29
30 //
31 // set GALERA_TEST_DETERMINISTIC env
32 // variable before running pc test suite.
33 //
34 static class deterministic_tests
35 {
36 public:
deterministic_tests()37 deterministic_tests()
38 : deterministic_tests_()
39 {
40 if (::getenv("GALERA_TEST_DETERMINISTIC"))
41 {
42 deterministic_tests_ = true;
43 }
44 else
45 {
46 deterministic_tests_ = false;
47 }
48 }
49
operator ()() const50 bool operator()() const { return deterministic_tests_; }
51
52 private:
53 bool deterministic_tests_;
54 } deterministic_tests;
55
56 using namespace std;
57 using namespace std::rel_ops;
58 using namespace gu::datetime;
59 using namespace gcomm;
60 using namespace gcomm::evs;
61 using gu::DeleteObject;
62
init_rand()63 void init_rand()
64 {
65 unsigned int seed(static_cast<unsigned int>(time(0)));
66 log_info << "rand seed " << seed;
67 srand(seed);
68 }
69
init_rand(unsigned int seed)70 void init_rand(unsigned int seed)
71 {
72 log_info << "rand seed " << seed;
73 srand(seed);
74 }
75
76
START_TEST(test_range)77 START_TEST(test_range)
78 {
79 log_info << "START";
80 Range r(3, 6);
81
82 check_serialization(r, 2 * sizeof(seqno_t), Range());
83
84 }
85 END_TEST
86
START_TEST(test_message)87 START_TEST(test_message)
88 {
89 log_info << "START";
90 UUID uuid1(0, 0);
91 ViewId view_id(V_TRANS, uuid1, 4567);
92 seqno_t seq(478), aru_seq(456), seq_range(7);
93
94 UserMessage um(0, uuid1, view_id, seq, aru_seq, seq_range, O_SAFE, 75433, 0xab,
95 Message::F_SOURCE);
96 ck_assert(um.serial_size() % 4 == 0);
97 check_serialization(um, um.serial_size(), UserMessage());
98
99 AggregateMessage am(0xab, 17457, 0x79);
100 check_serialization(am, 4, AggregateMessage());
101
102 DelegateMessage dm(0, uuid1, view_id);
103 dm.set_source(uuid1);
104 check_serialization(dm, dm.serial_size(), DelegateMessage());
105
106 MessageNodeList node_list;
107 node_list.insert(make_pair(uuid1, MessageNode()));
108 node_list.insert(make_pair(UUID(2), MessageNode(true, false, 254, true, 1,
109 ViewId(V_REG), 5,
110 Range(7, 8))));
111 JoinMessage jm(0, uuid1, view_id, 8, 5, 27, node_list);
112 jm.set_source(uuid1);
113 check_serialization(jm, jm.serial_size(), JoinMessage());
114
115 InstallMessage im(0, uuid1, view_id, ViewId(V_REG, view_id.uuid(),
116 view_id.seq()), 8, 5, 27, node_list);
117 im.set_source(uuid1);
118 check_serialization(im, im.serial_size(), InstallMessage());
119
120 LeaveMessage lm(0, uuid1, view_id, 45, 88, 3456);
121 lm.set_source(uuid1);
122 check_serialization(lm, lm.serial_size(), LeaveMessage());
123
124
125 DelayedListMessage dlm(0, uuid1, view_id, 4576);
126 dlm.add(UUID(2), 23);
127 dlm.add(UUID(3), 45);
128 dlm.add(UUID(5), 255);
129 check_serialization(dlm, dlm.serial_size(), DelayedListMessage());
130 }
131 END_TEST
132
START_TEST(test_input_map_insert)133 START_TEST(test_input_map_insert)
134 {
135 log_info << "START";
136 UUID uuid1(1), uuid2(2);
137 InputMap im;
138 ViewId view(V_REG, uuid1, 0);
139
140 try
141 {
142 im.insert(0, UserMessage(0, uuid1, view, 0));
143 ck_abort_msg("Exception not thrown, input map has not been "
144 "reset/initialized yet");
145 }
146 catch (...)
147 { }
148
149 im.reset(1);
150
151 im.insert(0, UserMessage(0, uuid1, view, 0));
152
153
154 im.clear();
155 im.reset(2);
156
157 for (seqno_t s = 0; s < 10; ++s)
158 {
159 im.insert(0, UserMessage(0, uuid1, view, s));
160 im.insert(1, UserMessage(0, uuid2, view, s));
161 }
162
163 for (seqno_t s = 0; s < 10; ++s)
164 {
165 InputMap::iterator i = im.find(0, s);
166 ck_assert(i != im.end());
167 ck_assert(InputMapMsgIndex::value(i).msg().source() == uuid1);
168 ck_assert(InputMapMsgIndex::value(i).msg().seq() == s);
169
170 i = im.find(1, s);
171 ck_assert(i != im.end());
172 ck_assert(InputMapMsgIndex::value(i).msg().source() == uuid2);
173 ck_assert(InputMapMsgIndex::value(i).msg().seq() == s);
174 }
175
176 }
177 END_TEST
178
START_TEST(test_input_map_find)179 START_TEST(test_input_map_find)
180 {
181 log_info << "START";
182 InputMap im;
183 UUID uuid1(1);
184 ViewId view(V_REG, uuid1, 0);
185
186 im.reset(1);
187
188 im.insert(0, UserMessage(0, uuid1, view, 0));
189
190 ck_assert(im.find(0, 0) != im.end());
191
192
193 im.insert(0, UserMessage(0, uuid1, view, 2));
194 im.insert(0, UserMessage(0, uuid1, view, 4));
195 im.insert(0, UserMessage(0, uuid1, view, 7));
196
197 ck_assert(im.find(0, 2) != im.end());
198 ck_assert(im.find(0, 4) != im.end());
199 ck_assert(im.find(0, 7) != im.end());
200
201 ck_assert(im.find(0, 3) == im.end());
202 ck_assert(im.find(0, 5) == im.end());
203 ck_assert(im.find(0, 6) == im.end());
204 ck_assert(im.find(0, 8) == im.end());
205 }
206 END_TEST
207
START_TEST(test_input_map_safety)208 START_TEST(test_input_map_safety)
209 {
210 log_info << "START";
211 InputMap im;
212 UUID uuid1(1);
213 size_t index1(0);
214 ViewId view(V_REG, uuid1, 0);
215
216 im.reset(1);
217
218 im.insert(index1, UserMessage(0, uuid1, view, 0));
219 ck_assert(im.aru_seq() == 0);
220 im.insert(index1, UserMessage(0, uuid1, view, 1));
221 ck_assert(im.aru_seq() == 1);
222 im.insert(index1, UserMessage(0, uuid1, view, 2));
223 ck_assert(im.aru_seq() == 2);
224 im.insert(index1, UserMessage(0, uuid1, view, 3));
225 ck_assert(im.aru_seq() == 3);
226 im.insert(index1, UserMessage(0, uuid1, view, 5));
227 ck_assert(im.aru_seq() == 3);
228
229 im.insert(index1, UserMessage(0, uuid1, view, 4));
230 ck_assert(im.aru_seq() == 5);
231
232 InputMap::iterator i = im.find(index1, 0);
233 ck_assert(im.is_fifo(i) == true);
234 ck_assert(im.is_agreed(i) == true);
235 ck_assert(im.is_safe(i) == false);
236 im.set_safe_seq(index1, 0);
237 ck_assert(im.is_safe(i) == true);
238
239 im.set_safe_seq(index1, 5);
240 i = im.find(index1, 5);
241 ck_assert(im.is_safe(i) == true);
242
243 im.insert(index1, UserMessage(0, uuid1, view, 7));
244 im.set_safe_seq(index1, im.aru_seq());
245 i = im.find(index1, 7);
246 ck_assert(im.is_safe(i) == false);
247
248 }
249 END_TEST
250
START_TEST(test_input_map_erase)251 START_TEST(test_input_map_erase)
252 {
253 log_info << "START";
254 InputMap im;
255 size_t index1(0);
256 UUID uuid1(1);
257 ViewId view(V_REG, uuid1, 1);
258
259 im.reset(1);
260
261 for (seqno_t s = 0; s < 10; ++s)
262 {
263 im.insert(index1, UserMessage(0, uuid1, view, s));
264 }
265
266 for (seqno_t s = 0; s < 10; ++s)
267 {
268 InputMap::iterator i = im.find(index1, s);
269 ck_assert(i != im.end());
270 im.erase(i);
271 i = im.find(index1, s);
272 ck_assert(i == im.end());
273 (void)im.recover(index1, s);
274 }
275 im.set_safe_seq(index1, 9);
276 try
277 {
278 im.recover(index1, 9);
279 ck_abort_msg("Exception not thrown, "
280 "setting safe seq should purge index");
281 }
282 catch (...) { }
283 }
284 END_TEST
285
START_TEST(test_input_map_overwrap)286 START_TEST(test_input_map_overwrap)
287 {
288 log_info << "START";
289 InputMap im;
290 const size_t n_nodes(5);
291 ViewId view(V_REG, UUID(1), 1);
292 vector<UUID> uuids;
293 for (size_t n = 0; n < n_nodes; ++n)
294 {
295 uuids.push_back(UUID(static_cast<int32_t>(n + 1)));
296 }
297
298 im.reset(n_nodes);
299
300
301 Date start(Date::monotonic());
302 size_t cnt(0);
303 seqno_t last_safe(-1);
304 for (seqno_t seq = 0; seq < 100000; ++seq)
305 {
306 for (size_t i = 0; i < n_nodes; ++i)
307 {
308 UserMessage um(0, uuids[i], view, seq);
309 (void)im.insert(i, um);
310 if ((seq + 5) % 10 == 0)
311 {
312 last_safe = um.seq() - 3;
313 im.set_safe_seq(i, last_safe);
314 for (InputMap::iterator ii = im.begin();
315 ii != im.end() && im.is_safe(ii) == true;
316 ii = im.begin())
317 {
318 im.erase(ii);
319 }
320 }
321 cnt++;
322 }
323 gcomm_assert(im.aru_seq() == seq);
324 gcomm_assert(im.safe_seq() == last_safe);
325 }
326 Date stop(Date::monotonic());
327
328 double div(double(stop.get_utc() - start.get_utc())/gu::datetime::Sec);
329 log_info << "input map msg rate " << double(cnt)/div;
330 }
331 END_TEST
332
333
334 class InputMapInserter
335 {
336 public:
InputMapInserter(InputMap & im)337 InputMapInserter(InputMap& im) : im_(im) { }
338
operator ()(const pair<size_t,UserMessage> & p) const339 void operator()(const pair<size_t, UserMessage>& p) const
340 {
341 im_.insert(p.first, p.second);
342 }
343 private:
344 InputMap& im_;
345 };
346
START_TEST(test_input_map_random_insert)347 START_TEST(test_input_map_random_insert)
348 {
349 log_info << "START";
350 init_rand();
351 seqno_t n_seqnos(1024);
352 size_t n_uuids(4);
353 vector<UUID> uuids(n_uuids);
354 vector<pair<size_t, UserMessage> > msgs(static_cast<size_t>(n_uuids*n_seqnos));
355 ViewId view_id(V_REG, UUID(1), 1);
356 InputMap im;
357
358 for (size_t i = 0; i < n_uuids; ++i)
359 {
360 uuids[i] = (static_cast<int32_t>(i + 1));
361 }
362
363 im.reset(n_uuids);
364
365 for (seqno_t j = 0; j < n_seqnos; ++j)
366 {
367 for (size_t i = 0; i < n_uuids; ++i)
368 {
369 msgs[static_cast<size_t>(j*n_uuids) + i] =
370 make_pair(i, UserMessage(0, uuids[i], view_id, j));
371 }
372 }
373
374 vector<pair<size_t, UserMessage> > random_msgs(msgs);
375 random_shuffle(random_msgs.begin(), random_msgs.end());
376 for_each(random_msgs.begin(), random_msgs.end(), InputMapInserter(im));
377
378 size_t n = 0;
379 for (InputMap::iterator i = im.begin(); i != im.end(); ++i)
380 {
381 const InputMapMsg& msg(InputMapMsgIndex::value(i));
382 ck_assert(msg.msg() == msgs[n].second);
383 ck_assert(im.is_safe(i) == false);
384 ++n;
385 }
386
387 ck_assert(im.aru_seq() == n_seqnos - 1);
388 ck_assert(im.safe_seq() == -1);
389
390 for (size_t i = 0; i < n_uuids; ++i)
391 {
392 ck_assert(im.range(i) ==
393 Range(n_seqnos,
394 n_seqnos - 1));
395
396 im.set_safe_seq(i, n_seqnos - 1);
397 }
398 ck_assert(im.safe_seq() == n_seqnos - 1);
399
400 }
401 END_TEST
402
START_TEST(test_input_map_gap_range_list)403 START_TEST(test_input_map_gap_range_list)
404 {
405 gcomm::evs::InputMap im;
406 im.reset(1);
407 gcomm::UUID uuid(1);
408 gcomm::ViewId view_id(gcomm::V_REG, uuid, 1);
409 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 0, 0));
410 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 2, 0));
411
412 std::vector<gcomm::evs::Range> gap_range(
413 im.gap_range_list(0, gcomm::evs::Range(0, 2)));
414 ck_assert(gap_range.size() == 1);
415 ck_assert(gap_range.begin()->lu() == 1);
416 ck_assert(gap_range.begin()->hs() == 1);
417
418 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 4, 0));
419 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 4));
420 ck_assert(gap_range.size() == 2);
421 ck_assert(gap_range.begin()->lu() == 1);
422 ck_assert(gap_range.begin()->hs() == 1);
423 ck_assert(gap_range.rbegin()->lu() == 3);
424 ck_assert(gap_range.rbegin()->hs() == 3);
425
426 // Although there are two messages missing, limiting the range to 0,2
427 // should return only the first one.
428 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 2));
429 ck_assert(gap_range.size() == 1);
430 ck_assert(gap_range.begin()->lu() == 1);
431 ck_assert(gap_range.begin()->hs() == 1);
432
433 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 8, 0));
434 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
435 ck_assert(gap_range.size() == 3);
436 ck_assert(gap_range.begin()->lu() == 1);
437 ck_assert(gap_range.begin()->hs() == 1);
438 ck_assert(gap_range.rbegin()->lu() == 5);
439 ck_assert(gap_range.rbegin()->hs() == 7);
440
441 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 3, 0));
442 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
443 ck_assert(gap_range.size() == 2);
444 ck_assert(gap_range.begin()->lu() == 1);
445 ck_assert(gap_range.begin()->hs() == 1);
446 ck_assert(gap_range.rbegin()->lu() == 5);
447 ck_assert(gap_range.rbegin()->hs() == 7);
448
449 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 1, 0));
450 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 5, 0));
451 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 6, 0));
452 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 7, 0));
453 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
454 ck_assert(gap_range.empty());
455 }
456 END_TEST
457
get_msg(DummyTransport * tp,Message * msg,bool release=true)458 static Datagram* get_msg(DummyTransport* tp, Message* msg, bool release = true)
459 {
460 Datagram* rb = tp->out();
461 if (rb != 0)
462 {
463 gu_trace(Proto::unserialize_message(tp->uuid(), *rb, msg));
464 if (release == true)
465 {
466 delete rb;
467 }
468 }
469 return rb;
470 }
471
single_join(DummyTransport * t,Proto * p)472 static void single_join(DummyTransport* t, Proto* p)
473 {
474 Message jm, im, gm;
475
476 // Initial state is joining
477 p->shift_to(Proto::S_JOINING);
478
479 // Send join must produce emitted join message
480 p->send_join();
481
482 Datagram* rb = get_msg(t, &jm);
483 ck_assert(rb != 0);
484 ck_assert(jm.type() == Message::EVS_T_JOIN);
485
486 // Install message is emitted at the end of JOIN handling
487 // 'cause this is the only instance and is always consistent
488 // with itself
489 rb = get_msg(t, &im);
490 ck_assert(rb != 0);
491 ck_assert(im.type() == Message::EVS_T_INSTALL);
492
493 // Handling INSTALL message emits three gap messages,
494 // one for receiving install message (commit gap), one for
495 // shift to install and one for shift to operational
496 rb = get_msg(t, &gm);
497 ck_assert(rb != 0);
498 ck_assert(gm.type() == Message::EVS_T_GAP);
499 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
500
501 rb = get_msg(t, &gm);
502 ck_assert(rb != 0);
503 ck_assert(gm.type() == Message::EVS_T_GAP);
504 ck_assert((gm.flags() & Message::F_COMMIT) == 0);
505
506 rb = get_msg(t, &gm);
507 ck_assert(rb != 0);
508 ck_assert(gm.type() == Message::EVS_T_GAP);
509 ck_assert((gm.flags() & Message::F_COMMIT) == 0);
510
511 // State must have evolved JOIN -> S_GATHER -> S_INSTALL -> S_OPERATIONAL
512 ck_assert(p->state() == Proto::S_OPERATIONAL);
513
514 // Handle join message again, must stay in S_OPERATIONAL, must not
515 // emit anything
516 p->handle_msg(jm);
517 rb = get_msg(t, &gm);
518 ck_assert(rb == 0);
519 ck_assert(p->state() == Proto::S_OPERATIONAL);
520
521 }
522
523 class DummyUser : public Toplay
524 {
525 public:
DummyUser(gu::Config & conf)526 DummyUser(gu::Config& conf) : Toplay(conf) { }
handle_up(const void *,const Datagram &,const ProtoUpMeta &)527 void handle_up(const void*, const Datagram&, const ProtoUpMeta&) { }
528 private:
529 };
530
531
START_TEST(test_proto_single_join)532 START_TEST(test_proto_single_join)
533 {
534 log_info << "START";
535 gu::Config conf;
536 mark_point();
537 gu::ssl_register_params(conf);
538 gcomm::Conf::register_params(conf);
539 UUID uuid(1);
540 DummyTransport t(uuid);
541 mark_point();
542 DummyUser u(conf);
543 mark_point();
544 Proto p(conf, uuid, 0);
545 mark_point();
546 gcomm::connect(&t, &p);
547 gcomm::connect(&p, &u);
548 single_join(&t, &p);
549 }
550 END_TEST
551
double_join(DummyTransport * t1,Proto * p1,DummyTransport * t2,Proto * p2)552 static void double_join(DummyTransport* t1, Proto* p1,
553 DummyTransport* t2, Proto* p2)
554 {
555
556 Message jm;
557 Message im;
558 Message gm;
559 Message gm2;
560 Message msg;
561
562 Datagram* rb;
563
564 // Initial states check
565 p2->shift_to(Proto::S_JOINING);
566 ck_assert(p1->state() == Proto::S_OPERATIONAL);
567 ck_assert(p2->state() == Proto::S_JOINING);
568
569 // Send join message, don't self handle immediately
570 // Expected output: one join message
571 p2->send_join(false);
572 ck_assert(p2->state() == Proto::S_JOINING);
573 rb = get_msg(t2, &jm);
574 ck_assert(rb != 0);
575 ck_assert(jm.type() == Message::EVS_T_JOIN);
576 rb = get_msg(t2, &msg);
577 ck_assert(rb == 0);
578
579 // Handle node 2's join on node 1
580 // Expected output: shift to S_GATHER and one join message
581 p1->handle_msg(jm);
582 ck_assert(p1->state() == Proto::S_GATHER);
583 rb = get_msg(t1, &jm);
584 ck_assert(rb != 0);
585 ck_assert(jm.type() == Message::EVS_T_JOIN);
586 rb = get_msg(t1, &msg);
587 ck_assert(rb == 0);
588
589 // Handle node 1's join on node 2
590 // Expected output: shift to S_GATHER and one join message
591 p2->handle_msg(jm);
592 ck_assert(p2->state() == Proto::S_GATHER);
593 rb = get_msg(t2, &jm);
594 ck_assert(rb != 0);
595 ck_assert(jm.type() == Message::EVS_T_JOIN);
596 rb = get_msg(t2, &msg);
597 ck_assert(rb == 0);
598
599 // Handle node 2's join on node 1
600 // Expected output: Install and commit gap messages, state stays in S_GATHER
601 p1->handle_msg(jm);
602 ck_assert(p1->state() == Proto::S_GATHER);
603 rb = get_msg(t1, &im);
604 ck_assert(rb != 0);
605 ck_assert(im.type() == Message::EVS_T_INSTALL);
606 rb = get_msg(t1, &gm);
607 ck_assert(rb != 0);
608 ck_assert(gm.type() == Message::EVS_T_GAP);
609 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
610 rb = get_msg(t1, &msg);
611 ck_assert(rb == 0);
612
613 // Handle install message on node 2
614 // Expected output: commit gap message and state stays in S_RECOVERY
615 p2->handle_msg(im);
616 ck_assert(p2->state() == Proto::S_GATHER);
617 rb = get_msg(t2, &gm2);
618 ck_assert(rb != 0);
619 ck_assert(gm2.type() == Message::EVS_T_GAP);
620 ck_assert((gm2.flags() & Message::F_COMMIT) != 0);
621 rb = get_msg(t2, &msg);
622 ck_assert(rb == 0);
623
624 // Handle gap messages
625 // Expected output: Both nodes shift to S_INSTALL,
626 // both send gap messages
627 p1->handle_msg(gm2);
628 ck_assert(p1->state() == Proto::S_INSTALL);
629 Message gm12;
630 rb = get_msg(t1, &gm12);
631 ck_assert(rb != 0);
632 ck_assert(gm12.type() == Message::EVS_T_GAP);
633 ck_assert((gm12.flags() & Message::F_COMMIT) == 0);
634 rb = get_msg(t1, &msg);
635 ck_assert(rb == 0);
636
637 p2->handle_msg(gm);
638 ck_assert(p2->state() == Proto::S_INSTALL);
639 Message gm22;
640 rb = get_msg(t2, &gm22);
641 ck_assert(rb != 0);
642 ck_assert(gm22.type() == Message::EVS_T_GAP);
643 ck_assert((gm22.flags() & Message::F_COMMIT) == 0);
644 rb = get_msg(t2, &msg);
645 ck_assert(rb == 0);
646
647 // Handle final gap messages, expected output shift to operational
648 // and gap message
649
650 p1->handle_msg(gm22);
651 ck_assert(p1->state() == Proto::S_OPERATIONAL);
652 rb = get_msg(t1, &msg);
653 ck_assert(rb != 0);
654 ck_assert(msg.type() == Message::EVS_T_GAP);
655 ck_assert((msg.flags() & Message::F_COMMIT) == 0);
656 rb = get_msg(t1, &msg);
657 ck_assert(rb == 0);
658
659 p2->handle_msg(gm12);
660 ck_assert(p2->state() == Proto::S_OPERATIONAL);
661 rb = get_msg(t2, &msg);
662 ck_assert(rb != 0);
663 ck_assert(msg.type() == Message::EVS_T_GAP);
664 ck_assert((msg.flags() & Message::F_COMMIT) == 0);
665 rb = get_msg(t2, &msg);
666 ck_assert(rb == 0);
667
668 }
669
670
START_TEST(test_proto_double_join)671 START_TEST(test_proto_double_join)
672 {
673 log_info << "START";
674 gu::Config conf;
675 mark_point();
676 gu::ssl_register_params(conf);
677 gcomm::Conf::register_params(conf);
678 UUID uuid1(1), uuid2(2);
679 DummyTransport t1(uuid1), t2(uuid2);
680 mark_point();
681 DummyUser u1(conf), u2(conf);
682 mark_point();
683 Proto p1(conf, uuid1, 0), p2(conf, uuid2, 0);
684
685 gcomm::connect(&t1, &p1);
686 gcomm::connect(&p1, &u1);
687
688 gcomm::connect(&t2, &p2);
689 gcomm::connect(&p2, &u2);
690
691 single_join(&t1, &p1);
692 double_join(&t1, &p1, &t2, &p2);
693
694 }
695 END_TEST
696
697 static gu::Config gu_conf;
698
create_dummy_node_with_uuid(size_t idx,const gcomm::UUID & uuid,int version,const string & suspect_timeout,const string & inactive_timeout,const string & retrans_period)699 static DummyNode* create_dummy_node_with_uuid(
700 size_t idx,
701 const gcomm::UUID& uuid,
702 int version,
703 const string& suspect_timeout,
704 const string& inactive_timeout,
705 const string& retrans_period)
706 {
707 // reset conf to avoid stale config in case of nofork
708 gu_conf = gu::Config();
709 gu::ssl_register_params(gu_conf);
710 gcomm::Conf::register_params(gu_conf);
711 string conf = "evs://?" + Conf::EvsViewForgetTimeout + "=PT1H&"
712 + Conf::EvsInactiveCheckPeriod + "=" + to_string(Period(suspect_timeout)/3) + "&"
713 + Conf::EvsSuspectTimeout + "=" + suspect_timeout + "&"
714 + Conf::EvsInactiveTimeout + "=" + inactive_timeout + "&"
715
716 + Conf::EvsKeepalivePeriod + "=" + retrans_period + "&"
717 + Conf::EvsJoinRetransPeriod + "=" + retrans_period + "&"
718 + Conf::EvsInfoLogMask + "=0x7" + "&"
719 + Conf::EvsDebugLogMask + "=0xfff" + "&"
720 + Conf::EvsVersion + "=" + gu::to_string<int>(version);
721 if (::getenv("EVS_DEBUG_MASK") != 0)
722 {
723 conf += "&" + Conf::EvsDebugLogMask + "="
724 + ::getenv("EVS_DEBUG_MASK");
725 }
726 list<Protolay*> protos;
727 protos.push_back(new DummyTransport(uuid, false));
728 protos.push_back(new Proto(gu_conf, uuid, 0, conf));
729 return new DummyNode(gu_conf, idx, uuid, protos);
730 }
731
create_dummy_node(size_t idx,int version,const string & suspect_timeout="PT1H",const string & inactive_timeout="PT1H",const string & retrans_period="PT10M")732 static DummyNode* create_dummy_node(
733 size_t idx,
734 int version,
735 const string& suspect_timeout = "PT1H",
736 const string& inactive_timeout = "PT1H",
737 const string& retrans_period = "PT10M")
738 {
739 UUID uuid(static_cast<int32_t>(idx));
740 return create_dummy_node_with_uuid(idx, uuid, version, suspect_timeout,
741 inactive_timeout, retrans_period);
742 }
743
744 namespace
745 {
evs_from_dummy(DummyNode * dn)746 gcomm::evs::Proto* evs_from_dummy(DummyNode* dn)
747 {
748 return static_cast<Proto*>(dn->protos().back());
749 }
750
transport_from_dummy(DummyNode * dn)751 DummyTransport* transport_from_dummy(DummyNode* dn)
752 {
753 return static_cast<DummyTransport*>(dn->protos().front());
754 }
755 }
756
757
join_node(PropagationMatrix * p,DummyNode * n,bool first=false)758 static void join_node(PropagationMatrix* p,
759 DummyNode* n, bool first = false)
760 {
761 gu_trace(p->insert_tp(n));
762 gu_trace(n->connect(first));
763 }
764
765
send_n(DummyNode * node,const size_t n)766 static void send_n(DummyNode* node, const size_t n)
767 {
768 for (size_t i = 0; i < n; ++i)
769 {
770 gu_trace(node->send());
771 }
772 }
773
set_cvi(vector<DummyNode * > & nvec,size_t i_begin,size_t i_end,size_t seq)774 static void set_cvi(vector<DummyNode*>& nvec, size_t i_begin, size_t i_end,
775 size_t seq)
776 {
777 for (size_t i = i_begin; i <= i_end; ++i)
778 {
779 nvec[i]->set_cvi(ViewId(V_REG, nvec[i_begin]->uuid(),
780 static_cast<uint32_t>(seq)));
781 }
782 }
783
784 template <class C>
785 class ViewSeq
786 {
787 public:
ViewSeq()788 ViewSeq() { }
operator ()(const C & a,const C & b) const789 bool operator()(const C& a, const C& b) const
790 {
791 return (a->trace().current_view_trace().view().id().seq() < b->trace().current_view_trace().view().id().seq());
792 }
793 };
794
get_max_view_seq(const std::vector<DummyNode * > & dnv,size_t i,size_t j)795 static uint32_t get_max_view_seq(const std::vector<DummyNode*>& dnv,
796 size_t i, size_t j)
797 {
798 if (i == dnv.size()) return static_cast<uint32_t>(-1);
799 return (*std::max_element(dnv.begin() + i,
800 dnv.begin() + j,
801 ViewSeq<const DummyNode*>()))->trace().current_view_trace().view().id().seq();
802 }
803
804
805
START_TEST(test_proto_join_n)806 START_TEST(test_proto_join_n)
807 {
808 log_info << "START (join_n)";
809 init_rand();
810
811 const size_t n_nodes(4);
812 PropagationMatrix prop;
813 vector<DummyNode*> dn;
814
815 for (size_t i = 1; i <= n_nodes; ++i)
816 {
817 gu_trace(dn.push_back(create_dummy_node(i, 0)));
818 }
819
820 uint32_t max_view_seq(0);
821 for (size_t i = 0; i < n_nodes; ++i)
822 {
823 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
824 set_cvi(dn, 0, i, max_view_seq + 1);
825 gu_trace(prop.propagate_until_cvi(false));
826 max_view_seq = get_max_view_seq(dn, 0, i);
827 }
828 gu_trace(check_trace(dn));
829 for_each(dn.begin(), dn.end(), DeleteObject());
830 }
831 END_TEST
832
833
START_TEST(test_proto_join_n_w_user_msg)834 START_TEST(test_proto_join_n_w_user_msg)
835 {
836 gu_conf_self_tstamp_on();
837 log_info << "START (join_n_w_user_msg)";
838 init_rand();
839
840 const size_t n_nodes(4);
841 PropagationMatrix prop;
842 vector<DummyNode*> dn;
843 // @todo This test should terminate without these timeouts
844 const string suspect_timeout("PT1H");
845 const string inactive_timeout("PT1H");
846 const string retrans_period("PT0.1S");
847
848 for (size_t i = 1; i <= n_nodes; ++i)
849 {
850 gu_trace(dn.push_back(
851 create_dummy_node(i, 0, suspect_timeout,
852 inactive_timeout, retrans_period)));
853 }
854
855 uint32_t max_view_seq(0);
856 for (size_t i = 0; i < n_nodes; ++i)
857 {
858 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
859 set_cvi(dn, 0, i, max_view_seq + 1);
860 gu_trace(prop.propagate_until_cvi(true));
861 for (size_t j = 0; j <= i; ++j)
862 {
863 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
864 }
865 gu_trace(prop.propagate_until_empty());
866 for (size_t j = 0; j <= i; ++j)
867 {
868 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
869 }
870 max_view_seq = get_max_view_seq(dn, 0, i);
871 }
872
873 gu_trace(check_trace(dn));
874 for_each(dn.begin(), dn.end(), DeleteObject());
875 }
876 END_TEST
877
878
START_TEST(test_proto_join_n_lossy)879 START_TEST(test_proto_join_n_lossy)
880 {
881 gu_conf_self_tstamp_on();
882 log_info << "START (join_n_lossy)";
883 init_rand();
884
885 const size_t n_nodes(4);
886 PropagationMatrix prop;
887 vector<DummyNode*> dn;
888 const string suspect_timeout("PT1H");
889 const string inactive_timeout("PT1H");
890 const string retrans_period("PT0.1S");
891
892
893 for (size_t i = 1; i <= n_nodes; ++i)
894 {
895 gu_trace(dn.push_back(
896 create_dummy_node(i, 0, suspect_timeout,
897 inactive_timeout, retrans_period)));
898 }
899
900 uint32_t max_view_seq(0);
901 for (size_t i = 0; i < n_nodes; ++i)
902 {
903 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
904 set_cvi(dn, 0, i, max_view_seq + 1);
905 for (size_t j = 1; j < i + 1; ++j)
906 {
907 prop.set_loss(i + 1, j, 0.9);
908 prop.set_loss(j, i + 1, 0.9);
909 }
910 gu_trace(prop.propagate_until_cvi(true));
911 max_view_seq = get_max_view_seq(dn, 0, i);
912 }
913 gu_trace(check_trace(dn));
914 for_each(dn.begin(), dn.end(), DeleteObject());
915 }
916 END_TEST
917
918
START_TEST(test_proto_join_n_lossy_w_user_msg)919 START_TEST(test_proto_join_n_lossy_w_user_msg)
920 {
921 gu_conf_self_tstamp_on();
922 log_info << "START (join_n_lossy_w_user_msg)";
923 init_rand();
924
925 const size_t n_nodes(4);
926 PropagationMatrix prop;
927 vector<DummyNode*> dn;
928 const string suspect_timeout("PT1H");
929 const string inactive_timeout("PT1H");
930 const string retrans_period("PT0.1S");
931
932 for (size_t i = 1; i <= n_nodes; ++i)
933 {
934 gu_trace(dn.push_back(
935 create_dummy_node(i, 0, suspect_timeout,
936 inactive_timeout, retrans_period)));
937 }
938
939 uint32_t max_view_seq(0);
940 for (size_t i = 0; i < n_nodes; ++i)
941 {
942 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
943 set_cvi(dn, 0, i, max_view_seq + 1);
944 for (size_t j = 1; j < i + 1; ++j)
945 {
946 prop.set_loss(i + 1, j, 0.9);
947 prop.set_loss(j, i + 1, 0.9);
948
949 }
950 gu_trace(prop.propagate_until_cvi(true));
951 for (size_t j = 0; j < i; ++j)
952 {
953 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
954 }
955 max_view_seq = get_max_view_seq(dn, 0, i);
956 }
957 gu_trace(check_trace(dn));
958 for_each(dn.begin(), dn.end(), DeleteObject());
959 }
960 END_TEST
961
START_TEST(test_proto_leave_n)962 START_TEST(test_proto_leave_n)
963 {
964 gu_conf_self_tstamp_on();
965 log_info << "START (leave_n)";
966 init_rand();
967
968 const size_t n_nodes(4);
969 PropagationMatrix prop;
970 vector<DummyNode*> dn;
971
972 for (size_t i = 1; i <= n_nodes; ++i)
973 {
974 gu_trace(dn.push_back(create_dummy_node(i, 0)));
975 }
976
977 for (size_t i = 0; i < n_nodes; ++i)
978 {
979 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
980 set_cvi(dn, 0, i, i + 1);
981 gu_trace(prop.propagate_until_cvi(true));
982 }
983
984 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
985
986 for (size_t i = 0; i < n_nodes; ++i)
987 {
988 dn[i]->close();
989 dn[i]->set_cvi(V_REG);
990 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
991 gu_trace(prop.propagate_until_cvi(true));
992 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
993 }
994
995 gu_trace(check_trace(dn));
996 for_each(dn.begin(), dn.end(), DeleteObject());
997 }
998 END_TEST
999
START_TEST(test_proto_leave_n_w_user_msg)1000 START_TEST(test_proto_leave_n_w_user_msg)
1001 {
1002 gu_conf_self_tstamp_on();
1003 log_info << "START (leave_n_w_user_msg)";
1004 init_rand();
1005
1006 const size_t n_nodes(4);
1007 PropagationMatrix prop;
1008 vector<DummyNode*> dn;
1009 const string suspect_timeout("PT1H");
1010 const string inactive_timeout("PT1H");
1011 const string retrans_period("PT0.1S");
1012
1013 for (size_t i = 1; i <= n_nodes; ++i)
1014 {
1015 gu_trace(dn.push_back(
1016 create_dummy_node(i, 0, suspect_timeout,
1017 inactive_timeout, retrans_period)));
1018 }
1019
1020 for (size_t i = 0; i < n_nodes; ++i)
1021 {
1022 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1023 set_cvi(dn, 0, i, i + 1);
1024 gu_trace(prop.propagate_until_cvi(false));
1025 }
1026
1027 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1028
1029 for (size_t i = 0; i < n_nodes; ++i)
1030 {
1031 for (size_t j = i; j < n_nodes; ++j)
1032 {
1033 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1034 }
1035 dn[i]->close();
1036 dn[i]->set_cvi(V_REG);
1037 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1038 gu_trace(prop.propagate_until_cvi(true));
1039 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1040 }
1041
1042 gu_trace(check_trace(dn));
1043 for_each(dn.begin(), dn.end(), DeleteObject());
1044 }
1045 END_TEST
1046
1047
START_TEST(test_proto_leave_n_lossy)1048 START_TEST(test_proto_leave_n_lossy)
1049 {
1050 if (deterministic_tests()) return;
1051
1052 gu_conf_self_tstamp_on();
1053 log_info << "START (leave_n_lossy)";
1054 init_rand();
1055 const size_t n_nodes(4);
1056 PropagationMatrix prop;
1057 vector<DummyNode*> dn;
1058 const string suspect_timeout("PT15S");
1059 const string inactive_timeout("PT30S");
1060 const string retrans_period("PT1S");
1061
1062 for (size_t i = 1; i <= n_nodes; ++i)
1063 {
1064 gu_trace(dn.push_back(
1065 create_dummy_node(i, 0, suspect_timeout,
1066 inactive_timeout, retrans_period)));
1067 }
1068
1069 for (size_t i = 0; i < n_nodes; ++i)
1070 {
1071 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1072 set_cvi(dn, 0, i, i + 1);
1073 gu_trace(prop.propagate_until_cvi(false));
1074 }
1075
1076 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1077
1078 for (size_t i = 0; i < n_nodes; ++i)
1079 {
1080 for (size_t j = 1; j < i + 1; ++j)
1081 {
1082 prop.set_loss(i + 1, j, 0.9);
1083 prop.set_loss(j, i + 1, 0.9);
1084 }
1085 }
1086
1087 for (size_t i = 0; i < n_nodes; ++i)
1088 {
1089 dn[i]->set_cvi(V_REG);
1090 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1091 dn[i]->close();
1092 gu_trace(prop.propagate_until_cvi(true));
1093 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1094 }
1095
1096 gu_trace(check_trace(dn));
1097 for_each(dn.begin(), dn.end(), DeleteObject());
1098 }
1099 END_TEST
1100
1101
1102
START_TEST(test_proto_leave_n_lossy_w_user_msg)1103 START_TEST(test_proto_leave_n_lossy_w_user_msg)
1104 {
1105 if (deterministic_tests()) return;
1106
1107 gu_conf_self_tstamp_on();
1108 log_info << "START (leave_n_lossy_w_user_msg)";
1109 init_rand();
1110
1111 const size_t n_nodes(4);
1112 PropagationMatrix prop;
1113 vector<DummyNode*> dn;
1114
1115 const string suspect_timeout("PT15S");
1116 const string inactive_timeout("PT30S");
1117 const string retrans_period("PT1S");
1118
1119 for (size_t i = 1; i <= n_nodes; ++i)
1120 {
1121 gu_trace(dn.push_back(
1122 create_dummy_node(i, 0, suspect_timeout,
1123 inactive_timeout, retrans_period)));
1124 }
1125
1126 for (size_t i = 0; i < n_nodes; ++i)
1127 {
1128 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1129 set_cvi(dn, 0, i, i + 1);
1130 gu_trace(prop.propagate_until_cvi(false));
1131 }
1132
1133
1134 for (size_t i = 0; i < n_nodes; ++i)
1135 {
1136 for (size_t j = 1; j < i + 1; ++j)
1137 {
1138 prop.set_loss(i + 1, j, 0.9);
1139 prop.set_loss(j, i + 1, 0.9);
1140 }
1141 }
1142
1143 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1144
1145 for (size_t i = 0; i < n_nodes; ++i)
1146 {
1147 for (size_t j = i; j < n_nodes; ++j)
1148 {
1149 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1150 }
1151 dn[i]->set_cvi(V_REG);
1152 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1153 dn[i]->close();
1154 gu_trace(prop.propagate_until_cvi(true));
1155 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1156 }
1157
1158 gu_trace(check_trace(dn));
1159 for_each(dn.begin(), dn.end(), DeleteObject());
1160 }
1161 END_TEST
1162
1163
1164 // Generic test code for split/merge cases
test_proto_split_merge_gen(const size_t n_nodes,const bool send_msgs,const double loss)1165 static void test_proto_split_merge_gen(const size_t n_nodes,
1166 const bool send_msgs,
1167 const double loss)
1168 {
1169 PropagationMatrix prop;
1170 vector<DummyNode*> dn;
1171 const string suspect_timeout("PT15S");
1172 const string inactive_timeout("PT30S");
1173 const string retrans_period("PT1S");
1174
1175 for (size_t i = 1; i <= n_nodes; ++i)
1176 {
1177 gu_trace(dn.push_back(
1178 create_dummy_node(i, 0, suspect_timeout,
1179 inactive_timeout, retrans_period)));
1180 }
1181
1182 for (size_t i = 0; i < n_nodes; ++i)
1183 {
1184 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1185 set_cvi(dn, 0, i, i + 1);
1186 gu_trace(prop.propagate_until_cvi(false));
1187 }
1188
1189 for (size_t i = 0; i < n_nodes; ++i)
1190 {
1191 for (size_t j = 1; j < i + 1; ++j)
1192 {
1193 prop.set_loss(i + 1, j, loss);
1194 prop.set_loss(j, i + 1, loss);
1195 }
1196 }
1197
1198 vector<int32_t> split;
1199 for (size_t i = 0; i < n_nodes; ++i)
1200 {
1201 split.push_back(static_cast<int32_t>(i + 1));
1202 }
1203
1204 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1205
1206 for (size_t i = 1; i < n_nodes; ++i)
1207 {
1208 if (send_msgs == true)
1209 {
1210 for (size_t k = 0; k < 5; ++k)
1211 {
1212 for (size_t j = 0; j < n_nodes; ++j)
1213 {
1214 gu_trace(send_n(dn[j], 1 + j));
1215 }
1216 gu_trace(prop.propagate_n(7));
1217 }
1218 }
1219
1220 log_info << "split " << i;
1221 for (size_t j = 0; j < i; ++j)
1222 {
1223 for (size_t k = i; k < n_nodes; ++k)
1224 {
1225 gu_trace(prop.set_loss(split[j], split[k], 0.));
1226 gu_trace(prop.set_loss(split[k], split[j], 0.));
1227 }
1228 }
1229
1230 set_cvi(dn, 0, i - 1, max_view_seq + 1);
1231 set_cvi(dn, i, n_nodes - 1, max_view_seq + 1);
1232
1233 if (send_msgs == true)
1234 {
1235 for (size_t j = 0; j < n_nodes; ++j)
1236 {
1237 gu_trace(send_n(dn[j], 5 + rand() % 4));
1238 }
1239 }
1240
1241 gu_trace(prop.propagate_until_cvi(true));
1242 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1243 log_info << "merge " << i;
1244 for (size_t j = 0; j < i; ++j)
1245 {
1246 for (size_t k = i; k < n_nodes; ++k)
1247 {
1248 gu_trace(prop.set_loss(split[j], split[k], loss));
1249 gu_trace(prop.set_loss(split[k], split[j], loss));
1250 }
1251 }
1252
1253 set_cvi(dn, 0, n_nodes - 1, max_view_seq + 1);
1254
1255 if (send_msgs == true)
1256 {
1257 for (size_t j = 0; j < n_nodes; ++j)
1258 {
1259 gu_trace(send_n(dn[j], 5 + rand() % 4));
1260 }
1261 }
1262 gu_trace(prop.propagate_until_cvi(true));
1263 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1264 }
1265
1266 gu_trace(prop.propagate_until_empty());
1267
1268 gu_trace(check_trace(dn));
1269 for_each(dn.begin(), dn.end(), DeleteObject());
1270 }
1271
1272
1273
START_TEST(test_proto_split_merge)1274 START_TEST(test_proto_split_merge)
1275 {
1276 gu_conf_self_tstamp_on();
1277 log_info << "START (split_merge)";
1278 init_rand();
1279
1280 test_proto_split_merge_gen(4, false, 1.);
1281 }
1282 END_TEST
1283
1284
START_TEST(test_proto_split_merge_lossy)1285 START_TEST(test_proto_split_merge_lossy)
1286 {
1287 if (deterministic_tests()) return;
1288
1289 gu_conf_self_tstamp_on();
1290 log_info << "START (split_merge_lossy)";
1291 init_rand();
1292
1293 test_proto_split_merge_gen(4, false, .9);
1294 }
1295 END_TEST
1296
1297
1298
START_TEST(test_proto_split_merge_w_user_msg)1299 START_TEST(test_proto_split_merge_w_user_msg)
1300 {
1301 gu_conf_self_tstamp_on();
1302 log_info << "START (split_merge_w_user_msg)";
1303 init_rand();
1304
1305 test_proto_split_merge_gen(4, true, 1.);
1306
1307 }
1308 END_TEST
1309
1310
START_TEST(test_proto_split_merge_lossy_w_user_msg)1311 START_TEST(test_proto_split_merge_lossy_w_user_msg)
1312 {
1313 if (deterministic_tests()) return;
1314
1315 gu_conf_self_tstamp_on();
1316 log_info << "START (split_merge_lossy_w_user_msg)";
1317 init_rand();
1318
1319 test_proto_split_merge_gen(4, true, .9);
1320 }
1321 END_TEST
1322
START_TEST(test_proto_stop_cont)1323 START_TEST(test_proto_stop_cont)
1324 {
1325 log_info << "START";
1326 init_rand();
1327
1328 const size_t n_nodes(4);
1329 PropagationMatrix prop;
1330 vector<DummyNode*> dn;
1331 const string suspect_timeout("PT0.31S");
1332 const string inactive_timeout("PT0.31S");
1333 const string retrans_period("PT0.1S");
1334
1335 for (size_t i = 1; i <= n_nodes; ++i)
1336 {
1337 gu_trace(dn.push_back(
1338 create_dummy_node(i, 0, suspect_timeout,
1339 inactive_timeout, retrans_period)));
1340 }
1341
1342 for (size_t i = 0; i < n_nodes; ++i)
1343 {
1344 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1345 set_cvi(dn, 0, i, i + 1);
1346 gu_trace(prop.propagate_until_cvi(false));
1347 }
1348 uint32_t view_seq = n_nodes + 1;
1349
1350 for (size_t i = 0; i < n_nodes; ++i)
1351 {
1352 for (size_t j = 0; j < n_nodes; ++j)
1353 {
1354 if (j != i)
1355 {
1356 dn[j]->close(dn[i]->uuid());
1357 }
1358 }
1359 set_cvi(dn, 0, n_nodes - 1, view_seq + 1);
1360 gu_trace(prop.propagate_until_cvi(true));
1361 view_seq += 2;
1362
1363 }
1364 gu_trace(check_trace(dn));
1365 for_each(dn.begin(), dn.end(), DeleteObject());
1366 }
1367 END_TEST
1368
1369
START_TEST(test_proto_arbitrate)1370 START_TEST(test_proto_arbitrate)
1371 {
1372 log_info << "START";
1373 const size_t n_nodes(3);
1374 PropagationMatrix prop;
1375 vector<DummyNode*> dn;
1376 const string suspect_timeout("PT0.5S");
1377 const string inactive_timeout("PT0.5S");
1378 const string retrans_period("PT0.1S");
1379
1380 for (size_t i = 1; i <= n_nodes; ++i)
1381 {
1382 gu_trace(dn.push_back(
1383 create_dummy_node(i, 0,
1384 suspect_timeout,
1385 inactive_timeout, retrans_period)));
1386 }
1387
1388 for (size_t i = 0; i < n_nodes; ++i)
1389 {
1390 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1391 set_cvi(dn, 0, i, i + 1);
1392 gu_trace(prop.propagate_until_cvi(false));
1393 }
1394 uint32_t view_seq = n_nodes + 1;
1395
1396 dn[0]->close(dn[1]->uuid());
1397 dn[1]->close(dn[0]->uuid());
1398 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1399 dn[2]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1400 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), view_seq));
1401 gu_trace(prop.propagate_until_cvi(true));
1402
1403 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1404 dn[1]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1405 dn[2]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1406 gu_trace(prop.propagate_until_cvi(true));
1407
1408 gu_trace(check_trace(dn));
1409
1410 for_each(dn.begin(), dn.end(), DeleteObject());
1411 }
1412 END_TEST
1413
1414
START_TEST(test_proto_split_two)1415 START_TEST(test_proto_split_two)
1416 {
1417 log_info << "START";
1418 const size_t n_nodes(2);
1419 PropagationMatrix prop;
1420 vector<DummyNode*> dn;
1421 const string suspect_timeout("PT0.31S");
1422 const string inactive_timeout("PT0.31S");
1423 const string retrans_period("PT0.1S");
1424
1425 for (size_t i = 1; i <= n_nodes; ++i)
1426 {
1427 gu_trace(dn.push_back(
1428 create_dummy_node(i, 0, suspect_timeout,
1429 inactive_timeout, retrans_period)));
1430 }
1431
1432 for (size_t i = 0; i < n_nodes; ++i)
1433 {
1434 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1435 set_cvi(dn, 0, i, i + 1);
1436 gu_trace(prop.propagate_until_cvi(false));
1437 }
1438 uint32_t view_seq = n_nodes + 1;
1439
1440 dn[0]->close(dn[1]->uuid());
1441 dn[1]->close(dn[0]->uuid());
1442 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1443 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), view_seq));
1444
1445 gu_trace(prop.propagate_until_cvi(true));
1446
1447 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1448 dn[1]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1449 gu_trace(prop.propagate_until_cvi(true));
1450
1451 gu_trace(check_trace(dn));
1452
1453 for_each(dn.begin(), dn.end(), DeleteObject());
1454 }
1455 END_TEST
1456
START_TEST(test_aggreg)1457 START_TEST(test_aggreg)
1458 {
1459 log_info << "START";
1460 const size_t n_nodes(2);
1461 PropagationMatrix prop;
1462 vector<DummyNode*> dn;
1463 const string suspect_timeout("PT0.31S");
1464 const string inactive_timeout("PT0.31S");
1465 const string retrans_period("PT0.1S");
1466
1467 for (size_t i = 1; i <= n_nodes; ++i)
1468 {
1469 gu_trace(dn.push_back(
1470 create_dummy_node(i, 0, suspect_timeout,
1471 inactive_timeout, retrans_period)));
1472 }
1473
1474 for (size_t i = 0; i < n_nodes; ++i)
1475 {
1476 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1477 set_cvi(dn, 0, i, i + 1);
1478 gu_trace(prop.propagate_until_cvi(false));
1479 }
1480
1481 for (size_t i = 0; i < n_nodes; ++i)
1482 {
1483 gu_trace(send_n(dn[i], 8));
1484 }
1485
1486 gu_trace(prop.propagate_until_empty());
1487 gu_trace(check_trace(dn));
1488
1489 for_each(dn.begin(), dn.end(), DeleteObject());
1490 }
1491 END_TEST
1492
START_TEST(test_trac_538)1493 START_TEST(test_trac_538)
1494 {
1495 gu_conf_self_tstamp_on();
1496 log_info << "START (test_trac_538)";
1497 init_rand();
1498 const size_t n_nodes(5);
1499 PropagationMatrix prop;
1500 vector<DummyNode*> dn;
1501 const string suspect_timeout("PT0.5S");
1502 const string inactive_timeout("PT2S");
1503 const string retrans_period("PT0.1S");
1504
1505 for (size_t i = 1; i <= n_nodes; ++i)
1506 {
1507 gu_trace(dn.push_back(
1508 create_dummy_node(i, 0, suspect_timeout,
1509 inactive_timeout,
1510 retrans_period)));
1511 }
1512
1513 for (size_t i = 0; i < n_nodes - 1; ++i)
1514 {
1515 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1516 set_cvi(dn, 0, i, i + 1);
1517 gu_trace(prop.propagate_until_cvi(false));
1518 }
1519
1520 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes - 1));
1521
1522 gu_trace(join_node(&prop, dn[n_nodes - 1], false));
1523 for (size_t i = 1; i <= n_nodes; ++i)
1524 {
1525 if (i != n_nodes - 1)
1526 {
1527 prop.set_loss(i, n_nodes - 1, 0);
1528 prop.set_loss(n_nodes - 1, i, 0);
1529 }
1530 }
1531 set_cvi(dn, 0, n_nodes - 1, max_view_seq + 1);
1532 dn[n_nodes - 2]->set_cvi(ViewId(V_REG, n_nodes - 1, max_view_seq + 1));
1533 gu_trace(prop.propagate_until_cvi(true));
1534 gu_trace(check_trace(dn));
1535 for_each(dn.begin(), dn.end(), DeleteObject());
1536 }
1537 END_TEST
1538
1539
START_TEST(test_trac_552)1540 START_TEST(test_trac_552)
1541 {
1542 log_info << "START (trac_552)";
1543 init_rand();
1544
1545 const size_t n_nodes(3);
1546 PropagationMatrix prop;
1547 vector<DummyNode*> dn;
1548
1549 const string suspect_timeout("PT15S");
1550 const string inactive_timeout("PT30S");
1551 const string retrans_period("PT1S");
1552
1553 for (size_t i = 1; i <= n_nodes; ++i)
1554 {
1555 gu_trace(dn.push_back(
1556 create_dummy_node(i, 0, suspect_timeout,
1557 inactive_timeout, retrans_period)));
1558 }
1559
1560 for (size_t i = 0; i < n_nodes; ++i)
1561 {
1562 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1563 set_cvi(dn, 0, i, i + 1);
1564 gu_trace(prop.propagate_until_cvi(false));
1565 }
1566
1567
1568 for (size_t i = 0; i < n_nodes; ++i)
1569 {
1570 for (size_t j = 1; j < i + 1; ++j)
1571 {
1572 prop.set_loss(i + 1, j, 0.9);
1573 prop.set_loss(j, i + 1, 0.9);
1574 }
1575 }
1576
1577 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1578
1579 for (size_t j = 0; j < n_nodes; ++j)
1580 {
1581 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1582 }
1583 dn[0]->set_cvi(V_REG);
1584 dn[1]->set_cvi(V_REG);
1585 set_cvi(dn, 2, n_nodes - 1, max_view_seq + 1);
1586 dn[0]->close();
1587 dn[1]->close();
1588 gu_trace(prop.propagate_until_cvi(true));
1589
1590 gu_trace(check_trace(dn));
1591 for_each(dn.begin(), dn.end(), DeleteObject());
1592 }
1593 END_TEST
1594
1595
START_TEST(test_trac_607)1596 START_TEST(test_trac_607)
1597 {
1598 gu_conf_self_tstamp_on();
1599 log_info << "START (trac_607)";
1600
1601 const size_t n_nodes(3);
1602 PropagationMatrix prop;
1603 vector<DummyNode*> dn;
1604
1605 const string suspect_timeout("PT0.5S");
1606 const string inactive_timeout("PT1S");
1607 const string retrans_period("PT0.1S");
1608
1609 for (size_t i = 1; i <= n_nodes; ++i)
1610 {
1611 gu_trace(dn.push_back(
1612 create_dummy_node(i, 0, suspect_timeout,
1613 inactive_timeout, retrans_period)));
1614 }
1615
1616 for (size_t i = 0; i < n_nodes; ++i)
1617 {
1618 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1619 set_cvi(dn, 0, i, i + 1);
1620 gu_trace(prop.propagate_until_cvi(false));
1621 }
1622
1623 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1624 dn[0]->set_cvi(V_REG);
1625 dn[0]->close();
1626
1627 while (evs_from_dummy(dn[1])->state() != Proto::S_INSTALL)
1628 {
1629 prop.propagate_n(1);
1630 }
1631
1632 // this used to cause exception:
1633 // Forbidden state transition: INSTALL -> LEAVING (FATAL)
1634 dn[1]->close();
1635
1636 // expected behavior:
1637 // dn[1], dn[2] reach S_OPERATIONAL and then dn[1] leaves gracefully
1638 set_cvi(dn, 1, n_nodes - 1, max_view_seq + 1);
1639
1640 gu_trace(prop.propagate_until_cvi(true));
1641 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1642 dn[1]->set_cvi(V_REG);
1643 set_cvi(dn, 2, 2, max_view_seq + 1);
1644
1645 gu_trace(prop.propagate_until_cvi(true));
1646
1647 gu_trace(check_trace(dn));
1648 for_each(dn.begin(), dn.end(), DeleteObject());
1649 }
1650 END_TEST
1651
1652
START_TEST(test_trac_724)1653 START_TEST(test_trac_724)
1654 {
1655 gu_conf_self_tstamp_on();
1656 log_info << "START (trac_724)";
1657 init_rand();
1658
1659 const size_t n_nodes(2);
1660 PropagationMatrix prop;
1661 vector<DummyNode*> dn;
1662 Protolay::sync_param_cb_t sync_param_cb;
1663
1664 const string suspect_timeout("PT0.5S");
1665 const string inactive_timeout("PT1S");
1666 const string retrans_period("PT0.1S");
1667
1668 for (size_t i = 1; i <= n_nodes; ++i)
1669 {
1670 gu_trace(dn.push_back(
1671 create_dummy_node(i, 0, suspect_timeout,
1672 inactive_timeout, retrans_period)));
1673 }
1674
1675 for (size_t i = 0; i < n_nodes; ++i)
1676 {
1677 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1678 set_cvi(dn, 0, i, i + 1);
1679 gu_trace(prop.propagate_until_cvi(false));
1680 }
1681
1682 // Slightly asymmetric settings and evs.use_aggregate=false to
1683 // allow completion window to grow over 0xff.
1684 Proto* evs0(evs_from_dummy(dn[0]));
1685
1686 bool ret(evs0->set_param("evs.use_aggregate", "false", sync_param_cb));
1687 ck_assert(ret == true);
1688 ret = evs0->set_param("evs.send_window", "1024", sync_param_cb);
1689 ck_assert(ret == true);
1690 ret = evs0->set_param("evs.user_send_window", "515", sync_param_cb);
1691 Proto* evs1(evs_from_dummy(dn[1]));
1692 ret = evs1->set_param("evs.use_aggregate", "false", sync_param_cb);
1693 ck_assert(ret == true);
1694 ret = evs1->set_param("evs.send_window", "1024", sync_param_cb);
1695 ck_assert(ret == true);
1696 ret = evs1->set_param("evs.user_send_window", "512", sync_param_cb);
1697
1698 prop.set_loss(1, 2, 0.);
1699
1700 for (size_t i(0); i < 256; ++i)
1701 {
1702 dn[0]->send();
1703 dn[0]->send();
1704 dn[1]->send();
1705 gu_trace(prop.propagate_until_empty());
1706 }
1707 dn[0]->send();
1708 prop.set_loss(1, 2, 1.);
1709
1710 dn[0]->send();
1711 gu_trace(prop.propagate_until_empty());
1712
1713 gu_trace(check_trace(dn));
1714 for_each(dn.begin(), dn.end(), DeleteObject());
1715 }
1716 END_TEST
1717
1718
START_TEST(test_trac_760)1719 START_TEST(test_trac_760)
1720 {
1721 gu_conf_self_tstamp_on();
1722 log_info << "START (trac_760)";
1723 init_rand();
1724
1725 const size_t n_nodes(3);
1726 PropagationMatrix prop;
1727 vector<DummyNode*> dn;
1728
1729 const string suspect_timeout("PT0.5S");
1730 const string inactive_timeout("PT1S");
1731 const string retrans_period("PT0.1S");
1732
1733 for (size_t i = 1; i <= n_nodes; ++i)
1734 {
1735 gu_trace(dn.push_back(
1736 create_dummy_node(i, 0, suspect_timeout,
1737 inactive_timeout, retrans_period)));
1738 }
1739
1740 for (size_t i = 0; i < n_nodes; ++i)
1741 {
1742 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1743 set_cvi(dn, 0, i, i + 1);
1744 gu_trace(prop.propagate_until_cvi(false));
1745 }
1746
1747 for (size_t i = 0; i < n_nodes; ++i)
1748 {
1749 gu_trace(send_n(dn[i], 2));
1750
1751 }
1752 gu_trace(prop.propagate_until_empty());
1753
1754
1755 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1756 gu_trace(send_n(dn[0], 1));
1757 gu_trace(send_n(dn[1], 1));
1758 // gu_trace(send_n(dn[2], 1));
1759
1760 set_cvi(dn, 0, 1, max_view_seq + 1);
1761 dn[2]->set_cvi(V_REG);
1762 dn[2]->close();
1763
1764 Proto* evs0(evs_from_dummy(dn[0]));
1765 Proto* evs1(evs_from_dummy(dn[1]));
1766 while (evs1->state() != Proto::S_GATHER && evs0->state() != Proto::S_GATHER)
1767 {
1768 gu_trace(prop.propagate_n(1));
1769 }
1770 dn[1]->close();
1771
1772 gu_trace(prop.propagate_until_cvi(true));
1773
1774 gu_trace(check_trace(dn));
1775 for_each(dn.begin(), dn.end(), DeleteObject());
1776 }
1777 END_TEST
1778
START_TEST(test_gh_41)1779 START_TEST(test_gh_41)
1780 {
1781 gu_conf_self_tstamp_on();
1782 log_info << "START (gh_41)";
1783
1784 const size_t n_nodes(3);
1785 PropagationMatrix prop;
1786 vector<DummyNode*> dn;
1787
1788 const string suspect_timeout("PT0.5S");
1789 const string inactive_timeout("PT1S");
1790 const string retrans_period("PT0.1S");
1791
1792 for (size_t i = 1; i <= n_nodes; ++i)
1793 {
1794 gu_trace(dn.push_back(
1795 create_dummy_node(i, 0, suspect_timeout,
1796 inactive_timeout, retrans_period)));
1797 }
1798
1799 for (size_t i = 0; i < n_nodes; ++i)
1800 {
1801 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1802 set_cvi(dn, 0, i, i + 1);
1803 gu_trace(prop.propagate_until_cvi(false));
1804 }
1805
1806 // Generate partitioning so that the node with smallest UUID
1807 // creates singleton view
1808 log_info << "partition";
1809 prop.set_loss(1, 2, 0.);
1810 prop.set_loss(2, 1, 0.);
1811 prop.set_loss(1, 3, 0.);
1812 prop.set_loss(3, 1, 0.);
1813 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1814
1815 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), max_view_seq + 1));
1816 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1817 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1818
1819 prop.propagate_until_cvi(true);
1820
1821 // Merge groups and make node 1 leave so that nodes 2 and 3 see
1822 // leave message from unknown origin
1823 log_info << "merge";
1824 prop.set_loss(1, 2, 1.);
1825 prop.set_loss(2, 1, 1.);
1826 prop.set_loss(1, 3, 1.);
1827 prop.set_loss(3, 1, 1.);
1828
1829 // Send message so that nodes 2 and 3 shift to GATHER. This must be done
1830 // because LEAVE message is ignored in handle_foreign()
1831 dn[0]->send();
1832 dn[0]->close();
1833
1834 dn[0]->set_cvi(V_REG);
1835 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 2));
1836 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 2));
1837
1838 prop.propagate_until_cvi(true);
1839 check_trace(dn);
1840 for_each(dn.begin(), dn.end(), DeleteObject());
1841 }
1842 END_TEST
1843
START_TEST(test_gh_37)1844 START_TEST(test_gh_37)
1845 {
1846 gu_conf_self_tstamp_on();
1847 log_info << "START (gh_37)";
1848
1849 const size_t n_nodes(3);
1850 PropagationMatrix prop;
1851 vector<DummyNode*> dn;
1852
1853 const string suspect_timeout("PT0.5S");
1854 const string inactive_timeout("PT1S");
1855 const string retrans_period("PT0.1S");
1856
1857 for (size_t i = 1; i <= n_nodes; ++i)
1858 {
1859 gu_trace(dn.push_back(
1860 create_dummy_node(i, 0, suspect_timeout,
1861 inactive_timeout, retrans_period)));
1862 }
1863
1864 for (size_t i = 0; i < n_nodes; ++i)
1865 {
1866 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1867 set_cvi(dn, 0, i, i + 1);
1868 gu_trace(prop.propagate_until_cvi(false));
1869 }
1870
1871 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1872 // node 0 is gonna to leave
1873 for(size_t i = 2; i <= n_nodes; i++)
1874 {
1875 // leaving node(LN) is able to send messages to remaining nodes.
1876 // prop.set_loss(1, i, 0.);
1877 // but remaining nodes(RNS) won't be able to ack these messages.
1878 prop.set_loss(i, 1, 0.);
1879 // so RNS aru_seq are the same and higher than LN aru_seq.
1880 }
1881 // LN ss=-1, ir=[2,1]
1882 // RNS ss=1, ir=[2,1]
1883 dn[0]->send();
1884 dn[0]->send();
1885 dn[0]->close();
1886
1887 dn[0]->set_cvi(V_REG);
1888 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1889 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1890
1891 prop.propagate_until_cvi(true);
1892 check_trace(dn);
1893 for_each(dn.begin(), dn.end(), DeleteObject());
1894 }
1895 END_TEST
1896
START_TEST(test_gh_40)1897 START_TEST(test_gh_40)
1898 {
1899 gu_conf_self_tstamp_on();
1900 log_info << "START (gh_40)";
1901
1902 const size_t n_nodes(3);
1903 PropagationMatrix prop;
1904 vector<DummyNode*> dn;
1905
1906 const string suspect_timeout("PT0.5S");
1907 const string inactive_timeout("PT1S");
1908 const string retrans_period("PT0.1S");
1909
1910 for (size_t i = 1; i <= n_nodes; ++i)
1911 {
1912 gu_trace(dn.push_back(
1913 create_dummy_node(i, 0, suspect_timeout,
1914 inactive_timeout, retrans_period)));
1915 }
1916
1917 for (size_t i = 0; i < n_nodes; ++i)
1918 {
1919 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1920 set_cvi(dn, 0, i, i + 1);
1921 gu_trace(prop.propagate_until_cvi(false));
1922 }
1923 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1924
1925 // ss=0, ir=[1,0];
1926 dn[0]->send();
1927 gu_trace(prop.propagate_until_empty());
1928 log_info << "gh_40 all got operational state";
1929
1930 // cut dn[0] from dn[1] and dn[2].
1931 for (size_t i = 2; i <= n_nodes; ++i)
1932 {
1933 prop.set_loss(1, i, 0.);
1934 prop.set_loss(i, 1, 0.);
1935 }
1936
1937 // ss=0, ir=[2,1];
1938 // dn[1] send msg(seq=1)
1939 dn[1]->send();
1940
1941 Proto* evs1 = evs_from_dummy(dn[1]);
1942 Proto* evs2 = evs_from_dummy(dn[2]);
1943 ck_assert(evs1->state() == Proto::S_OPERATIONAL);
1944 ck_assert(evs2->state() == Proto::S_OPERATIONAL);
1945 evs1->set_inactive(dn[0]->uuid());
1946 evs2->set_inactive(dn[0]->uuid());
1947 evs1->check_inactive();
1948 evs2->check_inactive();
1949 ck_assert(evs1->state() == Proto::S_GATHER);
1950 ck_assert(evs2->state() == Proto::S_GATHER);
1951
1952 // Advance clock to get over join message rate limiting.
1953 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
1954 while(!(evs1->state() == Proto::S_GATHER &&
1955 evs1->is_install_message()))
1956 {
1957 gu_trace(prop.propagate_n(1));
1958 }
1959
1960 // dn[0] comes back.
1961 // here we have to set message F_RETRANS
1962 // otherwise handle_msg ignores this msg.
1963 // @todo:why?
1964
1965 // dn[0] ack dn[1] msg(seq=1) with flags F_RETRANS.
1966 Datagram dg1 = dn[0]->create_datagram();
1967 UserMessage msg1(0,
1968 dn[0]->uuid(),
1969 ViewId(V_REG, dn[0]->uuid(), max_view_seq),
1970 1, 0, 0, O_DROP, 1, 0xff,
1971 Message::F_RETRANS);
1972 // dn[0] msg(seq=2) leak into dn[1] input_map.
1973 Datagram dg2 = dn[0]->create_datagram();
1974 UserMessage msg2(0,
1975 dn[0]->uuid(),
1976 ViewId(V_REG, dn[0]->uuid(), max_view_seq),
1977 2, 0, 0, O_SAFE, 2, 0xff,
1978 Message::F_RETRANS);
1979 // so for dn[1]
1980 // input_map: ss=0, ir=[3,2]
1981 // install message: ss=0, ir=[2,1]
1982 // seq 1 = O_SAFE message.(initiated by self)
1983 // seq 2 = O_DROP message.(complete_user)
1984 push_header(msg1, dg1);
1985 evs1->handle_up(0, dg1, ProtoUpMeta(dn[0]->uuid()));
1986 push_header(msg2, dg2);
1987 log_info << "evs1 handle msg " << msg2;
1988 log_info << "before handle msg: " << *evs1;
1989 evs1->handle_up(0, dg2, ProtoUpMeta(dn[0]->uuid()));
1990 log_info << "after handle msg: " << *evs1;
1991
1992 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), max_view_seq + 1));
1993 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1994 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1995 prop.propagate_until_cvi(true);
1996 check_trace(dn);
1997 for_each(dn.begin(), dn.end(), DeleteObject());
1998 }
1999 END_TEST
2000
2001
START_TEST(test_gh_100)2002 START_TEST(test_gh_100)
2003 {
2004 log_info << "START (test_gh_100)";
2005 gu::Config conf;
2006 mark_point();
2007 gu::ssl_register_params(conf);
2008 gcomm::Conf::register_params(conf);
2009 conf.set("evs.info_log_mask", "0x3");
2010 conf.set("evs.debug_log_mask", "0xa0");
2011 UUID uuid1(1), uuid2(2);
2012 DummyTransport t1(uuid1), t2(uuid2);
2013 mark_point();
2014 DummyUser u1(conf), u2(conf);
2015 mark_point();
2016 Proto p1(conf, uuid1, 0, gu::URI("evs://"), 10000, 0);
2017 // Start p2 view seqno from higher value than p1
2018 View p2_rst_view(0, ViewId(V_REG, uuid2, 3));
2019 Proto p2(conf, uuid2, 0, gu::URI("evs://"), 10000, &p2_rst_view);
2020
2021 gcomm::connect(&t1, &p1);
2022 gcomm::connect(&p1, &u1);
2023
2024 gcomm::connect(&t2, &p2);
2025 gcomm::connect(&p2, &u2);
2026
2027 single_join(&t1, &p1);
2028
2029
2030 // The following is from double_join(). Process messages until
2031 // install message is generated. After that handle install timer
2032 // on p1 and verify that the newly generated install message has
2033 // greater install view id seqno than the first one.
2034 Message jm;
2035 Message im;
2036 Message im2;
2037 Message gm;
2038 Message gm2;
2039 Message msg;
2040
2041 Datagram* rb;
2042
2043 // Initial states check
2044 p2.shift_to(Proto::S_JOINING);
2045 ck_assert(p1.state() == Proto::S_OPERATIONAL);
2046 ck_assert(p2.state() == Proto::S_JOINING);
2047
2048 // Send join message, don't self handle immediately
2049 // Expected output: one join message
2050 p2.send_join(false);
2051 ck_assert(p2.state() == Proto::S_JOINING);
2052 rb = get_msg(&t2, &jm);
2053 ck_assert(rb != 0);
2054 ck_assert(jm.type() == Message::EVS_T_JOIN);
2055 rb = get_msg(&t2, &msg);
2056 ck_assert(rb == 0);
2057
2058 // Handle node 2's join on node 1
2059 // Expected output: shift to S_GATHER and one join message
2060 p1.handle_msg(jm);
2061 ck_assert(p1.state() == Proto::S_GATHER);
2062 rb = get_msg(&t1, &jm);
2063 ck_assert(rb != 0);
2064 ck_assert(jm.type() == Message::EVS_T_JOIN);
2065 rb = get_msg(&t1, &msg);
2066 ck_assert(rb == 0);
2067
2068 // Handle node 1's join on node 2
2069 // Expected output: shift to S_GATHER and one join message
2070 p2.handle_msg(jm);
2071 ck_assert(p2.state() == Proto::S_GATHER);
2072 rb = get_msg(&t2, &jm);
2073 ck_assert(rb != 0);
2074 ck_assert(jm.type() == Message::EVS_T_JOIN);
2075 rb = get_msg(&t2, &msg);
2076 ck_assert(rb == 0);
2077
2078 // Handle node 2's join on node 1
2079 // Expected output: Install and commit gap messages, state stays in S_GATHER
2080 p1.handle_msg(jm);
2081 ck_assert(p1.state() == Proto::S_GATHER);
2082 rb = get_msg(&t1, &im);
2083 ck_assert(rb != 0);
2084 ck_assert(im.type() == Message::EVS_T_INSTALL);
2085 rb = get_msg(&t1, &gm);
2086 ck_assert(rb != 0);
2087 ck_assert(gm.type() == Message::EVS_T_GAP);
2088 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
2089 rb = get_msg(&t1, &msg);
2090 ck_assert(rb == 0);
2091
2092 // Handle timers to to generate shift to GATHER
2093 p1.handle_inactivity_timer();
2094 p1.handle_install_timer();
2095 rb = get_msg(&t1, &jm);
2096 ck_assert(rb != 0);
2097 ck_assert(jm.type() == Message::EVS_T_JOIN);
2098 rb = get_msg(&t1, &im2);
2099 ck_assert(rb != 0);
2100 ck_assert(im2.type() == Message::EVS_T_INSTALL);
2101 ck_assert(im2.install_view_id().seq() > im.install_view_id().seq());
2102
2103 gcomm::Datagram* tmp;
2104 while ((tmp = t1.out())) delete tmp;
2105 while ((tmp = t2.out())) delete tmp;
2106 }
2107 END_TEST
2108
START_TEST(test_evs_protocol_upgrade)2109 START_TEST(test_evs_protocol_upgrade)
2110 {
2111 log_info << "START (test_evs_protocol_upgrade)";
2112 PropagationMatrix prop;
2113 vector<DummyNode*> dn;
2114
2115 uint32_t view_seq(0);
2116 for (int i(0); i <= GCOMM_PROTOCOL_MAX_VERSION; ++i)
2117 {
2118 gu_trace(dn.push_back(create_dummy_node(i + 1, i)));
2119 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
2120 set_cvi(dn, 0, i, view_seq + 1);
2121 gu_trace(prop.propagate_until_cvi(false));
2122 ++view_seq;
2123 for (int j(0); j <= i; ++j)
2124 {
2125 ck_assert(evs_from_dummy(dn[j])->current_view().version() == 0);
2126 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2127 }
2128 }
2129
2130 for (int i(0); i < GCOMM_PROTOCOL_MAX_VERSION; ++i)
2131 {
2132 for (int j(i); j <= GCOMM_PROTOCOL_MAX_VERSION; ++j)
2133 {
2134 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2135 }
2136 dn[i]->close();
2137 dn[i]->set_cvi(V_REG);
2138 set_cvi(dn, i + 1, GCOMM_PROTOCOL_MAX_VERSION, view_seq);
2139 gu_trace(prop.propagate_until_cvi(true));
2140 ++view_seq;
2141 for (int j(i + 1); j <= GCOMM_PROTOCOL_MAX_VERSION; ++j)
2142 {
2143 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2144 }
2145 gu_trace(prop.propagate_until_empty());
2146 }
2147 ck_assert(evs_from_dummy(dn[GCOMM_PROTOCOL_MAX_VERSION])->current_view().version() == GCOMM_PROTOCOL_MAX_VERSION);
2148 check_trace(dn);
2149 for_each(dn.begin(), dn.end(), DeleteObject());
2150 }
2151 END_TEST
2152
2153
START_TEST(test_gal_521)2154 START_TEST(test_gal_521)
2155 {
2156 // Test the case where two nodes exhaust their user send windows
2157 // simultaneously.
2158 log_info << "Start test_gal_521";
2159
2160 std::vector<DummyNode*> dn;
2161 Protolay::sync_param_cb_t sync_param_cb;
2162
2163 dn.push_back(create_dummy_node(1, 0));
2164 dn.push_back(create_dummy_node(2, 0));
2165
2166
2167 gcomm::evs::Proto *evs1(evs_from_dummy(dn[0]));
2168 DummyTransport* t1(transport_from_dummy(dn[0]));
2169 t1->set_queueing(true);
2170
2171 gcomm::evs::Proto *evs2(evs_from_dummy(dn[1]));
2172 DummyTransport* t2(transport_from_dummy(dn[1]));
2173 t2->set_queueing(true);
2174
2175 single_join(t1, evs1);
2176 double_join(t1, evs1, t2, evs2);
2177
2178 ck_assert(t1->empty() == true);
2179 ck_assert(t2->empty() == true);
2180
2181 // Adjust send windows to allow sending only single user generated
2182 // message at the time
2183 evs1->set_param(gcomm::Conf::EvsUserSendWindow, "1", sync_param_cb);
2184 evs1->set_param(gcomm::Conf::EvsSendWindow, "1", sync_param_cb);
2185
2186 evs2->set_param(gcomm::Conf::EvsUserSendWindow, "1", sync_param_cb);
2187 evs2->set_param(gcomm::Conf::EvsSendWindow, "1", sync_param_cb);
2188
2189 // Make both sides send two messages without communicating with
2190 // each other. This will place one user message into transport
2191 // queue and one into send queue for both nodes.
2192 send_n(dn[0], 2);
2193 ck_assert(t1->empty() == false);
2194 send_n(dn[1], 2);
2195 ck_assert(t2->empty() == false);
2196
2197 Datagram *d1;
2198 Message um1;
2199 ck_assert((d1 = get_msg(t1, &um1, false)) != 0);
2200 ck_assert(um1.type() == Message::EVS_T_USER);
2201 ck_assert(t1->empty() == true);
2202 Datagram *d2;
2203 Message um2;
2204 ck_assert((d2 = get_msg(t2, &um2, false)) != 0);
2205 ck_assert(um2.type() == Message::EVS_T_USER);
2206 ck_assert(t2->empty() == true);
2207
2208 // Both of the nodes handle each other's messages. Now due to
2209 // send_window == 1 they are not allowed to send the second
2210 // message since safe_seq has not been incremented. Instead, they
2211 // must emit gap messages to make safe_seq to progress.
2212 evs1->handle_up(0, *d2, ProtoUpMeta(dn[1]->uuid()));
2213 delete d2;
2214 Message gm1;
2215 ck_assert(get_msg(t1, &gm1) != 0);
2216 ck_assert(gm1.type() == Message::EVS_T_GAP);
2217 ck_assert(t1->empty() == true);
2218
2219 evs2->handle_up(0, *d1, ProtoUpMeta(dn[0]->uuid()));
2220 delete d1;
2221 Message gm2;
2222 ck_assert(get_msg(t2, &gm2) != 0);
2223 ck_assert(gm2.type() == Message::EVS_T_GAP);
2224 ck_assert(t2->empty() == true);
2225
2226 // Handle gap messages. The safe_seq is now incremented so the
2227 // second user messages are now sent from output queue.
2228 evs1->handle_msg(gm2);
2229 ck_assert((d1 = get_msg(t1, &um1, false)) != 0);
2230 ck_assert(um1.type() == Message::EVS_T_USER);
2231 ck_assert(t1->empty() == true);
2232
2233 evs2->handle_msg(gm1);
2234 ck_assert((d2 = get_msg(t2, &um2, false)) != 0);
2235 ck_assert(um2.type() == Message::EVS_T_USER);
2236 ck_assert(t2->empty() == true);
2237
2238 // Handle user messages. Each node should now emit gap
2239 // because the output queue is empty.
2240 evs1->handle_up(0, *d2, ProtoUpMeta(dn[1]->uuid()));
2241 delete d2;
2242 ck_assert(get_msg(t1, &gm1) != 0);
2243 ck_assert(gm1.type() == Message::EVS_T_GAP);
2244 ck_assert(t1->empty() == true);
2245
2246 evs2->handle_up(0, *d1, ProtoUpMeta(dn[0]->uuid()));
2247 delete d1;
2248 ck_assert(get_msg(t2, &gm2) != 0);
2249 ck_assert(gm2.type() == Message::EVS_T_GAP);
2250 ck_assert(t2->empty() == true);
2251
2252 // Handle gap messages. No further messages should be emitted
2253 // since both user messages have been delivered, there are
2254 // no pending user messages in the output queue and no timers
2255 // have been expired.
2256 evs1->handle_msg(gm2);
2257 ck_assert((d1 = get_msg(t1, &um1, false)) == 0);
2258
2259 evs2->handle_msg(gm1);
2260 ck_assert((d2 = get_msg(t2, &um2, false)) == 0);
2261
2262
2263 std::for_each(dn.begin(), dn.end(), DeleteObject());
2264 }
2265 END_TEST
2266
2267 struct TwoNodeFixture
2268 {
2269 struct Configs
2270 {
ConfigsTwoNodeFixture::Configs2271 Configs()
2272 : conf1()
2273 , conf2()
2274 {
2275 gu::ssl_register_params(conf1);
2276 gcomm::Conf::register_params(conf1);
2277 gcomm::Conf::register_params(conf2);
2278 }
2279 gu::Config conf1; // Config for node1
2280 gu::Config conf2; // Config for node2
2281 };
TwoNodeFixtureTwoNodeFixture2282 TwoNodeFixture()
2283 : conf()
2284 , uuid1(1)
2285 , uuid2(2)
2286 , tr1(uuid1)
2287 , tr2(uuid2)
2288 , evs1(conf.conf1, uuid1, 0)
2289 , evs2(conf.conf2, uuid2, 0)
2290 , top1(conf.conf1)
2291 , top2(conf.conf2)
2292 {
2293 gcomm::connect(&tr1, &evs1);
2294 gcomm::connect(&evs1, &top1);
2295 gcomm::connect(&tr2, &evs2);
2296 gcomm::connect(&evs2, &top2);
2297 single_join(&tr1, &evs1);
2298 double_join(&tr1, &evs1, &tr2, &evs2);
2299 }
2300 Configs conf;
2301 const gcomm::UUID uuid1; // UUID of node1
2302 const gcomm::UUID uuid2; // UUID if node2
2303 DummyTransport tr1; // Transport for node1
2304 DummyTransport tr2; // Transport for node2
2305 gcomm::evs::Proto evs1; // Proto for node1
2306 gcomm::evs::Proto evs2; // Proto for node2
2307 DummyUser top1; // Top level layer for node1
2308 DummyUser top2; // Top level layer for node2
2309 };
2310
2311 // Verify that gap messages are rate limited when a node receives
2312 // several out of order messages.
START_TEST(test_gap_rate_limit)2313 START_TEST(test_gap_rate_limit)
2314 {
2315 log_info << "START test_gap_rate_limit";
2316 // Start time from 1 sec to avoid hitting gap rate limit for the first
2317 // gap message.
2318 gu::datetime::SimClock::init(gu::datetime::Sec);
2319 gu_log_max_level = GU_LOG_DEBUG;
2320 TwoNodeFixture f;
2321 gcomm::Protolay::sync_param_cb_t spcb;
2322
2323 // Increase evs1 send windows to allow generating out of order messages.
2324 f.evs1.set_param("evs.send_window", "4", spcb);
2325 f.evs1.set_param("evs.user_send_window", "4", spcb);
2326 // Print all debug logging on node2 for test troubleshooting.
2327 f.evs2.set_param("evs.debug_log_mask", "0xffff", spcb);
2328 f.evs2.set_param("evs.info_log_mask", "0xff", spcb);
2329 char data[1] = { 0 };
2330 gcomm::Datagram dg(gu::SharedBuffer(new gu::Buffer(data, data + 1)));
2331 // Generate four messages from node1. The first one is ignored,
2332 // the rest are handled by node2 for generating gap messages.
2333 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2334 gcomm::Datagram* read_dg;
2335 gcomm::evs::Message um1;
2336 read_dg = get_msg(&f.tr1, &um1);
2337 ck_assert(read_dg != 0);
2338 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2339 gcomm::evs::Message um2;
2340 read_dg = get_msg(&f.tr1, &um2);
2341 ck_assert(read_dg != 0);
2342 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2343 gcomm::evs::Message um3;
2344 read_dg = get_msg(&f.tr1, &um3);
2345 ck_assert(read_dg != 0);
2346 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2347 gcomm::evs::Message um4;
2348 read_dg = get_msg(&f.tr1, &um4);
2349 ck_assert(read_dg != 0);
2350
2351 // Make node2 handle an out of order message and verify that gap is emitted
2352 f.evs2.handle_msg(um2);
2353 gcomm::evs::Message gm1;
2354 read_dg = get_msg(&f.tr2, &gm1);
2355 ck_assert(read_dg != 0);
2356 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2357 ck_assert(gm1.range_uuid() == f.uuid1);
2358 ck_assert(gm1.range().lu() == 0);
2359 ck_assert(gm1.range().hs() == 0);
2360 // The node2 will also send an user message to complete the sequence
2361 // number. Consume it.
2362 gcomm::evs::Message comp_um1;
2363 read_dg = get_msg(&f.tr2, &comp_um1);
2364 ck_assert(read_dg != 0);
2365 ck_assert(comp_um1.type() == gcomm::evs::Message::EVS_T_USER);
2366 ck_assert(comp_um1.seq() + comp_um1.seq_range() == 1);
2367 // No further messages should be emitted
2368 read_dg = get_msg(&f.tr2, &comp_um1);
2369 ck_assert(read_dg == 0);
2370
2371 // Handle the second out of order message, gap should not be emitted.
2372 // There will be a next user message which completes the um3.
2373 f.evs2.handle_msg(um3);
2374 gcomm::evs::Message comp_um2;
2375 read_dg = get_msg(&f.tr2, &comp_um2);
2376 ck_assert(read_dg != 0);
2377 ck_assert(comp_um2.type() == gcomm::evs::Message::EVS_T_USER);
2378 ck_assert(comp_um2.seq() + comp_um2.seq_range() == 2);
2379
2380 // There should not be any more gap messages.
2381 read_dg = get_msg(&f.tr2, &gm1);
2382 ck_assert(read_dg == 0);
2383
2384 // Move the clock forwards and handle the fourth message, gap should
2385 // now emitted.
2386 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
2387 gcomm::evs::Message gm2;
2388 f.evs2.handle_msg(um4);
2389 read_dg = get_msg(&f.tr2, &gm2);
2390 ck_assert(read_dg != 0);
2391 ck_assert(gm2.type() == gcomm::evs::Message::EVS_T_GAP);
2392 ck_assert(gm2.range().lu() == 0);
2393 ck_assert(gm2.range().hs() == 0);
2394
2395 gcomm::evs::Message comp_u4;
2396 read_dg = get_msg(&f.tr2, &comp_u4);
2397 ck_assert(read_dg != 0);
2398 ck_assert(comp_u4.type() == gcomm::evs::Message::EVS_T_USER);
2399 log_info << "END test_gap_rate_limit";
2400 }
2401 END_TEST
2402
2403 // Verify that gap messages are rate limited when the liveness check finds
2404 // delayed node.
START_TEST(test_gap_rate_limit_delayed)2405 START_TEST(test_gap_rate_limit_delayed)
2406 {
2407 log_info << "START test_gap_rate_limit_delayed";
2408 // Start time from 1 sec to avoid hitting gap rate limit for the first
2409 // gap message.
2410 gu::datetime::SimClock::init(gu::datetime::Sec);
2411 gu_log_max_level = GU_LOG_DEBUG;
2412 TwoNodeFixture f;
2413 gcomm::Protolay::sync_param_cb_t spcb;
2414
2415 // Increase evs1 send windows to allow generating out of order messages.
2416 f.evs1.set_param("evs.send_window", "4", spcb);
2417 f.evs1.set_param("evs.user_send_window", "4", spcb);
2418 // Print all debug logging on node2 for test troubleshooting.
2419 f.evs2.set_param("evs.debug_log_mask", "0xffff", spcb);
2420 f.evs2.set_param("evs.info_log_mask", "0xff", spcb);
2421 // The retransmission request is done for delayed only if
2422 // auto evict is on.
2423 f.evs2.set_param("evs.auto_evict", "1", spcb);
2424 const char data[1] = { 0 };
2425 gcomm::Datagram dg(gu::SharedBuffer(new gu::Buffer(data, data + 1)));
2426 // Generate four messages from node1. The first one is ignored,
2427 // the rest are handled by node2 for generating gap messages.
2428 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2429 gcomm::Datagram* read_dg;
2430 gcomm::evs::Message um1;
2431 read_dg = get_msg(&f.tr1, &um1);
2432 ck_assert(read_dg != 0);
2433 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2434 gcomm::evs::Message um2;
2435 read_dg = get_msg(&f.tr1, &um2);
2436 ck_assert(read_dg != 0);
2437 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2438 gcomm::evs::Message um3;
2439 read_dg = get_msg(&f.tr1, &um3);
2440 ck_assert(read_dg != 0);
2441 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2442 gcomm::evs::Message um4;
2443 read_dg = get_msg(&f.tr1, &um4);
2444 ck_assert(read_dg != 0);
2445
2446 // Make node2 handle an out of order message and verify that gap is emitted
2447 f.evs2.handle_msg(um2);
2448 gcomm::evs::Message gm1;
2449 read_dg = get_msg(&f.tr2, &gm1);
2450 ck_assert(read_dg != 0);
2451 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2452 ck_assert(gm1.range_uuid() == f.uuid1);
2453 ck_assert(gm1.range().lu() == 0);
2454 ck_assert(gm1.range().hs() == 0);
2455 // The node2 will also send an user message to complete the sequence
2456 // number. Consume it.
2457 gcomm::evs::Message comp_um1;
2458 read_dg = get_msg(&f.tr2, &comp_um1);
2459 ck_assert(read_dg != 0);
2460 ck_assert(comp_um1.type() == gcomm::evs::Message::EVS_T_USER);
2461 ck_assert(comp_um1.seq() + comp_um1.seq_range() == 1);
2462 // No further messages should be emitted
2463 read_dg = get_msg(&f.tr2, &comp_um1);
2464 ck_assert(read_dg == 0);
2465
2466 // Move time forwards in 1 sec interval and make inactivity check
2467 // in between. No gap messages should be emitted.
2468 gu::datetime::SimClock::inc_time(gu::datetime::Sec);
2469 f.evs2.handle_inactivity_timer();
2470 gcomm::evs::Message gm_discard;
2471 read_dg = get_msg(&f.tr2, &gm_discard);
2472 ck_assert(read_dg == 0);
2473 // The clock is now advanced over retrans_period + delay margin. Next
2474 // call to handle_inactivity_timer() should fire the check. Gap message
2475 // is emitted.
2476 gu::datetime::SimClock::inc_time(gu::datetime::Sec);
2477 f.evs2.handle_inactivity_timer();
2478 read_dg = get_msg(&f.tr2, &gm1);
2479 ck_assert(read_dg != 0);
2480 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2481 // Now call handle_inactivity_timer() again, gap message should not
2482 // be emitted due to rate limit.
2483 // Galera 4 will run with evs protocol version 1 and will emit
2484 // delayed list at this point.
2485 f.evs2.handle_inactivity_timer();
2486 gcomm::evs::Message dm;
2487 read_dg = get_msg(&f.tr2, &dm);
2488 ck_assert(read_dg != 0);
2489 ck_assert(dm.type() == gcomm::evs::Message::EVS_T_DELAYED_LIST);
2490 read_dg = get_msg(&f.tr2, &gm_discard);
2491 ck_assert(read_dg == 0);
2492 // Move clock forward 100msec, new gap should be now emitted.
2493 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
2494 f.evs2.handle_inactivity_timer();
2495 gcomm::evs::Message gm2;
2496 read_dg = get_msg(&f.tr2, &gm2);
2497 ck_assert(read_dg != 0);
2498 ck_assert(gm2.type() == gcomm::evs::Message::EVS_T_GAP);
2499 log_info << "END test_gap_rate_limit_delayed";
2500
2501 gcomm::Datagram* tmp;
2502 while ((tmp = f.tr1.out())) delete tmp;
2503 while ((tmp = f.tr2.out())) delete tmp;
2504 }
2505 END_TEST
2506
START_TEST(test_out_queue_limit)2507 START_TEST(test_out_queue_limit)
2508 {
2509 TwoNodeFixture f;
2510
2511 std::vector<char> data(1 << 15);
2512 gcomm::Datagram dg(gu::SharedBuffer(
2513 new gu::Buffer(data.begin(), data.end())));
2514 // Default user send window is 2 and out queue limit is 1M,
2515 // so we can write 2 + 32 messages without blocking.
2516 for (size_t i(0); i < 34; ++i)
2517 {
2518 ck_assert(f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE)) == 0);
2519 }
2520 // The next write should fill the out_queue and return EAGAIN
2521 const char small_data[1] = { 0 };
2522 dg = gu::SharedBuffer(new gu::Buffer(small_data, small_data + 1));
2523 ck_assert(f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE)) == EAGAIN);
2524
2525 gcomm::Datagram* tmp;
2526 while ((tmp = f.tr1.out())) delete tmp;
2527 }
2528 END_TEST
2529
2530 // Test outline: The representative of the group is isolated out and a
2531 // new instance is joined with uuid with incremented incarnation.
2532 // The following install message should have also the old incarnation
2533 // present. This is checked by sending an user message from old
2534 // representative incarnation before isolation. If the old incarnation
2535 // is not present in the install message, the delivery of the user message
2536 // in transitional configuration will throw an exception.
START_TEST(test_representative_incarnation_change)2537 START_TEST(test_representative_incarnation_change)
2538 {
2539 log_info << "START test_representative_incarnation_change";
2540 const size_t n_nodes(3);
2541 PropagationMatrix prop;
2542 vector<DummyNode*> dn;
2543 const int protocol_version(1);
2544
2545 const string suspect_timeout("PT0.5S");
2546 const string inactive_timeout("PT1S");
2547 const string retrans_period("PT0.1S");
2548
2549 for (size_t i = 1; i <= n_nodes; ++i)
2550 {
2551 gu_trace(dn.push_back(
2552 create_dummy_node_with_uuid(
2553 i, gcomm::UUID(i), protocol_version, suspect_timeout,
2554 inactive_timeout, retrans_period)));
2555 }
2556
2557 for (size_t i = 0; i < n_nodes; ++i)
2558 {
2559 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
2560 set_cvi(dn, 0, i, i + 1);
2561 gu_trace(prop.propagate_until_cvi(false));
2562 }
2563 prop.propagate_until_empty();
2564 // Send a message from the representative and propagate enough messages
2565 // to make sure that other nodes received the message but didn't deliver
2566 // yet.
2567 dn[0]->send();
2568 prop.propagate_n(3);
2569
2570 // Isolate the representative.
2571 for (size_t i = 2; i <= n_nodes; ++i)
2572 {
2573 prop.set_loss(1, i, 0.);
2574 prop.set_loss(i, 1, 0.);
2575 }
2576
2577 // Shift clock and handle timers to bring other nodes into gather
2578 // state.
2579 gu::datetime::SimClock::inc_time(300 * gu::datetime::MSec);
2580 evs_from_dummy(dn[1])->handle_timers();
2581 evs_from_dummy(dn[2])->handle_timers();
2582 prop.propagate_until_empty();
2583 gu::datetime::SimClock::inc_time(300 * gu::datetime::MSec);
2584 evs_from_dummy(dn[1])->handle_timers();
2585 evs_from_dummy(dn[2])->handle_timers();
2586 ck_assert(evs_from_dummy(dn[1])->state() == gcomm::evs::Proto::S_GATHER);
2587 ck_assert(evs_from_dummy(dn[2])->state() == gcomm::evs::Proto::S_GATHER);
2588
2589 // Create a new instance with old representative uuid with incarnation
2590 // incremented. Keep it isolated from old representative.
2591 gcomm::UUID uuid_new_incarnation(evs_from_dummy(dn[0])->uuid());
2592 uuid_new_incarnation.increment_incarnation();
2593 dn.push_back(create_dummy_node_with_uuid(4, uuid_new_incarnation,
2594 protocol_version,
2595 suspect_timeout, inactive_timeout,
2596 retrans_period));
2597 join_node(&prop, dn[3], false);
2598 prop.set_loss(1, 4, 0.);
2599 prop.set_loss(4, 1, 0.);
2600 prop.propagate_until_empty();
2601 std::for_each(dn.begin(), dn.end(), DeleteObject());
2602 }
2603 END_TEST
2604
evs2_suite()2605 Suite* evs2_suite()
2606 {
2607 Suite* s = suite_create("gcomm::evs");
2608 TCase* tc;
2609
2610 tc = tcase_create("test_range");
2611 tcase_add_test(tc, test_range);
2612 suite_add_tcase(s, tc);
2613
2614 tc = tcase_create("test_message");
2615 tcase_add_test(tc, test_message);
2616 suite_add_tcase(s, tc);
2617
2618 tc = tcase_create("test_input_map_insert");
2619 tcase_add_test(tc, test_input_map_insert);
2620 suite_add_tcase(s, tc);
2621
2622 tc = tcase_create("test_input_map_find");
2623 tcase_add_test(tc, test_input_map_find);
2624 suite_add_tcase(s, tc);
2625
2626 tc = tcase_create("test_input_map_safety");
2627 tcase_add_test(tc, test_input_map_safety);
2628 suite_add_tcase(s, tc);
2629
2630 tc = tcase_create("test_input_map_erase");
2631 tcase_add_test(tc, test_input_map_erase);
2632 suite_add_tcase(s, tc);
2633
2634 tc = tcase_create("test_input_map_overwrap");
2635 tcase_add_test(tc, test_input_map_overwrap);
2636 tcase_set_timeout(tc, 60);
2637 suite_add_tcase(s, tc);
2638
2639 tc = tcase_create("test_input_map_random_insert");
2640 tcase_add_test(tc, test_input_map_random_insert);
2641 suite_add_tcase(s, tc);
2642
2643 tc = tcase_create("test_input_map_gap_range_list");
2644 tcase_add_test(tc, test_input_map_gap_range_list);
2645 suite_add_tcase(s, tc);
2646
2647 tc = tcase_create("test_proto_single_join");
2648 tcase_add_test(tc, test_proto_single_join);
2649 suite_add_tcase(s, tc);
2650
2651 tc = tcase_create("test_proto_double_join");
2652 tcase_add_test(tc, test_proto_double_join);
2653 suite_add_tcase(s, tc);
2654
2655 tc = tcase_create("test_proto_join_n");
2656 tcase_add_test(tc, test_proto_join_n);
2657 suite_add_tcase(s, tc);
2658
2659 tc = tcase_create("test_proto_join_n_w_user_msg");
2660 tcase_add_test(tc, test_proto_join_n_w_user_msg);
2661 suite_add_tcase(s, tc);
2662
2663 tc = tcase_create("test_proto_join_n_lossy");
2664 tcase_add_test(tc, test_proto_join_n_lossy);
2665 suite_add_tcase(s, tc);
2666
2667 tc = tcase_create("test_proto_join_n_lossy_w_user_msg");
2668 tcase_add_test(tc, test_proto_join_n_lossy_w_user_msg);
2669 suite_add_tcase(s, tc);
2670
2671 tc = tcase_create("test_proto_leave_n");
2672 tcase_add_test(tc, test_proto_leave_n);
2673 tcase_set_timeout(tc, 20);
2674 suite_add_tcase(s, tc);
2675
2676 tc = tcase_create("test_proto_leave_n_w_user_msg");
2677 tcase_add_test(tc, test_proto_leave_n_w_user_msg);
2678 tcase_set_timeout(tc, 20);
2679 suite_add_tcase(s, tc);
2680
2681 tc = tcase_create("test_proto_leave_n_lossy");
2682 tcase_add_test(tc, test_proto_leave_n_lossy);
2683 tcase_set_timeout(tc, 25);
2684 suite_add_tcase(s, tc);
2685
2686 tc = tcase_create("test_proto_leave_n_lossy_w_user_msg");
2687 tcase_add_test(tc, test_proto_leave_n_lossy_w_user_msg);
2688 tcase_set_timeout(tc, 25);
2689 suite_add_tcase(s, tc);
2690
2691 tc = tcase_create("test_proto_split_merge");
2692 tcase_add_test(tc, test_proto_split_merge);
2693 tcase_set_timeout(tc, 20);
2694 suite_add_tcase(s, tc);
2695
2696 tc = tcase_create("test_proto_split_merge_lossy");
2697 tcase_add_test(tc, test_proto_split_merge_lossy);
2698 tcase_set_timeout(tc, 20);
2699 suite_add_tcase(s, tc);
2700
2701 tc = tcase_create("test_proto_split_merge_w_user_msg");
2702 tcase_add_test(tc, test_proto_split_merge_w_user_msg);
2703 tcase_set_timeout(tc, 60);
2704 suite_add_tcase(s, tc);
2705
2706 tc = tcase_create("test_proto_split_merge_lossy_w_user_msg");
2707 tcase_add_test(tc, test_proto_split_merge_lossy_w_user_msg);
2708 tcase_set_timeout(tc, 60);
2709 suite_add_tcase(s, tc);
2710
2711 tc = tcase_create("test_proto_stop_cont");
2712 tcase_add_test(tc, test_proto_stop_cont);
2713 tcase_set_timeout(tc, 10);
2714 suite_add_tcase(s, tc);
2715
2716 tc = tcase_create("test_proto_split_two");
2717 tcase_add_test(tc, test_proto_split_two);
2718 suite_add_tcase(s, tc);
2719
2720 tc = tcase_create("test_aggreg");
2721 tcase_add_test(tc, test_aggreg);
2722 suite_add_tcase(s, tc);
2723
2724 tc = tcase_create("test_proto_arbitrate");
2725 tcase_add_test(tc, test_proto_arbitrate);
2726 suite_add_tcase(s, tc);
2727
2728 tc = tcase_create("test_trac_538");
2729 tcase_add_test(tc, test_trac_538);
2730 tcase_set_timeout(tc, 15);
2731 suite_add_tcase(s, tc);
2732
2733 tc = tcase_create("test_trac_552");
2734 tcase_add_test(tc, test_trac_552);
2735 tcase_set_timeout(tc, 15);
2736 suite_add_tcase(s, tc);
2737
2738 tc = tcase_create("test_trac_607");
2739 tcase_add_test(tc, test_trac_607);
2740 tcase_set_timeout(tc, 15);
2741 suite_add_tcase(s, tc);
2742
2743 tc = tcase_create("test_trac_724");
2744 tcase_add_test(tc, test_trac_724);
2745 tcase_set_timeout(tc, 15);
2746 suite_add_tcase(s, tc);
2747
2748 tc = tcase_create("test_trac_760");
2749 tcase_add_test(tc, test_trac_760);
2750 tcase_set_timeout(tc, 15);
2751 suite_add_tcase(s, tc);
2752
2753 tc = tcase_create("test_gh_41");
2754 tcase_add_test(tc, test_gh_41);
2755 tcase_set_timeout(tc, 15);
2756 suite_add_tcase(s, tc);
2757
2758 tc = tcase_create("test_gh_37");
2759 tcase_add_test(tc, test_gh_37);
2760 tcase_set_timeout(tc, 15);
2761 suite_add_tcase(s, tc);
2762
2763 tc = tcase_create("test_gh_40");
2764 tcase_add_test(tc, test_gh_40);
2765 tcase_set_timeout(tc, 5);
2766 suite_add_tcase(s, tc);
2767
2768 tc = tcase_create("test_gh_100");
2769 tcase_add_test(tc, test_gh_100);
2770 suite_add_tcase(s, tc);
2771
2772 tc = tcase_create("test_evs_protocol_upgrade");
2773 tcase_add_test(tc, test_evs_protocol_upgrade);
2774 suite_add_tcase(s, tc);
2775
2776 tc = tcase_create("test_gal_521");
2777 tcase_add_test(tc, test_gal_521);
2778 suite_add_tcase(s, tc);
2779
2780 tc = tcase_create("test_gap_rate_limit");
2781 tcase_add_test(tc, test_gap_rate_limit);
2782 suite_add_tcase(s, tc);
2783
2784 tc = tcase_create("test_gap_rate_limit_delayed");
2785 tcase_add_test(tc, test_gap_rate_limit_delayed);
2786 suite_add_tcase(s, tc);
2787
2788 tc = tcase_create("test_out_queue_limit");
2789 tcase_add_test(tc, test_out_queue_limit);
2790 suite_add_tcase(s, tc);
2791
2792 tc = tcase_create("test_representative_incarnation_change");
2793 tcase_add_test(tc, test_representative_incarnation_change);
2794 suite_add_tcase(s, tc);
2795
2796 return s;
2797 }
2798