1 /*
2 * Copyright (C) 2009-2020 Codership Oy <info@codership.com>
3 */
4
5 /*!
6 * @file Unit tests for refactored EVS
7 */
8
9
10
11 #include "evs_proto.hpp"
12 #include "evs_input_map2.hpp"
13 #include "evs_message2.hpp"
14 #include "evs_seqno.hpp"
15
16 #include "check_gcomm.hpp"
17 #include "check_templ.hpp"
18 #include "check_trace.hpp"
19
20 #include "gcomm/conf.hpp"
21
22 #include "gu_asio.hpp" // gu::ssl_register_params()
23
24 #include <stdexcept>
25 #include <vector>
26 #include <set>
27
28 #include "check.h"
29
30 //
31 // set GALERA_TEST_DETERMINISTIC env
32 // variable before running pc test suite.
33 //
34 static class deterministic_tests
35 {
36 public:
deterministic_tests()37 deterministic_tests()
38 : deterministic_tests_()
39 {
40 if (::getenv("GALERA_TEST_DETERMINISTIC"))
41 {
42 deterministic_tests_ = true;
43 }
44 else
45 {
46 deterministic_tests_ = false;
47 }
48 }
49
operator ()() const50 bool operator()() const { return deterministic_tests_; }
51
52 private:
53 bool deterministic_tests_;
54 } deterministic_tests;
55
56 using namespace std;
57 using namespace std::rel_ops;
58 using namespace gu::datetime;
59 using namespace gcomm;
60 using namespace gcomm::evs;
61 using gu::DeleteObject;
62
init_rand()63 void init_rand()
64 {
65 unsigned int seed(static_cast<unsigned int>(time(0)));
66 log_info << "rand seed " << seed;
67 srand(seed);
68 }
69
init_rand(unsigned int seed)70 void init_rand(unsigned int seed)
71 {
72 log_info << "rand seed " << seed;
73 srand(seed);
74 }
75
76
START_TEST(test_range)77 START_TEST(test_range)
78 {
79 log_info << "START";
80 Range r(3, 6);
81
82 check_serialization(r, 2 * sizeof(seqno_t), Range());
83
84 }
85 END_TEST
86
START_TEST(test_message)87 START_TEST(test_message)
88 {
89 log_info << "START";
90 UUID uuid1(0, 0);
91 ViewId view_id(V_TRANS, uuid1, 4567);
92 seqno_t seq(478), aru_seq(456), seq_range(7);
93
94 UserMessage um(0, uuid1, view_id, seq, aru_seq, seq_range, O_SAFE, 75433, 0xab,
95 Message::F_SOURCE);
96 ck_assert(um.serial_size() % 4 == 0);
97 check_serialization(um, um.serial_size(), UserMessage());
98
99 AggregateMessage am(0xab, 17457, 0x79);
100 check_serialization(am, 4, AggregateMessage());
101
102 DelegateMessage dm(0, uuid1, view_id);
103 dm.set_source(uuid1);
104 check_serialization(dm, dm.serial_size(), DelegateMessage());
105
106 MessageNodeList node_list;
107 node_list.insert(make_pair(uuid1, MessageNode()));
108 node_list.insert(make_pair(UUID(2), MessageNode(true, false, 254, true, 1,
109 ViewId(V_REG), 5,
110 Range(7, 8))));
111 JoinMessage jm(0, uuid1, view_id, 8, 5, 27, node_list);
112 jm.set_source(uuid1);
113 check_serialization(jm, jm.serial_size(), JoinMessage());
114
115 InstallMessage im(0, uuid1, view_id, ViewId(V_REG, view_id.uuid(),
116 view_id.seq()), 8, 5, 27, node_list);
117 im.set_source(uuid1);
118 check_serialization(im, im.serial_size(), InstallMessage());
119
120 LeaveMessage lm(0, uuid1, view_id, 45, 88, 3456);
121 lm.set_source(uuid1);
122 check_serialization(lm, lm.serial_size(), LeaveMessage());
123
124
125 DelayedListMessage dlm(0, uuid1, view_id, 4576);
126 dlm.add(UUID(2), 23);
127 dlm.add(UUID(3), 45);
128 dlm.add(UUID(5), 255);
129 check_serialization(dlm, dlm.serial_size(), DelayedListMessage());
130 }
131 END_TEST
132
START_TEST(test_input_map_insert)133 START_TEST(test_input_map_insert)
134 {
135 log_info << "START";
136 UUID uuid1(1), uuid2(2);
137 InputMap im;
138 ViewId view(V_REG, uuid1, 0);
139
140 try
141 {
142 im.insert(0, UserMessage(0, uuid1, view, 0));
143 ck_abort_msg("Exception not thrown, input map has not been "
144 "reset/initialized yet");
145 }
146 catch (...)
147 { }
148
149 im.reset(1);
150
151 im.insert(0, UserMessage(0, uuid1, view, 0));
152
153
154 im.clear();
155 im.reset(2);
156
157 for (seqno_t s = 0; s < 10; ++s)
158 {
159 im.insert(0, UserMessage(0, uuid1, view, s));
160 im.insert(1, UserMessage(0, uuid2, view, s));
161 }
162
163 for (seqno_t s = 0; s < 10; ++s)
164 {
165 InputMap::iterator i = im.find(0, s);
166 ck_assert(i != im.end());
167 ck_assert(InputMapMsgIndex::value(i).msg().source() == uuid1);
168 ck_assert(InputMapMsgIndex::value(i).msg().seq() == s);
169
170 i = im.find(1, s);
171 ck_assert(i != im.end());
172 ck_assert(InputMapMsgIndex::value(i).msg().source() == uuid2);
173 ck_assert(InputMapMsgIndex::value(i).msg().seq() == s);
174 }
175
176 }
177 END_TEST
178
START_TEST(test_input_map_find)179 START_TEST(test_input_map_find)
180 {
181 log_info << "START";
182 InputMap im;
183 UUID uuid1(1);
184 ViewId view(V_REG, uuid1, 0);
185
186 im.reset(1);
187
188 im.insert(0, UserMessage(0, uuid1, view, 0));
189
190 ck_assert(im.find(0, 0) != im.end());
191
192
193 im.insert(0, UserMessage(0, uuid1, view, 2));
194 im.insert(0, UserMessage(0, uuid1, view, 4));
195 im.insert(0, UserMessage(0, uuid1, view, 7));
196
197 ck_assert(im.find(0, 2) != im.end());
198 ck_assert(im.find(0, 4) != im.end());
199 ck_assert(im.find(0, 7) != im.end());
200
201 ck_assert(im.find(0, 3) == im.end());
202 ck_assert(im.find(0, 5) == im.end());
203 ck_assert(im.find(0, 6) == im.end());
204 ck_assert(im.find(0, 8) == im.end());
205 }
206 END_TEST
207
START_TEST(test_input_map_safety)208 START_TEST(test_input_map_safety)
209 {
210 log_info << "START";
211 InputMap im;
212 UUID uuid1(1);
213 size_t index1(0);
214 ViewId view(V_REG, uuid1, 0);
215
216 im.reset(1);
217
218 im.insert(index1, UserMessage(0, uuid1, view, 0));
219 ck_assert(im.aru_seq() == 0);
220 im.insert(index1, UserMessage(0, uuid1, view, 1));
221 ck_assert(im.aru_seq() == 1);
222 im.insert(index1, UserMessage(0, uuid1, view, 2));
223 ck_assert(im.aru_seq() == 2);
224 im.insert(index1, UserMessage(0, uuid1, view, 3));
225 ck_assert(im.aru_seq() == 3);
226 im.insert(index1, UserMessage(0, uuid1, view, 5));
227 ck_assert(im.aru_seq() == 3);
228
229 im.insert(index1, UserMessage(0, uuid1, view, 4));
230 ck_assert(im.aru_seq() == 5);
231
232 InputMap::iterator i = im.find(index1, 0);
233 ck_assert(im.is_fifo(i) == true);
234 ck_assert(im.is_agreed(i) == true);
235 ck_assert(im.is_safe(i) == false);
236 im.set_safe_seq(index1, 0);
237 ck_assert(im.is_safe(i) == true);
238
239 im.set_safe_seq(index1, 5);
240 i = im.find(index1, 5);
241 ck_assert(im.is_safe(i) == true);
242
243 im.insert(index1, UserMessage(0, uuid1, view, 7));
244 im.set_safe_seq(index1, im.aru_seq());
245 i = im.find(index1, 7);
246 ck_assert(im.is_safe(i) == false);
247
248 }
249 END_TEST
250
START_TEST(test_input_map_erase)251 START_TEST(test_input_map_erase)
252 {
253 log_info << "START";
254 InputMap im;
255 size_t index1(0);
256 UUID uuid1(1);
257 ViewId view(V_REG, uuid1, 1);
258
259 im.reset(1);
260
261 for (seqno_t s = 0; s < 10; ++s)
262 {
263 im.insert(index1, UserMessage(0, uuid1, view, s));
264 }
265
266 for (seqno_t s = 0; s < 10; ++s)
267 {
268 InputMap::iterator i = im.find(index1, s);
269 ck_assert(i != im.end());
270 im.erase(i);
271 i = im.find(index1, s);
272 ck_assert(i == im.end());
273 (void)im.recover(index1, s);
274 }
275 im.set_safe_seq(index1, 9);
276 try
277 {
278 im.recover(index1, 9);
279 ck_abort_msg("Exception not thrown, "
280 "setting safe seq should purge index");
281 }
282 catch (...) { }
283 }
284 END_TEST
285
START_TEST(test_input_map_overwrap)286 START_TEST(test_input_map_overwrap)
287 {
288 log_info << "START";
289 InputMap im;
290 const size_t n_nodes(5);
291 ViewId view(V_REG, UUID(1), 1);
292 vector<UUID> uuids;
293 for (size_t n = 0; n < n_nodes; ++n)
294 {
295 uuids.push_back(UUID(static_cast<int32_t>(n + 1)));
296 }
297
298 im.reset(n_nodes);
299
300
301 Date start(Date::monotonic());
302 size_t cnt(0);
303 seqno_t last_safe(-1);
304 for (seqno_t seq = 0; seq < 100000; ++seq)
305 {
306 for (size_t i = 0; i < n_nodes; ++i)
307 {
308 UserMessage um(0, uuids[i], view, seq);
309 (void)im.insert(i, um);
310 if ((seq + 5) % 10 == 0)
311 {
312 last_safe = um.seq() - 3;
313 im.set_safe_seq(i, last_safe);
314 for (InputMap::iterator ii = im.begin();
315 ii != im.end() && im.is_safe(ii) == true;
316 ii = im.begin())
317 {
318 im.erase(ii);
319 }
320 }
321 cnt++;
322 }
323 gcomm_assert(im.aru_seq() == seq);
324 gcomm_assert(im.safe_seq() == last_safe);
325 }
326 Date stop(Date::monotonic());
327
328 double div(double(stop.get_utc() - start.get_utc())/gu::datetime::Sec);
329 log_info << "input map msg rate " << double(cnt)/div;
330 }
331 END_TEST
332
333
334 class InputMapInserter
335 {
336 public:
InputMapInserter(InputMap & im)337 InputMapInserter(InputMap& im) : im_(im) { }
338
operator ()(const pair<size_t,UserMessage> & p) const339 void operator()(const pair<size_t, UserMessage>& p) const
340 {
341 im_.insert(p.first, p.second);
342 }
343 private:
344 InputMap& im_;
345 };
346
START_TEST(test_input_map_random_insert)347 START_TEST(test_input_map_random_insert)
348 {
349 log_info << "START";
350 init_rand();
351 seqno_t n_seqnos(1024);
352 size_t n_uuids(4);
353 vector<UUID> uuids(n_uuids);
354 vector<pair<size_t, UserMessage> > msgs(static_cast<size_t>(n_uuids*n_seqnos));
355 ViewId view_id(V_REG, UUID(1), 1);
356 InputMap im;
357
358 for (size_t i = 0; i < n_uuids; ++i)
359 {
360 uuids[i] = (static_cast<int32_t>(i + 1));
361 }
362
363 im.reset(n_uuids);
364
365 for (seqno_t j = 0; j < n_seqnos; ++j)
366 {
367 for (size_t i = 0; i < n_uuids; ++i)
368 {
369 msgs[static_cast<size_t>(j*n_uuids) + i] =
370 make_pair(i, UserMessage(0, uuids[i], view_id, j));
371 }
372 }
373
374 vector<pair<size_t, UserMessage> > random_msgs(msgs);
375 random_shuffle(random_msgs.begin(), random_msgs.end());
376 for_each(random_msgs.begin(), random_msgs.end(), InputMapInserter(im));
377
378 size_t n = 0;
379 for (InputMap::iterator i = im.begin(); i != im.end(); ++i)
380 {
381 const InputMapMsg& msg(InputMapMsgIndex::value(i));
382 ck_assert(msg.msg() == msgs[n].second);
383 ck_assert(im.is_safe(i) == false);
384 ++n;
385 }
386
387 ck_assert(im.aru_seq() == n_seqnos - 1);
388 ck_assert(im.safe_seq() == -1);
389
390 for (size_t i = 0; i < n_uuids; ++i)
391 {
392 ck_assert(im.range(i) ==
393 Range(n_seqnos,
394 n_seqnos - 1));
395
396 im.set_safe_seq(i, n_seqnos - 1);
397 }
398 ck_assert(im.safe_seq() == n_seqnos - 1);
399
400 }
401 END_TEST
402
START_TEST(test_input_map_gap_range_list)403 START_TEST(test_input_map_gap_range_list)
404 {
405 gcomm::evs::InputMap im;
406 im.reset(1);
407 gcomm::UUID uuid(1);
408 gcomm::ViewId view_id(gcomm::V_REG, uuid, 1);
409 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 0, 0));
410 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 2, 0));
411
412 std::vector<gcomm::evs::Range> gap_range(
413 im.gap_range_list(0, gcomm::evs::Range(0, 2)));
414 ck_assert(gap_range.size() == 1);
415 ck_assert(gap_range.begin()->lu() == 1);
416 ck_assert(gap_range.begin()->hs() == 1);
417
418 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 4, 0));
419 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 4));
420 ck_assert(gap_range.size() == 2);
421 ck_assert(gap_range.begin()->lu() == 1);
422 ck_assert(gap_range.begin()->hs() == 1);
423 ck_assert(gap_range.rbegin()->lu() == 3);
424 ck_assert(gap_range.rbegin()->hs() == 3);
425
426 // Although there are two messages missing, limiting the range to 0,2
427 // should return only the first one.
428 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 2));
429 ck_assert(gap_range.size() == 1);
430 ck_assert(gap_range.begin()->lu() == 1);
431 ck_assert(gap_range.begin()->hs() == 1);
432
433 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 8, 0));
434 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
435 ck_assert(gap_range.size() == 3);
436 ck_assert(gap_range.begin()->lu() == 1);
437 ck_assert(gap_range.begin()->hs() == 1);
438 ck_assert(gap_range.rbegin()->lu() == 5);
439 ck_assert(gap_range.rbegin()->hs() == 7);
440
441 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 3, 0));
442 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
443 ck_assert(gap_range.size() == 2);
444 ck_assert(gap_range.begin()->lu() == 1);
445 ck_assert(gap_range.begin()->hs() == 1);
446 ck_assert(gap_range.rbegin()->lu() == 5);
447 ck_assert(gap_range.rbegin()->hs() == 7);
448
449 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 1, 0));
450 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 5, 0));
451 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 6, 0));
452 im.insert(0, gcomm::evs::UserMessage(0, uuid, view_id, 7, 0));
453 gap_range = im.gap_range_list(0, gcomm::evs::Range(0, 8));
454 ck_assert(gap_range.empty());
455 }
456 END_TEST
457
get_msg(DummyTransport * tp,Message * msg,bool release=true)458 static Datagram* get_msg(DummyTransport* tp, Message* msg, bool release = true)
459 {
460 Datagram* rb = tp->out();
461 if (rb != 0)
462 {
463 gu_trace(Proto::unserialize_message(tp->uuid(), *rb, msg));
464 if (release == true)
465 {
466 delete rb;
467 }
468 }
469 return rb;
470 }
471
single_join(DummyTransport * t,Proto * p)472 static void single_join(DummyTransport* t, Proto* p)
473 {
474 Message jm, im, gm;
475
476 // Initial state is joining
477 p->shift_to(Proto::S_JOINING);
478
479 // Send join must produce emitted join message
480 p->send_join();
481
482 Datagram* rb = get_msg(t, &jm);
483 ck_assert(rb != 0);
484 ck_assert(jm.type() == Message::EVS_T_JOIN);
485
486 // Install message is emitted at the end of JOIN handling
487 // 'cause this is the only instance and is always consistent
488 // with itself
489 rb = get_msg(t, &im);
490 ck_assert(rb != 0);
491 ck_assert(im.type() == Message::EVS_T_INSTALL);
492
493 // Handling INSTALL message emits three gap messages,
494 // one for receiving install message (commit gap), one for
495 // shift to install and one for shift to operational
496 rb = get_msg(t, &gm);
497 ck_assert(rb != 0);
498 ck_assert(gm.type() == Message::EVS_T_GAP);
499 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
500
501 rb = get_msg(t, &gm);
502 ck_assert(rb != 0);
503 ck_assert(gm.type() == Message::EVS_T_GAP);
504 ck_assert((gm.flags() & Message::F_COMMIT) == 0);
505
506 rb = get_msg(t, &gm);
507 ck_assert(rb != 0);
508 ck_assert(gm.type() == Message::EVS_T_GAP);
509 ck_assert((gm.flags() & Message::F_COMMIT) == 0);
510
511 // State must have evolved JOIN -> S_GATHER -> S_INSTALL -> S_OPERATIONAL
512 ck_assert(p->state() == Proto::S_OPERATIONAL);
513
514 // Handle join message again, must stay in S_OPERATIONAL, must not
515 // emit anything
516 p->handle_msg(jm);
517 rb = get_msg(t, &gm);
518 ck_assert(rb == 0);
519 ck_assert(p->state() == Proto::S_OPERATIONAL);
520
521 }
522
523 class DummyUser : public Toplay
524 {
525 public:
DummyUser(gu::Config & conf)526 DummyUser(gu::Config& conf) : Toplay(conf) { }
handle_up(const void *,const Datagram &,const ProtoUpMeta &)527 void handle_up(const void*, const Datagram&, const ProtoUpMeta&) { }
528 private:
529 };
530
531
START_TEST(test_proto_single_join)532 START_TEST(test_proto_single_join)
533 {
534 log_info << "START";
535 gu::Config conf;
536 mark_point();
537 gu::ssl_register_params(conf);
538 gcomm::Conf::register_params(conf);
539 UUID uuid(1);
540 DummyTransport t(uuid);
541 mark_point();
542 DummyUser u(conf);
543 mark_point();
544 Proto p(conf, uuid, 0);
545 mark_point();
546 gcomm::connect(&t, &p);
547 gcomm::connect(&p, &u);
548 single_join(&t, &p);
549 }
550 END_TEST
551
double_join(DummyTransport * t1,Proto * p1,DummyTransport * t2,Proto * p2)552 static void double_join(DummyTransport* t1, Proto* p1,
553 DummyTransport* t2, Proto* p2)
554 {
555
556 Message jm;
557 Message im;
558 Message gm;
559 Message gm2;
560 Message msg;
561
562 Datagram* rb;
563
564 // Initial states check
565 p2->shift_to(Proto::S_JOINING);
566 ck_assert(p1->state() == Proto::S_OPERATIONAL);
567 ck_assert(p2->state() == Proto::S_JOINING);
568
569 // Send join message, don't self handle immediately
570 // Expected output: one join message
571 p2->send_join(false);
572 ck_assert(p2->state() == Proto::S_JOINING);
573 rb = get_msg(t2, &jm);
574 ck_assert(rb != 0);
575 ck_assert(jm.type() == Message::EVS_T_JOIN);
576 rb = get_msg(t2, &msg);
577 ck_assert(rb == 0);
578
579 // Handle node 2's join on node 1
580 // Expected output: shift to S_GATHER and one join message
581 p1->handle_msg(jm);
582 ck_assert(p1->state() == Proto::S_GATHER);
583 rb = get_msg(t1, &jm);
584 ck_assert(rb != 0);
585 ck_assert(jm.type() == Message::EVS_T_JOIN);
586 rb = get_msg(t1, &msg);
587 ck_assert(rb == 0);
588
589 // Handle node 1's join on node 2
590 // Expected output: shift to S_GATHER and one join message
591 p2->handle_msg(jm);
592 ck_assert(p2->state() == Proto::S_GATHER);
593 rb = get_msg(t2, &jm);
594 ck_assert(rb != 0);
595 ck_assert(jm.type() == Message::EVS_T_JOIN);
596 rb = get_msg(t2, &msg);
597 ck_assert(rb == 0);
598
599 // Handle node 2's join on node 1
600 // Expected output: Install and commit gap messages, state stays in S_GATHER
601 p1->handle_msg(jm);
602 ck_assert(p1->state() == Proto::S_GATHER);
603 rb = get_msg(t1, &im);
604 ck_assert(rb != 0);
605 ck_assert(im.type() == Message::EVS_T_INSTALL);
606 rb = get_msg(t1, &gm);
607 ck_assert(rb != 0);
608 ck_assert(gm.type() == Message::EVS_T_GAP);
609 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
610 rb = get_msg(t1, &msg);
611 ck_assert(rb == 0);
612
613 // Handle install message on node 2
614 // Expected output: commit gap message and state stays in S_RECOVERY
615 p2->handle_msg(im);
616 ck_assert(p2->state() == Proto::S_GATHER);
617 rb = get_msg(t2, &gm2);
618 ck_assert(rb != 0);
619 ck_assert(gm2.type() == Message::EVS_T_GAP);
620 ck_assert((gm2.flags() & Message::F_COMMIT) != 0);
621 rb = get_msg(t2, &msg);
622 ck_assert(rb == 0);
623
624 // Handle gap messages
625 // Expected output: Both nodes shift to S_INSTALL,
626 // both send gap messages
627 p1->handle_msg(gm2);
628 ck_assert(p1->state() == Proto::S_INSTALL);
629 Message gm12;
630 rb = get_msg(t1, &gm12);
631 ck_assert(rb != 0);
632 ck_assert(gm12.type() == Message::EVS_T_GAP);
633 ck_assert((gm12.flags() & Message::F_COMMIT) == 0);
634 rb = get_msg(t1, &msg);
635 ck_assert(rb == 0);
636
637 p2->handle_msg(gm);
638 ck_assert(p2->state() == Proto::S_INSTALL);
639 Message gm22;
640 rb = get_msg(t2, &gm22);
641 ck_assert(rb != 0);
642 ck_assert(gm22.type() == Message::EVS_T_GAP);
643 ck_assert((gm22.flags() & Message::F_COMMIT) == 0);
644 rb = get_msg(t2, &msg);
645 ck_assert(rb == 0);
646
647 // Handle final gap messages, expected output shift to operational
648 // and gap message
649
650 p1->handle_msg(gm22);
651 ck_assert(p1->state() == Proto::S_OPERATIONAL);
652 rb = get_msg(t1, &msg);
653 ck_assert(rb != 0);
654 ck_assert(msg.type() == Message::EVS_T_GAP);
655 ck_assert((msg.flags() & Message::F_COMMIT) == 0);
656 rb = get_msg(t1, &msg);
657 ck_assert(rb == 0);
658
659 p2->handle_msg(gm12);
660 ck_assert(p2->state() == Proto::S_OPERATIONAL);
661 rb = get_msg(t2, &msg);
662 ck_assert(rb != 0);
663 ck_assert(msg.type() == Message::EVS_T_GAP);
664 ck_assert((msg.flags() & Message::F_COMMIT) == 0);
665 rb = get_msg(t2, &msg);
666 ck_assert(rb == 0);
667
668 }
669
670
START_TEST(test_proto_double_join)671 START_TEST(test_proto_double_join)
672 {
673 log_info << "START";
674 gu::Config conf;
675 mark_point();
676 gu::ssl_register_params(conf);
677 gcomm::Conf::register_params(conf);
678 UUID uuid1(1), uuid2(2);
679 DummyTransport t1(uuid1), t2(uuid2);
680 mark_point();
681 DummyUser u1(conf), u2(conf);
682 mark_point();
683 Proto p1(conf, uuid1, 0), p2(conf, uuid2, 0);
684
685 gcomm::connect(&t1, &p1);
686 gcomm::connect(&p1, &u1);
687
688 gcomm::connect(&t2, &p2);
689 gcomm::connect(&p2, &u2);
690
691 single_join(&t1, &p1);
692 double_join(&t1, &p1, &t2, &p2);
693
694 }
695 END_TEST
696
697 static gu::Config gu_conf;
698
create_dummy_node(size_t idx,int version,const string & suspect_timeout="PT1H",const string & inactive_timeout="PT1H",const string & retrans_period="PT10M")699 static DummyNode* create_dummy_node(size_t idx,
700 int version,
701 const string& suspect_timeout = "PT1H",
702 const string& inactive_timeout = "PT1H",
703 const string& retrans_period = "PT10M")
704 {
705 // reset conf to avoid stale config in case of nofork
706 gu_conf = gu::Config();
707 gu::ssl_register_params(gu_conf);
708 gcomm::Conf::register_params(gu_conf);
709 string conf = "evs://?" + Conf::EvsViewForgetTimeout + "=PT1H&"
710 + Conf::EvsInactiveCheckPeriod + "=" + to_string(Period(suspect_timeout)/3) + "&"
711 + Conf::EvsSuspectTimeout + "=" + suspect_timeout + "&"
712 + Conf::EvsInactiveTimeout + "=" + inactive_timeout + "&"
713
714 + Conf::EvsKeepalivePeriod + "=" + retrans_period + "&"
715 + Conf::EvsJoinRetransPeriod + "=" + retrans_period + "&"
716 + Conf::EvsInfoLogMask + "=0x7" + "&"
717 + Conf::EvsDebugLogMask + "=0xfff" + "&"
718 + Conf::EvsVersion + "=" + gu::to_string<int>(version);
719 if (::getenv("EVS_DEBUG_MASK") != 0)
720 {
721 conf += "&" + Conf::EvsDebugLogMask + "="
722 + ::getenv("EVS_DEBUG_MASK");
723 }
724 list<Protolay*> protos;
725 UUID uuid(static_cast<int32_t>(idx));
726 protos.push_back(new DummyTransport(uuid, false));
727 protos.push_back(new Proto(gu_conf, uuid, 0, conf));
728 return new DummyNode(gu_conf, idx, protos);
729 }
730
731 namespace
732 {
evs_from_dummy(DummyNode * dn)733 gcomm::evs::Proto* evs_from_dummy(DummyNode* dn)
734 {
735 return static_cast<Proto*>(dn->protos().back());
736 }
737
transport_from_dummy(DummyNode * dn)738 DummyTransport* transport_from_dummy(DummyNode* dn)
739 {
740 return static_cast<DummyTransport*>(dn->protos().front());
741 }
742 }
743
744
join_node(PropagationMatrix * p,DummyNode * n,bool first=false)745 static void join_node(PropagationMatrix* p,
746 DummyNode* n, bool first = false)
747 {
748 gu_trace(p->insert_tp(n));
749 gu_trace(n->connect(first));
750 }
751
752
send_n(DummyNode * node,const size_t n)753 static void send_n(DummyNode* node, const size_t n)
754 {
755 for (size_t i = 0; i < n; ++i)
756 {
757 gu_trace(node->send());
758 }
759 }
760
set_cvi(vector<DummyNode * > & nvec,size_t i_begin,size_t i_end,size_t seq)761 static void set_cvi(vector<DummyNode*>& nvec, size_t i_begin, size_t i_end,
762 size_t seq)
763 {
764 for (size_t i = i_begin; i <= i_end; ++i)
765 {
766 nvec[i]->set_cvi(ViewId(V_REG, nvec[i_begin]->uuid(),
767 static_cast<uint32_t>(seq)));
768 }
769 }
770
771 template <class C>
772 class ViewSeq
773 {
774 public:
ViewSeq()775 ViewSeq() { }
operator ()(const C & a,const C & b) const776 bool operator()(const C& a, const C& b) const
777 {
778 return (a->trace().current_view_trace().view().id().seq() < b->trace().current_view_trace().view().id().seq());
779 }
780 };
781
get_max_view_seq(const std::vector<DummyNode * > & dnv,size_t i,size_t j)782 static uint32_t get_max_view_seq(const std::vector<DummyNode*>& dnv,
783 size_t i, size_t j)
784 {
785 if (i == dnv.size()) return static_cast<uint32_t>(-1);
786 return (*std::max_element(dnv.begin() + i,
787 dnv.begin() + j,
788 ViewSeq<const DummyNode*>()))->trace().current_view_trace().view().id().seq();
789 }
790
791
792
START_TEST(test_proto_join_n)793 START_TEST(test_proto_join_n)
794 {
795 log_info << "START (join_n)";
796 init_rand();
797
798 const size_t n_nodes(4);
799 PropagationMatrix prop;
800 vector<DummyNode*> dn;
801
802 for (size_t i = 1; i <= n_nodes; ++i)
803 {
804 gu_trace(dn.push_back(create_dummy_node(i, 0)));
805 }
806
807 uint32_t max_view_seq(0);
808 for (size_t i = 0; i < n_nodes; ++i)
809 {
810 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
811 set_cvi(dn, 0, i, max_view_seq + 1);
812 gu_trace(prop.propagate_until_cvi(false));
813 max_view_seq = get_max_view_seq(dn, 0, i);
814 }
815 gu_trace(check_trace(dn));
816 for_each(dn.begin(), dn.end(), DeleteObject());
817 }
818 END_TEST
819
820
START_TEST(test_proto_join_n_w_user_msg)821 START_TEST(test_proto_join_n_w_user_msg)
822 {
823 gu_conf_self_tstamp_on();
824 log_info << "START (join_n_w_user_msg)";
825 init_rand();
826
827 const size_t n_nodes(4);
828 PropagationMatrix prop;
829 vector<DummyNode*> dn;
830 // @todo This test should terminate without these timeouts
831 const string suspect_timeout("PT1H");
832 const string inactive_timeout("PT1H");
833 const string retrans_period("PT0.1S");
834
835 for (size_t i = 1; i <= n_nodes; ++i)
836 {
837 gu_trace(dn.push_back(
838 create_dummy_node(i, 0, suspect_timeout,
839 inactive_timeout, retrans_period)));
840 }
841
842 uint32_t max_view_seq(0);
843 for (size_t i = 0; i < n_nodes; ++i)
844 {
845 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
846 set_cvi(dn, 0, i, max_view_seq + 1);
847 gu_trace(prop.propagate_until_cvi(true));
848 for (size_t j = 0; j <= i; ++j)
849 {
850 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
851 }
852 gu_trace(prop.propagate_until_empty());
853 for (size_t j = 0; j <= i; ++j)
854 {
855 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
856 }
857 max_view_seq = get_max_view_seq(dn, 0, i);
858 }
859
860 gu_trace(check_trace(dn));
861 for_each(dn.begin(), dn.end(), DeleteObject());
862 }
863 END_TEST
864
865
START_TEST(test_proto_join_n_lossy)866 START_TEST(test_proto_join_n_lossy)
867 {
868 gu_conf_self_tstamp_on();
869 log_info << "START (join_n_lossy)";
870 init_rand();
871
872 const size_t n_nodes(4);
873 PropagationMatrix prop;
874 vector<DummyNode*> dn;
875 const string suspect_timeout("PT1H");
876 const string inactive_timeout("PT1H");
877 const string retrans_period("PT0.1S");
878
879
880 for (size_t i = 1; i <= n_nodes; ++i)
881 {
882 gu_trace(dn.push_back(
883 create_dummy_node(i, 0, suspect_timeout,
884 inactive_timeout, retrans_period)));
885 }
886
887 uint32_t max_view_seq(0);
888 for (size_t i = 0; i < n_nodes; ++i)
889 {
890 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
891 set_cvi(dn, 0, i, max_view_seq + 1);
892 for (size_t j = 1; j < i + 1; ++j)
893 {
894 prop.set_loss(i + 1, j, 0.9);
895 prop.set_loss(j, i + 1, 0.9);
896 }
897 gu_trace(prop.propagate_until_cvi(true));
898 max_view_seq = get_max_view_seq(dn, 0, i);
899 }
900 gu_trace(check_trace(dn));
901 for_each(dn.begin(), dn.end(), DeleteObject());
902 }
903 END_TEST
904
905
START_TEST(test_proto_join_n_lossy_w_user_msg)906 START_TEST(test_proto_join_n_lossy_w_user_msg)
907 {
908 gu_conf_self_tstamp_on();
909 log_info << "START (join_n_lossy_w_user_msg)";
910 init_rand();
911
912 const size_t n_nodes(4);
913 PropagationMatrix prop;
914 vector<DummyNode*> dn;
915 const string suspect_timeout("PT1H");
916 const string inactive_timeout("PT1H");
917 const string retrans_period("PT0.1S");
918
919 for (size_t i = 1; i <= n_nodes; ++i)
920 {
921 gu_trace(dn.push_back(
922 create_dummy_node(i, 0, suspect_timeout,
923 inactive_timeout, retrans_period)));
924 }
925
926 uint32_t max_view_seq(0);
927 for (size_t i = 0; i < n_nodes; ++i)
928 {
929 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
930 set_cvi(dn, 0, i, max_view_seq + 1);
931 for (size_t j = 1; j < i + 1; ++j)
932 {
933 prop.set_loss(i + 1, j, 0.9);
934 prop.set_loss(j, i + 1, 0.9);
935
936 }
937 gu_trace(prop.propagate_until_cvi(true));
938 for (size_t j = 0; j < i; ++j)
939 {
940 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
941 }
942 max_view_seq = get_max_view_seq(dn, 0, i);
943 }
944 gu_trace(check_trace(dn));
945 for_each(dn.begin(), dn.end(), DeleteObject());
946 }
947 END_TEST
948
START_TEST(test_proto_leave_n)949 START_TEST(test_proto_leave_n)
950 {
951 gu_conf_self_tstamp_on();
952 log_info << "START (leave_n)";
953 init_rand();
954
955 const size_t n_nodes(4);
956 PropagationMatrix prop;
957 vector<DummyNode*> dn;
958
959 for (size_t i = 1; i <= n_nodes; ++i)
960 {
961 gu_trace(dn.push_back(create_dummy_node(i, 0)));
962 }
963
964 for (size_t i = 0; i < n_nodes; ++i)
965 {
966 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
967 set_cvi(dn, 0, i, i + 1);
968 gu_trace(prop.propagate_until_cvi(true));
969 }
970
971 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
972
973 for (size_t i = 0; i < n_nodes; ++i)
974 {
975 dn[i]->close();
976 dn[i]->set_cvi(V_REG);
977 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
978 gu_trace(prop.propagate_until_cvi(true));
979 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
980 }
981
982 gu_trace(check_trace(dn));
983 for_each(dn.begin(), dn.end(), DeleteObject());
984 }
985 END_TEST
986
START_TEST(test_proto_leave_n_w_user_msg)987 START_TEST(test_proto_leave_n_w_user_msg)
988 {
989 gu_conf_self_tstamp_on();
990 log_info << "START (leave_n_w_user_msg)";
991 init_rand();
992
993 const size_t n_nodes(4);
994 PropagationMatrix prop;
995 vector<DummyNode*> dn;
996 const string suspect_timeout("PT1H");
997 const string inactive_timeout("PT1H");
998 const string retrans_period("PT0.1S");
999
1000 for (size_t i = 1; i <= n_nodes; ++i)
1001 {
1002 gu_trace(dn.push_back(
1003 create_dummy_node(i, 0, suspect_timeout,
1004 inactive_timeout, retrans_period)));
1005 }
1006
1007 for (size_t i = 0; i < n_nodes; ++i)
1008 {
1009 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1010 set_cvi(dn, 0, i, i + 1);
1011 gu_trace(prop.propagate_until_cvi(false));
1012 }
1013
1014 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1015
1016 for (size_t i = 0; i < n_nodes; ++i)
1017 {
1018 for (size_t j = i; j < n_nodes; ++j)
1019 {
1020 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1021 }
1022 dn[i]->close();
1023 dn[i]->set_cvi(V_REG);
1024 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1025 gu_trace(prop.propagate_until_cvi(true));
1026 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1027 }
1028
1029 gu_trace(check_trace(dn));
1030 for_each(dn.begin(), dn.end(), DeleteObject());
1031 }
1032 END_TEST
1033
1034
START_TEST(test_proto_leave_n_lossy)1035 START_TEST(test_proto_leave_n_lossy)
1036 {
1037 if (deterministic_tests()) return;
1038
1039 gu_conf_self_tstamp_on();
1040 log_info << "START (leave_n_lossy)";
1041 init_rand();
1042 const size_t n_nodes(4);
1043 PropagationMatrix prop;
1044 vector<DummyNode*> dn;
1045 const string suspect_timeout("PT15S");
1046 const string inactive_timeout("PT30S");
1047 const string retrans_period("PT1S");
1048
1049 for (size_t i = 1; i <= n_nodes; ++i)
1050 {
1051 gu_trace(dn.push_back(
1052 create_dummy_node(i, 0, suspect_timeout,
1053 inactive_timeout, retrans_period)));
1054 }
1055
1056 for (size_t i = 0; i < n_nodes; ++i)
1057 {
1058 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1059 set_cvi(dn, 0, i, i + 1);
1060 gu_trace(prop.propagate_until_cvi(false));
1061 }
1062
1063 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1064
1065 for (size_t i = 0; i < n_nodes; ++i)
1066 {
1067 for (size_t j = 1; j < i + 1; ++j)
1068 {
1069 prop.set_loss(i + 1, j, 0.9);
1070 prop.set_loss(j, i + 1, 0.9);
1071 }
1072 }
1073
1074 for (size_t i = 0; i < n_nodes; ++i)
1075 {
1076 dn[i]->set_cvi(V_REG);
1077 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1078 dn[i]->close();
1079 gu_trace(prop.propagate_until_cvi(true));
1080 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1081 }
1082
1083 gu_trace(check_trace(dn));
1084 for_each(dn.begin(), dn.end(), DeleteObject());
1085 }
1086 END_TEST
1087
1088
1089
START_TEST(test_proto_leave_n_lossy_w_user_msg)1090 START_TEST(test_proto_leave_n_lossy_w_user_msg)
1091 {
1092 if (deterministic_tests()) return;
1093
1094 gu_conf_self_tstamp_on();
1095 log_info << "START (leave_n_lossy_w_user_msg)";
1096 init_rand();
1097
1098 const size_t n_nodes(4);
1099 PropagationMatrix prop;
1100 vector<DummyNode*> dn;
1101
1102 const string suspect_timeout("PT15S");
1103 const string inactive_timeout("PT30S");
1104 const string retrans_period("PT1S");
1105
1106 for (size_t i = 1; i <= n_nodes; ++i)
1107 {
1108 gu_trace(dn.push_back(
1109 create_dummy_node(i, 0, suspect_timeout,
1110 inactive_timeout, retrans_period)));
1111 }
1112
1113 for (size_t i = 0; i < n_nodes; ++i)
1114 {
1115 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1116 set_cvi(dn, 0, i, i + 1);
1117 gu_trace(prop.propagate_until_cvi(false));
1118 }
1119
1120
1121 for (size_t i = 0; i < n_nodes; ++i)
1122 {
1123 for (size_t j = 1; j < i + 1; ++j)
1124 {
1125 prop.set_loss(i + 1, j, 0.9);
1126 prop.set_loss(j, i + 1, 0.9);
1127 }
1128 }
1129
1130 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1131
1132 for (size_t i = 0; i < n_nodes; ++i)
1133 {
1134 for (size_t j = i; j < n_nodes; ++j)
1135 {
1136 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1137 }
1138 dn[i]->set_cvi(V_REG);
1139 set_cvi(dn, i + 1, n_nodes - 1, max_view_seq + 1);
1140 dn[i]->close();
1141 gu_trace(prop.propagate_until_cvi(true));
1142 max_view_seq = get_max_view_seq(dn, i + 1, n_nodes);
1143 }
1144
1145 gu_trace(check_trace(dn));
1146 for_each(dn.begin(), dn.end(), DeleteObject());
1147 }
1148 END_TEST
1149
1150
1151 // Generic test code for split/merge cases
test_proto_split_merge_gen(const size_t n_nodes,const bool send_msgs,const double loss)1152 static void test_proto_split_merge_gen(const size_t n_nodes,
1153 const bool send_msgs,
1154 const double loss)
1155 {
1156 PropagationMatrix prop;
1157 vector<DummyNode*> dn;
1158 const string suspect_timeout("PT15S");
1159 const string inactive_timeout("PT30S");
1160 const string retrans_period("PT1S");
1161
1162 for (size_t i = 1; i <= n_nodes; ++i)
1163 {
1164 gu_trace(dn.push_back(
1165 create_dummy_node(i, 0, suspect_timeout,
1166 inactive_timeout, retrans_period)));
1167 }
1168
1169 for (size_t i = 0; i < n_nodes; ++i)
1170 {
1171 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1172 set_cvi(dn, 0, i, i + 1);
1173 gu_trace(prop.propagate_until_cvi(false));
1174 }
1175
1176 for (size_t i = 0; i < n_nodes; ++i)
1177 {
1178 for (size_t j = 1; j < i + 1; ++j)
1179 {
1180 prop.set_loss(i + 1, j, loss);
1181 prop.set_loss(j, i + 1, loss);
1182 }
1183 }
1184
1185 vector<int32_t> split;
1186 for (size_t i = 0; i < n_nodes; ++i)
1187 {
1188 split.push_back(static_cast<int32_t>(i + 1));
1189 }
1190
1191 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1192
1193 for (size_t i = 1; i < n_nodes; ++i)
1194 {
1195 if (send_msgs == true)
1196 {
1197 for (size_t k = 0; k < 5; ++k)
1198 {
1199 for (size_t j = 0; j < n_nodes; ++j)
1200 {
1201 gu_trace(send_n(dn[j], 1 + j));
1202 }
1203 gu_trace(prop.propagate_n(7));
1204 }
1205 }
1206
1207 log_info << "split " << i;
1208 for (size_t j = 0; j < i; ++j)
1209 {
1210 for (size_t k = i; k < n_nodes; ++k)
1211 {
1212 gu_trace(prop.set_loss(split[j], split[k], 0.));
1213 gu_trace(prop.set_loss(split[k], split[j], 0.));
1214 }
1215 }
1216
1217 set_cvi(dn, 0, i - 1, max_view_seq + 1);
1218 set_cvi(dn, i, n_nodes - 1, max_view_seq + 1);
1219
1220 if (send_msgs == true)
1221 {
1222 for (size_t j = 0; j < n_nodes; ++j)
1223 {
1224 gu_trace(send_n(dn[j], 5 + rand() % 4));
1225 }
1226 }
1227
1228 gu_trace(prop.propagate_until_cvi(true));
1229 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1230 log_info << "merge " << i;
1231 for (size_t j = 0; j < i; ++j)
1232 {
1233 for (size_t k = i; k < n_nodes; ++k)
1234 {
1235 gu_trace(prop.set_loss(split[j], split[k], loss));
1236 gu_trace(prop.set_loss(split[k], split[j], loss));
1237 }
1238 }
1239
1240 set_cvi(dn, 0, n_nodes - 1, max_view_seq + 1);
1241
1242 if (send_msgs == true)
1243 {
1244 for (size_t j = 0; j < n_nodes; ++j)
1245 {
1246 gu_trace(send_n(dn[j], 5 + rand() % 4));
1247 }
1248 }
1249 gu_trace(prop.propagate_until_cvi(true));
1250 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1251 }
1252
1253 gu_trace(prop.propagate_until_empty());
1254
1255 gu_trace(check_trace(dn));
1256 for_each(dn.begin(), dn.end(), DeleteObject());
1257 }
1258
1259
1260
START_TEST(test_proto_split_merge)1261 START_TEST(test_proto_split_merge)
1262 {
1263 gu_conf_self_tstamp_on();
1264 log_info << "START (split_merge)";
1265 init_rand();
1266
1267 test_proto_split_merge_gen(4, false, 1.);
1268 }
1269 END_TEST
1270
1271
START_TEST(test_proto_split_merge_lossy)1272 START_TEST(test_proto_split_merge_lossy)
1273 {
1274 if (deterministic_tests()) return;
1275
1276 gu_conf_self_tstamp_on();
1277 log_info << "START (split_merge_lossy)";
1278 init_rand();
1279
1280 test_proto_split_merge_gen(4, false, .9);
1281 }
1282 END_TEST
1283
1284
1285
START_TEST(test_proto_split_merge_w_user_msg)1286 START_TEST(test_proto_split_merge_w_user_msg)
1287 {
1288 gu_conf_self_tstamp_on();
1289 log_info << "START (split_merge_w_user_msg)";
1290 init_rand();
1291
1292 test_proto_split_merge_gen(4, true, 1.);
1293
1294 }
1295 END_TEST
1296
1297
START_TEST(test_proto_split_merge_lossy_w_user_msg)1298 START_TEST(test_proto_split_merge_lossy_w_user_msg)
1299 {
1300 if (deterministic_tests()) return;
1301
1302 gu_conf_self_tstamp_on();
1303 log_info << "START (split_merge_lossy_w_user_msg)";
1304 init_rand();
1305
1306 test_proto_split_merge_gen(4, true, .9);
1307 }
1308 END_TEST
1309
START_TEST(test_proto_stop_cont)1310 START_TEST(test_proto_stop_cont)
1311 {
1312 log_info << "START";
1313 init_rand();
1314
1315 const size_t n_nodes(4);
1316 PropagationMatrix prop;
1317 vector<DummyNode*> dn;
1318 const string suspect_timeout("PT0.31S");
1319 const string inactive_timeout("PT0.31S");
1320 const string retrans_period("PT0.1S");
1321
1322 for (size_t i = 1; i <= n_nodes; ++i)
1323 {
1324 gu_trace(dn.push_back(
1325 create_dummy_node(i, 0, suspect_timeout,
1326 inactive_timeout, retrans_period)));
1327 }
1328
1329 for (size_t i = 0; i < n_nodes; ++i)
1330 {
1331 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1332 set_cvi(dn, 0, i, i + 1);
1333 gu_trace(prop.propagate_until_cvi(false));
1334 }
1335 uint32_t view_seq = n_nodes + 1;
1336
1337 for (size_t i = 0; i < n_nodes; ++i)
1338 {
1339 for (size_t j = 0; j < n_nodes; ++j)
1340 {
1341 if (j != i)
1342 {
1343 dn[j]->close(dn[i]->uuid());
1344 }
1345 }
1346 set_cvi(dn, 0, n_nodes - 1, view_seq + 1);
1347 gu_trace(prop.propagate_until_cvi(true));
1348 view_seq += 2;
1349
1350 }
1351 gu_trace(check_trace(dn));
1352 for_each(dn.begin(), dn.end(), DeleteObject());
1353 }
1354 END_TEST
1355
1356
START_TEST(test_proto_arbitrate)1357 START_TEST(test_proto_arbitrate)
1358 {
1359 log_info << "START";
1360 const size_t n_nodes(3);
1361 PropagationMatrix prop;
1362 vector<DummyNode*> dn;
1363 const string suspect_timeout("PT0.5S");
1364 const string inactive_timeout("PT0.5S");
1365 const string retrans_period("PT0.1S");
1366
1367 for (size_t i = 1; i <= n_nodes; ++i)
1368 {
1369 gu_trace(dn.push_back(
1370 create_dummy_node(i, 0,
1371 suspect_timeout,
1372 inactive_timeout, retrans_period)));
1373 }
1374
1375 for (size_t i = 0; i < n_nodes; ++i)
1376 {
1377 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1378 set_cvi(dn, 0, i, i + 1);
1379 gu_trace(prop.propagate_until_cvi(false));
1380 }
1381 uint32_t view_seq = n_nodes + 1;
1382
1383 dn[0]->close(dn[1]->uuid());
1384 dn[1]->close(dn[0]->uuid());
1385 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1386 dn[2]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1387 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), view_seq));
1388 gu_trace(prop.propagate_until_cvi(true));
1389
1390 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1391 dn[1]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1392 dn[2]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1393 gu_trace(prop.propagate_until_cvi(true));
1394
1395 gu_trace(check_trace(dn));
1396
1397 for_each(dn.begin(), dn.end(), DeleteObject());
1398 }
1399 END_TEST
1400
1401
START_TEST(test_proto_split_two)1402 START_TEST(test_proto_split_two)
1403 {
1404 log_info << "START";
1405 const size_t n_nodes(2);
1406 PropagationMatrix prop;
1407 vector<DummyNode*> dn;
1408 const string suspect_timeout("PT0.31S");
1409 const string inactive_timeout("PT0.31S");
1410 const string retrans_period("PT0.1S");
1411
1412 for (size_t i = 1; i <= n_nodes; ++i)
1413 {
1414 gu_trace(dn.push_back(
1415 create_dummy_node(i, 0, suspect_timeout,
1416 inactive_timeout, retrans_period)));
1417 }
1418
1419 for (size_t i = 0; i < n_nodes; ++i)
1420 {
1421 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1422 set_cvi(dn, 0, i, i + 1);
1423 gu_trace(prop.propagate_until_cvi(false));
1424 }
1425 uint32_t view_seq = n_nodes + 1;
1426
1427 dn[0]->close(dn[1]->uuid());
1428 dn[1]->close(dn[0]->uuid());
1429 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq));
1430 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), view_seq));
1431
1432 gu_trace(prop.propagate_until_cvi(true));
1433
1434 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1435 dn[1]->set_cvi(ViewId(V_REG, dn[0]->uuid(), view_seq + 1));
1436 gu_trace(prop.propagate_until_cvi(true));
1437
1438 gu_trace(check_trace(dn));
1439
1440 for_each(dn.begin(), dn.end(), DeleteObject());
1441 }
1442 END_TEST
1443
START_TEST(test_aggreg)1444 START_TEST(test_aggreg)
1445 {
1446 log_info << "START";
1447 const size_t n_nodes(2);
1448 PropagationMatrix prop;
1449 vector<DummyNode*> dn;
1450 const string suspect_timeout("PT0.31S");
1451 const string inactive_timeout("PT0.31S");
1452 const string retrans_period("PT0.1S");
1453
1454 for (size_t i = 1; i <= n_nodes; ++i)
1455 {
1456 gu_trace(dn.push_back(
1457 create_dummy_node(i, 0, suspect_timeout,
1458 inactive_timeout, retrans_period)));
1459 }
1460
1461 for (size_t i = 0; i < n_nodes; ++i)
1462 {
1463 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1464 set_cvi(dn, 0, i, i + 1);
1465 gu_trace(prop.propagate_until_cvi(false));
1466 }
1467
1468 for (size_t i = 0; i < n_nodes; ++i)
1469 {
1470 gu_trace(send_n(dn[i], 8));
1471 }
1472
1473 gu_trace(prop.propagate_until_empty());
1474 gu_trace(check_trace(dn));
1475
1476 for_each(dn.begin(), dn.end(), DeleteObject());
1477 }
1478 END_TEST
1479
START_TEST(test_trac_538)1480 START_TEST(test_trac_538)
1481 {
1482 gu_conf_self_tstamp_on();
1483 log_info << "START (test_trac_538)";
1484 init_rand();
1485 const size_t n_nodes(5);
1486 PropagationMatrix prop;
1487 vector<DummyNode*> dn;
1488 const string suspect_timeout("PT0.5S");
1489 const string inactive_timeout("PT2S");
1490 const string retrans_period("PT0.1S");
1491
1492 for (size_t i = 1; i <= n_nodes; ++i)
1493 {
1494 gu_trace(dn.push_back(
1495 create_dummy_node(i, 0, suspect_timeout,
1496 inactive_timeout,
1497 retrans_period)));
1498 }
1499
1500 for (size_t i = 0; i < n_nodes - 1; ++i)
1501 {
1502 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1503 set_cvi(dn, 0, i, i + 1);
1504 gu_trace(prop.propagate_until_cvi(false));
1505 }
1506
1507 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes - 1));
1508
1509 gu_trace(join_node(&prop, dn[n_nodes - 1], false));
1510 for (size_t i = 1; i <= n_nodes; ++i)
1511 {
1512 if (i != n_nodes - 1)
1513 {
1514 prop.set_loss(i, n_nodes - 1, 0);
1515 prop.set_loss(n_nodes - 1, i, 0);
1516 }
1517 }
1518 set_cvi(dn, 0, n_nodes - 1, max_view_seq + 1);
1519 dn[n_nodes - 2]->set_cvi(ViewId(V_REG, n_nodes - 1, max_view_seq + 1));
1520 gu_trace(prop.propagate_until_cvi(true));
1521 gu_trace(check_trace(dn));
1522 for_each(dn.begin(), dn.end(), DeleteObject());
1523 }
1524 END_TEST
1525
1526
START_TEST(test_trac_552)1527 START_TEST(test_trac_552)
1528 {
1529 log_info << "START (trac_552)";
1530 init_rand();
1531
1532 const size_t n_nodes(3);
1533 PropagationMatrix prop;
1534 vector<DummyNode*> dn;
1535
1536 const string suspect_timeout("PT15S");
1537 const string inactive_timeout("PT30S");
1538 const string retrans_period("PT1S");
1539
1540 for (size_t i = 1; i <= n_nodes; ++i)
1541 {
1542 gu_trace(dn.push_back(
1543 create_dummy_node(i, 0, suspect_timeout,
1544 inactive_timeout, retrans_period)));
1545 }
1546
1547 for (size_t i = 0; i < n_nodes; ++i)
1548 {
1549 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1550 set_cvi(dn, 0, i, i + 1);
1551 gu_trace(prop.propagate_until_cvi(false));
1552 }
1553
1554
1555 for (size_t i = 0; i < n_nodes; ++i)
1556 {
1557 for (size_t j = 1; j < i + 1; ++j)
1558 {
1559 prop.set_loss(i + 1, j, 0.9);
1560 prop.set_loss(j, i + 1, 0.9);
1561 }
1562 }
1563
1564 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1565
1566 for (size_t j = 0; j < n_nodes; ++j)
1567 {
1568 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
1569 }
1570 dn[0]->set_cvi(V_REG);
1571 dn[1]->set_cvi(V_REG);
1572 set_cvi(dn, 2, n_nodes - 1, max_view_seq + 1);
1573 dn[0]->close();
1574 dn[1]->close();
1575 gu_trace(prop.propagate_until_cvi(true));
1576
1577 gu_trace(check_trace(dn));
1578 for_each(dn.begin(), dn.end(), DeleteObject());
1579 }
1580 END_TEST
1581
1582
START_TEST(test_trac_607)1583 START_TEST(test_trac_607)
1584 {
1585 gu_conf_self_tstamp_on();
1586 log_info << "START (trac_607)";
1587
1588 const size_t n_nodes(3);
1589 PropagationMatrix prop;
1590 vector<DummyNode*> dn;
1591
1592 const string suspect_timeout("PT0.5S");
1593 const string inactive_timeout("PT1S");
1594 const string retrans_period("PT0.1S");
1595
1596 for (size_t i = 1; i <= n_nodes; ++i)
1597 {
1598 gu_trace(dn.push_back(
1599 create_dummy_node(i, 0, suspect_timeout,
1600 inactive_timeout, retrans_period)));
1601 }
1602
1603 for (size_t i = 0; i < n_nodes; ++i)
1604 {
1605 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1606 set_cvi(dn, 0, i, i + 1);
1607 gu_trace(prop.propagate_until_cvi(false));
1608 }
1609
1610 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1611 dn[0]->set_cvi(V_REG);
1612 dn[0]->close();
1613
1614 while (evs_from_dummy(dn[1])->state() != Proto::S_INSTALL)
1615 {
1616 prop.propagate_n(1);
1617 }
1618
1619 // this used to cause exception:
1620 // Forbidden state transition: INSTALL -> LEAVING (FATAL)
1621 dn[1]->close();
1622
1623 // expected behavior:
1624 // dn[1], dn[2] reach S_OPERATIONAL and then dn[1] leaves gracefully
1625 set_cvi(dn, 1, n_nodes - 1, max_view_seq + 1);
1626
1627 gu_trace(prop.propagate_until_cvi(true));
1628 max_view_seq = get_max_view_seq(dn, 0, n_nodes);
1629 dn[1]->set_cvi(V_REG);
1630 set_cvi(dn, 2, 2, max_view_seq + 1);
1631
1632 gu_trace(prop.propagate_until_cvi(true));
1633
1634 gu_trace(check_trace(dn));
1635 for_each(dn.begin(), dn.end(), DeleteObject());
1636 }
1637 END_TEST
1638
1639
START_TEST(test_trac_724)1640 START_TEST(test_trac_724)
1641 {
1642 gu_conf_self_tstamp_on();
1643 log_info << "START (trac_724)";
1644 init_rand();
1645
1646 const size_t n_nodes(2);
1647 PropagationMatrix prop;
1648 vector<DummyNode*> dn;
1649 Protolay::sync_param_cb_t sync_param_cb;
1650
1651 const string suspect_timeout("PT0.5S");
1652 const string inactive_timeout("PT1S");
1653 const string retrans_period("PT0.1S");
1654
1655 for (size_t i = 1; i <= n_nodes; ++i)
1656 {
1657 gu_trace(dn.push_back(
1658 create_dummy_node(i, 0, suspect_timeout,
1659 inactive_timeout, retrans_period)));
1660 }
1661
1662 for (size_t i = 0; i < n_nodes; ++i)
1663 {
1664 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1665 set_cvi(dn, 0, i, i + 1);
1666 gu_trace(prop.propagate_until_cvi(false));
1667 }
1668
1669 // Slightly asymmetric settings and evs.use_aggregate=false to
1670 // allow completion window to grow over 0xff.
1671 Proto* evs0(evs_from_dummy(dn[0]));
1672
1673 bool ret(evs0->set_param("evs.use_aggregate", "false", sync_param_cb));
1674 ck_assert(ret == true);
1675 ret = evs0->set_param("evs.send_window", "1024", sync_param_cb);
1676 ck_assert(ret == true);
1677 ret = evs0->set_param("evs.user_send_window", "515", sync_param_cb);
1678 Proto* evs1(evs_from_dummy(dn[1]));
1679 ret = evs1->set_param("evs.use_aggregate", "false", sync_param_cb);
1680 ck_assert(ret == true);
1681 ret = evs1->set_param("evs.send_window", "1024", sync_param_cb);
1682 ck_assert(ret == true);
1683 ret = evs1->set_param("evs.user_send_window", "512", sync_param_cb);
1684
1685 prop.set_loss(1, 2, 0.);
1686
1687 for (size_t i(0); i < 256; ++i)
1688 {
1689 dn[0]->send();
1690 dn[0]->send();
1691 dn[1]->send();
1692 gu_trace(prop.propagate_until_empty());
1693 }
1694 dn[0]->send();
1695 prop.set_loss(1, 2, 1.);
1696
1697 dn[0]->send();
1698 gu_trace(prop.propagate_until_empty());
1699
1700 gu_trace(check_trace(dn));
1701 for_each(dn.begin(), dn.end(), DeleteObject());
1702 }
1703 END_TEST
1704
1705
START_TEST(test_trac_760)1706 START_TEST(test_trac_760)
1707 {
1708 gu_conf_self_tstamp_on();
1709 log_info << "START (trac_760)";
1710 init_rand();
1711
1712 const size_t n_nodes(3);
1713 PropagationMatrix prop;
1714 vector<DummyNode*> dn;
1715
1716 const string suspect_timeout("PT0.5S");
1717 const string inactive_timeout("PT1S");
1718 const string retrans_period("PT0.1S");
1719
1720 for (size_t i = 1; i <= n_nodes; ++i)
1721 {
1722 gu_trace(dn.push_back(
1723 create_dummy_node(i, 0, suspect_timeout,
1724 inactive_timeout, retrans_period)));
1725 }
1726
1727 for (size_t i = 0; i < n_nodes; ++i)
1728 {
1729 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1730 set_cvi(dn, 0, i, i + 1);
1731 gu_trace(prop.propagate_until_cvi(false));
1732 }
1733
1734 for (size_t i = 0; i < n_nodes; ++i)
1735 {
1736 gu_trace(send_n(dn[i], 2));
1737
1738 }
1739 gu_trace(prop.propagate_until_empty());
1740
1741
1742 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1743 gu_trace(send_n(dn[0], 1));
1744 gu_trace(send_n(dn[1], 1));
1745 // gu_trace(send_n(dn[2], 1));
1746
1747 set_cvi(dn, 0, 1, max_view_seq + 1);
1748 dn[2]->set_cvi(V_REG);
1749 dn[2]->close();
1750
1751 Proto* evs0(evs_from_dummy(dn[0]));
1752 Proto* evs1(evs_from_dummy(dn[1]));
1753 while (evs1->state() != Proto::S_GATHER && evs0->state() != Proto::S_GATHER)
1754 {
1755 gu_trace(prop.propagate_n(1));
1756 }
1757 dn[1]->close();
1758
1759 gu_trace(prop.propagate_until_cvi(true));
1760
1761 gu_trace(check_trace(dn));
1762 for_each(dn.begin(), dn.end(), DeleteObject());
1763 }
1764 END_TEST
1765
START_TEST(test_gh_41)1766 START_TEST(test_gh_41)
1767 {
1768 gu_conf_self_tstamp_on();
1769 log_info << "START (gh_41)";
1770
1771 const size_t n_nodes(3);
1772 PropagationMatrix prop;
1773 vector<DummyNode*> dn;
1774
1775 const string suspect_timeout("PT0.5S");
1776 const string inactive_timeout("PT1S");
1777 const string retrans_period("PT0.1S");
1778
1779 for (size_t i = 1; i <= n_nodes; ++i)
1780 {
1781 gu_trace(dn.push_back(
1782 create_dummy_node(i, 0, suspect_timeout,
1783 inactive_timeout, retrans_period)));
1784 }
1785
1786 for (size_t i = 0; i < n_nodes; ++i)
1787 {
1788 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1789 set_cvi(dn, 0, i, i + 1);
1790 gu_trace(prop.propagate_until_cvi(false));
1791 }
1792
1793 // Generate partitioning so that the node with smallest UUID
1794 // creates singleton view
1795 log_info << "partition";
1796 prop.set_loss(1, 2, 0.);
1797 prop.set_loss(2, 1, 0.);
1798 prop.set_loss(1, 3, 0.);
1799 prop.set_loss(3, 1, 0.);
1800 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1801
1802 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), max_view_seq + 1));
1803 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1804 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1805
1806 prop.propagate_until_cvi(true);
1807
1808 // Merge groups and make node 1 leave so that nodes 2 and 3 see
1809 // leave message from unknown origin
1810 log_info << "merge";
1811 prop.set_loss(1, 2, 1.);
1812 prop.set_loss(2, 1, 1.);
1813 prop.set_loss(1, 3, 1.);
1814 prop.set_loss(3, 1, 1.);
1815
1816 // Send message so that nodes 2 and 3 shift to GATHER. This must be done
1817 // because LEAVE message is ignored in handle_foreign()
1818 dn[0]->send();
1819 dn[0]->close();
1820
1821 dn[0]->set_cvi(V_REG);
1822 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 2));
1823 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 2));
1824
1825 prop.propagate_until_cvi(true);
1826 check_trace(dn);
1827 for_each(dn.begin(), dn.end(), DeleteObject());
1828 }
1829 END_TEST
1830
START_TEST(test_gh_37)1831 START_TEST(test_gh_37)
1832 {
1833 gu_conf_self_tstamp_on();
1834 log_info << "START (gh_37)";
1835
1836 const size_t n_nodes(3);
1837 PropagationMatrix prop;
1838 vector<DummyNode*> dn;
1839
1840 const string suspect_timeout("PT0.5S");
1841 const string inactive_timeout("PT1S");
1842 const string retrans_period("PT0.1S");
1843
1844 for (size_t i = 1; i <= n_nodes; ++i)
1845 {
1846 gu_trace(dn.push_back(
1847 create_dummy_node(i, 0, suspect_timeout,
1848 inactive_timeout, retrans_period)));
1849 }
1850
1851 for (size_t i = 0; i < n_nodes; ++i)
1852 {
1853 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1854 set_cvi(dn, 0, i, i + 1);
1855 gu_trace(prop.propagate_until_cvi(false));
1856 }
1857
1858 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1859 // node 0 is gonna to leave
1860 for(size_t i = 2; i <= n_nodes; i++)
1861 {
1862 // leaving node(LN) is able to send messages to remaining nodes.
1863 // prop.set_loss(1, i, 0.);
1864 // but remaining nodes(RNS) won't be able to ack these messages.
1865 prop.set_loss(i, 1, 0.);
1866 // so RNS aru_seq are the same and higher than LN aru_seq.
1867 }
1868 // LN ss=-1, ir=[2,1]
1869 // RNS ss=1, ir=[2,1]
1870 dn[0]->send();
1871 dn[0]->send();
1872 dn[0]->close();
1873
1874 dn[0]->set_cvi(V_REG);
1875 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1876 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1877
1878 prop.propagate_until_cvi(true);
1879 check_trace(dn);
1880 for_each(dn.begin(), dn.end(), DeleteObject());
1881 }
1882 END_TEST
1883
START_TEST(test_gh_40)1884 START_TEST(test_gh_40)
1885 {
1886 gu_conf_self_tstamp_on();
1887 log_info << "START (gh_40)";
1888
1889 const size_t n_nodes(3);
1890 PropagationMatrix prop;
1891 vector<DummyNode*> dn;
1892
1893 const string suspect_timeout("PT0.5S");
1894 const string inactive_timeout("PT1S");
1895 const string retrans_period("PT0.1S");
1896
1897 for (size_t i = 1; i <= n_nodes; ++i)
1898 {
1899 gu_trace(dn.push_back(
1900 create_dummy_node(i, 0, suspect_timeout,
1901 inactive_timeout, retrans_period)));
1902 }
1903
1904 for (size_t i = 0; i < n_nodes; ++i)
1905 {
1906 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
1907 set_cvi(dn, 0, i, i + 1);
1908 gu_trace(prop.propagate_until_cvi(false));
1909 }
1910 uint32_t max_view_seq(get_max_view_seq(dn, 0, n_nodes));
1911
1912 // ss=0, ir=[1,0];
1913 dn[0]->send();
1914 gu_trace(prop.propagate_until_empty());
1915 log_info << "gh_40 all got operational state";
1916
1917 // cut dn[0] from dn[1] and dn[2].
1918 for (size_t i = 2; i <= n_nodes; ++i)
1919 {
1920 prop.set_loss(1, i, 0.);
1921 prop.set_loss(i, 1, 0.);
1922 }
1923
1924 // ss=0, ir=[2,1];
1925 // dn[1] send msg(seq=1)
1926 dn[1]->send();
1927
1928 Proto* evs1 = evs_from_dummy(dn[1]);
1929 Proto* evs2 = evs_from_dummy(dn[2]);
1930 ck_assert(evs1->state() == Proto::S_OPERATIONAL);
1931 ck_assert(evs2->state() == Proto::S_OPERATIONAL);
1932 evs1->set_inactive(dn[0]->uuid());
1933 evs2->set_inactive(dn[0]->uuid());
1934 evs1->check_inactive();
1935 evs2->check_inactive();
1936 ck_assert(evs1->state() == Proto::S_GATHER);
1937 ck_assert(evs2->state() == Proto::S_GATHER);
1938
1939 // Advance clock to get over join message rate limiting.
1940 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
1941 while(!(evs1->state() == Proto::S_GATHER &&
1942 evs1->is_install_message()))
1943 {
1944 gu_trace(prop.propagate_n(1));
1945 }
1946
1947 // dn[0] comes back.
1948 // here we have to set message F_RETRANS
1949 // otherwise handle_msg ignores this msg.
1950 // @todo:why?
1951
1952 // dn[0] ack dn[1] msg(seq=1) with flags F_RETRANS.
1953 Datagram dg1 = dn[0]->create_datagram();
1954 UserMessage msg1(0,
1955 dn[0]->uuid(),
1956 ViewId(V_REG, dn[0]->uuid(), max_view_seq),
1957 1, 0, 0, O_DROP, 1, 0xff,
1958 Message::F_RETRANS);
1959 // dn[0] msg(seq=2) leak into dn[1] input_map.
1960 Datagram dg2 = dn[0]->create_datagram();
1961 UserMessage msg2(0,
1962 dn[0]->uuid(),
1963 ViewId(V_REG, dn[0]->uuid(), max_view_seq),
1964 2, 0, 0, O_SAFE, 2, 0xff,
1965 Message::F_RETRANS);
1966 // so for dn[1]
1967 // input_map: ss=0, ir=[3,2]
1968 // install message: ss=0, ir=[2,1]
1969 // seq 1 = O_SAFE message.(initiated by self)
1970 // seq 2 = O_DROP message.(complete_user)
1971 push_header(msg1, dg1);
1972 evs1->handle_up(0, dg1, ProtoUpMeta(dn[0]->uuid()));
1973 push_header(msg2, dg2);
1974 log_info << "evs1 handle msg " << msg2;
1975 log_info << "before handle msg: " << *evs1;
1976 evs1->handle_up(0, dg2, ProtoUpMeta(dn[0]->uuid()));
1977 log_info << "after handle msg: " << *evs1;
1978
1979 dn[0]->set_cvi(ViewId(V_REG, dn[0]->uuid(), max_view_seq + 1));
1980 dn[1]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1981 dn[2]->set_cvi(ViewId(V_REG, dn[1]->uuid(), max_view_seq + 1));
1982 prop.propagate_until_cvi(true);
1983 check_trace(dn);
1984 for_each(dn.begin(), dn.end(), DeleteObject());
1985 }
1986 END_TEST
1987
1988
START_TEST(test_gh_100)1989 START_TEST(test_gh_100)
1990 {
1991 log_info << "START (test_gh_100)";
1992 gu::Config conf;
1993 mark_point();
1994 gu::ssl_register_params(conf);
1995 gcomm::Conf::register_params(conf);
1996 conf.set("evs.info_log_mask", "0x3");
1997 conf.set("evs.debug_log_mask", "0xa0");
1998 UUID uuid1(1), uuid2(2);
1999 DummyTransport t1(uuid1), t2(uuid2);
2000 mark_point();
2001 DummyUser u1(conf), u2(conf);
2002 mark_point();
2003 Proto p1(conf, uuid1, 0, gu::URI("evs://"), 10000, 0);
2004 // Start p2 view seqno from higher value than p1
2005 View p2_rst_view(0, ViewId(V_REG, uuid2, 3));
2006 Proto p2(conf, uuid2, 0, gu::URI("evs://"), 10000, &p2_rst_view);
2007
2008 gcomm::connect(&t1, &p1);
2009 gcomm::connect(&p1, &u1);
2010
2011 gcomm::connect(&t2, &p2);
2012 gcomm::connect(&p2, &u2);
2013
2014 single_join(&t1, &p1);
2015
2016
2017 // The following is from double_join(). Process messages until
2018 // install message is generated. After that handle install timer
2019 // on p1 and verify that the newly generated install message has
2020 // greater install view id seqno than the first one.
2021 Message jm;
2022 Message im;
2023 Message im2;
2024 Message gm;
2025 Message gm2;
2026 Message msg;
2027
2028 Datagram* rb;
2029
2030 // Initial states check
2031 p2.shift_to(Proto::S_JOINING);
2032 ck_assert(p1.state() == Proto::S_OPERATIONAL);
2033 ck_assert(p2.state() == Proto::S_JOINING);
2034
2035 // Send join message, don't self handle immediately
2036 // Expected output: one join message
2037 p2.send_join(false);
2038 ck_assert(p2.state() == Proto::S_JOINING);
2039 rb = get_msg(&t2, &jm);
2040 ck_assert(rb != 0);
2041 ck_assert(jm.type() == Message::EVS_T_JOIN);
2042 rb = get_msg(&t2, &msg);
2043 ck_assert(rb == 0);
2044
2045 // Handle node 2's join on node 1
2046 // Expected output: shift to S_GATHER and one join message
2047 p1.handle_msg(jm);
2048 ck_assert(p1.state() == Proto::S_GATHER);
2049 rb = get_msg(&t1, &jm);
2050 ck_assert(rb != 0);
2051 ck_assert(jm.type() == Message::EVS_T_JOIN);
2052 rb = get_msg(&t1, &msg);
2053 ck_assert(rb == 0);
2054
2055 // Handle node 1's join on node 2
2056 // Expected output: shift to S_GATHER and one join message
2057 p2.handle_msg(jm);
2058 ck_assert(p2.state() == Proto::S_GATHER);
2059 rb = get_msg(&t2, &jm);
2060 ck_assert(rb != 0);
2061 ck_assert(jm.type() == Message::EVS_T_JOIN);
2062 rb = get_msg(&t2, &msg);
2063 ck_assert(rb == 0);
2064
2065 // Handle node 2's join on node 1
2066 // Expected output: Install and commit gap messages, state stays in S_GATHER
2067 p1.handle_msg(jm);
2068 ck_assert(p1.state() == Proto::S_GATHER);
2069 rb = get_msg(&t1, &im);
2070 ck_assert(rb != 0);
2071 ck_assert(im.type() == Message::EVS_T_INSTALL);
2072 rb = get_msg(&t1, &gm);
2073 ck_assert(rb != 0);
2074 ck_assert(gm.type() == Message::EVS_T_GAP);
2075 ck_assert((gm.flags() & Message::F_COMMIT) != 0);
2076 rb = get_msg(&t1, &msg);
2077 ck_assert(rb == 0);
2078
2079 // Handle timers to to generate shift to GATHER
2080 p1.handle_inactivity_timer();
2081 p1.handle_install_timer();
2082 rb = get_msg(&t1, &jm);
2083 ck_assert(rb != 0);
2084 ck_assert(jm.type() == Message::EVS_T_JOIN);
2085 rb = get_msg(&t1, &im2);
2086 ck_assert(rb != 0);
2087 ck_assert(im2.type() == Message::EVS_T_INSTALL);
2088 ck_assert(im2.install_view_id().seq() > im.install_view_id().seq());
2089
2090 gcomm::Datagram* tmp;
2091 while ((tmp = t1.out())) delete tmp;
2092 while ((tmp = t2.out())) delete tmp;
2093 }
2094 END_TEST
2095
START_TEST(test_evs_protocol_upgrade)2096 START_TEST(test_evs_protocol_upgrade)
2097 {
2098 log_info << "START (test_evs_protocol_upgrade)";
2099 PropagationMatrix prop;
2100 vector<DummyNode*> dn;
2101
2102 uint32_t view_seq(0);
2103 for (int i(0); i <= GCOMM_PROTOCOL_MAX_VERSION; ++i)
2104 {
2105 gu_trace(dn.push_back(create_dummy_node(i + 1, i)));
2106 gu_trace(join_node(&prop, dn[i], i == 0 ? true : false));
2107 set_cvi(dn, 0, i, view_seq + 1);
2108 gu_trace(prop.propagate_until_cvi(false));
2109 ++view_seq;
2110 for (int j(0); j <= i; ++j)
2111 {
2112 ck_assert(evs_from_dummy(dn[j])->current_view().version() == 0);
2113 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2114 }
2115 }
2116
2117 for (int i(0); i < GCOMM_PROTOCOL_MAX_VERSION; ++i)
2118 {
2119 for (int j(i); j <= GCOMM_PROTOCOL_MAX_VERSION; ++j)
2120 {
2121 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2122 }
2123 dn[i]->close();
2124 dn[i]->set_cvi(V_REG);
2125 set_cvi(dn, i + 1, GCOMM_PROTOCOL_MAX_VERSION, view_seq);
2126 gu_trace(prop.propagate_until_cvi(true));
2127 ++view_seq;
2128 for (int j(i + 1); j <= GCOMM_PROTOCOL_MAX_VERSION; ++j)
2129 {
2130 gu_trace(send_n(dn[j], 5 + ::rand() % 4));
2131 }
2132 gu_trace(prop.propagate_until_empty());
2133 }
2134 ck_assert(evs_from_dummy(dn[GCOMM_PROTOCOL_MAX_VERSION])->current_view().version() == GCOMM_PROTOCOL_MAX_VERSION);
2135 check_trace(dn);
2136 for_each(dn.begin(), dn.end(), DeleteObject());
2137 }
2138 END_TEST
2139
2140
START_TEST(test_gal_521)2141 START_TEST(test_gal_521)
2142 {
2143 // Test the case where two nodes exhaust their user send windows
2144 // simultaneously.
2145 log_info << "Start test_gal_521";
2146
2147 std::vector<DummyNode*> dn;
2148 Protolay::sync_param_cb_t sync_param_cb;
2149
2150 dn.push_back(create_dummy_node(1, 0));
2151 dn.push_back(create_dummy_node(2, 0));
2152
2153
2154 gcomm::evs::Proto *evs1(evs_from_dummy(dn[0]));
2155 DummyTransport* t1(transport_from_dummy(dn[0]));
2156 t1->set_queueing(true);
2157
2158 gcomm::evs::Proto *evs2(evs_from_dummy(dn[1]));
2159 DummyTransport* t2(transport_from_dummy(dn[1]));
2160 t2->set_queueing(true);
2161
2162 single_join(t1, evs1);
2163 double_join(t1, evs1, t2, evs2);
2164
2165 ck_assert(t1->empty() == true);
2166 ck_assert(t2->empty() == true);
2167
2168 // Adjust send windows to allow sending only single user generated
2169 // message at the time
2170 evs1->set_param(gcomm::Conf::EvsUserSendWindow, "1", sync_param_cb);
2171 evs1->set_param(gcomm::Conf::EvsSendWindow, "1", sync_param_cb);
2172
2173 evs2->set_param(gcomm::Conf::EvsUserSendWindow, "1", sync_param_cb);
2174 evs2->set_param(gcomm::Conf::EvsSendWindow, "1", sync_param_cb);
2175
2176 // Make both sides send two messages without communicating with
2177 // each other. This will place one user message into transport
2178 // queue and one into send queue for both nodes.
2179 send_n(dn[0], 2);
2180 ck_assert(t1->empty() == false);
2181 send_n(dn[1], 2);
2182 ck_assert(t2->empty() == false);
2183
2184 Datagram *d1;
2185 Message um1;
2186 ck_assert((d1 = get_msg(t1, &um1, false)) != 0);
2187 ck_assert(um1.type() == Message::EVS_T_USER);
2188 ck_assert(t1->empty() == true);
2189 Datagram *d2;
2190 Message um2;
2191 ck_assert((d2 = get_msg(t2, &um2, false)) != 0);
2192 ck_assert(um2.type() == Message::EVS_T_USER);
2193 ck_assert(t2->empty() == true);
2194
2195 // Both of the nodes handle each other's messages. Now due to
2196 // send_window == 1 they are not allowed to send the second
2197 // message since safe_seq has not been incremented. Instead, they
2198 // must emit gap messages to make safe_seq to progress.
2199 evs1->handle_up(0, *d2, ProtoUpMeta(dn[1]->uuid()));
2200 delete d2;
2201 Message gm1;
2202 ck_assert(get_msg(t1, &gm1) != 0);
2203 ck_assert(gm1.type() == Message::EVS_T_GAP);
2204 ck_assert(t1->empty() == true);
2205
2206 evs2->handle_up(0, *d1, ProtoUpMeta(dn[0]->uuid()));
2207 delete d1;
2208 Message gm2;
2209 ck_assert(get_msg(t2, &gm2) != 0);
2210 ck_assert(gm2.type() == Message::EVS_T_GAP);
2211 ck_assert(t2->empty() == true);
2212
2213 // Handle gap messages. The safe_seq is now incremented so the
2214 // second user messages are now sent from output queue.
2215 evs1->handle_msg(gm2);
2216 ck_assert((d1 = get_msg(t1, &um1, false)) != 0);
2217 ck_assert(um1.type() == Message::EVS_T_USER);
2218 ck_assert(t1->empty() == true);
2219
2220 evs2->handle_msg(gm1);
2221 ck_assert((d2 = get_msg(t2, &um2, false)) != 0);
2222 ck_assert(um2.type() == Message::EVS_T_USER);
2223 ck_assert(t2->empty() == true);
2224
2225 // Handle user messages. Each node should now emit gap
2226 // because the output queue is empty.
2227 evs1->handle_up(0, *d2, ProtoUpMeta(dn[1]->uuid()));
2228 delete d2;
2229 ck_assert(get_msg(t1, &gm1) != 0);
2230 ck_assert(gm1.type() == Message::EVS_T_GAP);
2231 ck_assert(t1->empty() == true);
2232
2233 evs2->handle_up(0, *d1, ProtoUpMeta(dn[0]->uuid()));
2234 delete d1;
2235 ck_assert(get_msg(t2, &gm2) != 0);
2236 ck_assert(gm2.type() == Message::EVS_T_GAP);
2237 ck_assert(t2->empty() == true);
2238
2239 // Handle gap messages. No further messages should be emitted
2240 // since both user messages have been delivered, there are
2241 // no pending user messages in the output queue and no timers
2242 // have been expired.
2243 evs1->handle_msg(gm2);
2244 ck_assert((d1 = get_msg(t1, &um1, false)) == 0);
2245
2246 evs2->handle_msg(gm1);
2247 ck_assert((d2 = get_msg(t2, &um2, false)) == 0);
2248
2249
2250 std::for_each(dn.begin(), dn.end(), DeleteObject());
2251 }
2252 END_TEST
2253
2254 struct TwoNodeFixture
2255 {
2256 struct Configs
2257 {
ConfigsTwoNodeFixture::Configs2258 Configs()
2259 : conf1()
2260 , conf2()
2261 {
2262 gu::ssl_register_params(conf1);
2263 gcomm::Conf::register_params(conf1);
2264 gcomm::Conf::register_params(conf2);
2265 }
2266 gu::Config conf1; // Config for node1
2267 gu::Config conf2; // Config for node2
2268 };
TwoNodeFixtureTwoNodeFixture2269 TwoNodeFixture()
2270 : conf()
2271 , uuid1(1)
2272 , uuid2(2)
2273 , tr1(uuid1)
2274 , tr2(uuid2)
2275 , evs1(conf.conf1, uuid1, 0)
2276 , evs2(conf.conf2, uuid2, 0)
2277 , top1(conf.conf1)
2278 , top2(conf.conf2)
2279 {
2280 gcomm::connect(&tr1, &evs1);
2281 gcomm::connect(&evs1, &top1);
2282 gcomm::connect(&tr2, &evs2);
2283 gcomm::connect(&evs2, &top2);
2284 single_join(&tr1, &evs1);
2285 double_join(&tr1, &evs1, &tr2, &evs2);
2286 }
2287 Configs conf;
2288 const gcomm::UUID uuid1; // UUID of node1
2289 const gcomm::UUID uuid2; // UUID if node2
2290 DummyTransport tr1; // Transport for node1
2291 DummyTransport tr2; // Transport for node2
2292 gcomm::evs::Proto evs1; // Proto for node1
2293 gcomm::evs::Proto evs2; // Proto for node2
2294 DummyUser top1; // Top level layer for node1
2295 DummyUser top2; // Top level layer for node2
2296 };
2297
2298 // Verify that gap messages are rate limited when a node receives
2299 // several out of order messages.
START_TEST(test_gap_rate_limit)2300 START_TEST(test_gap_rate_limit)
2301 {
2302 log_info << "START test_gap_rate_limit";
2303 // Start time from 1 sec to avoid hitting gap rate limit for the first
2304 // gap message.
2305 gu::datetime::SimClock::init(gu::datetime::Sec);
2306 gu_log_max_level = GU_LOG_DEBUG;
2307 TwoNodeFixture f;
2308 gcomm::Protolay::sync_param_cb_t spcb;
2309
2310 // Increase evs1 send windows to allow generating out of order messages.
2311 f.evs1.set_param("evs.send_window", "4", spcb);
2312 f.evs1.set_param("evs.user_send_window", "4", spcb);
2313 // Print all debug logging on node2 for test troubleshooting.
2314 f.evs2.set_param("evs.debug_log_mask", "0xffff", spcb);
2315 f.evs2.set_param("evs.info_log_mask", "0xff", spcb);
2316 char data[1] = { 0 };
2317 gcomm::Datagram dg(gu::SharedBuffer(new gu::Buffer(data, data + 1)));
2318 // Generate four messages from node1. The first one is ignored,
2319 // the rest are handled by node2 for generating gap messages.
2320 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2321 gcomm::Datagram* read_dg;
2322 gcomm::evs::Message um1;
2323 read_dg = get_msg(&f.tr1, &um1);
2324 ck_assert(read_dg != 0);
2325 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2326 gcomm::evs::Message um2;
2327 read_dg = get_msg(&f.tr1, &um2);
2328 ck_assert(read_dg != 0);
2329 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2330 gcomm::evs::Message um3;
2331 read_dg = get_msg(&f.tr1, &um3);
2332 ck_assert(read_dg != 0);
2333 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2334 gcomm::evs::Message um4;
2335 read_dg = get_msg(&f.tr1, &um4);
2336 ck_assert(read_dg != 0);
2337
2338 // Make node2 handle an out of order message and verify that gap is emitted
2339 f.evs2.handle_msg(um2);
2340 gcomm::evs::Message gm1;
2341 read_dg = get_msg(&f.tr2, &gm1);
2342 ck_assert(read_dg != 0);
2343 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2344 ck_assert(gm1.range_uuid() == f.uuid1);
2345 ck_assert(gm1.range().lu() == 0);
2346 ck_assert(gm1.range().hs() == 0);
2347 // The node2 will also send an user message to complete the sequence
2348 // number. Consume it.
2349 gcomm::evs::Message comp_um1;
2350 read_dg = get_msg(&f.tr2, &comp_um1);
2351 ck_assert(read_dg != 0);
2352 ck_assert(comp_um1.type() == gcomm::evs::Message::EVS_T_USER);
2353 ck_assert(comp_um1.seq() + comp_um1.seq_range() == 1);
2354 // No further messages should be emitted
2355 read_dg = get_msg(&f.tr2, &comp_um1);
2356 ck_assert(read_dg == 0);
2357
2358 // Handle the second out of order message, gap should not be emitted.
2359 // There will be a next user message which completes the um3.
2360 f.evs2.handle_msg(um3);
2361 gcomm::evs::Message comp_um2;
2362 read_dg = get_msg(&f.tr2, &comp_um2);
2363 ck_assert(read_dg != 0);
2364 ck_assert(comp_um2.type() == gcomm::evs::Message::EVS_T_USER);
2365 ck_assert(comp_um2.seq() + comp_um2.seq_range() == 2);
2366
2367 // There should not be any more gap messages.
2368 read_dg = get_msg(&f.tr2, &gm1);
2369 ck_assert(read_dg == 0);
2370
2371 // Move the clock forwards and handle the fourth message, gap should
2372 // now emitted.
2373 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
2374 gcomm::evs::Message gm2;
2375 f.evs2.handle_msg(um4);
2376 read_dg = get_msg(&f.tr2, &gm2);
2377 ck_assert(read_dg != 0);
2378 ck_assert(gm2.type() == gcomm::evs::Message::EVS_T_GAP);
2379 ck_assert(gm2.range().lu() == 0);
2380 ck_assert(gm2.range().hs() == 0);
2381
2382 gcomm::evs::Message comp_u4;
2383 read_dg = get_msg(&f.tr2, &comp_u4);
2384 ck_assert(read_dg != 0);
2385 ck_assert(comp_u4.type() == gcomm::evs::Message::EVS_T_USER);
2386 log_info << "END test_gap_rate_limit";
2387 }
2388 END_TEST
2389
2390 // Verify that gap messages are rate limited when the liveness check finds
2391 // delayed node.
START_TEST(test_gap_rate_limit_delayed)2392 START_TEST(test_gap_rate_limit_delayed)
2393 {
2394 log_info << "START test_gap_rate_limit_delayed";
2395 // Start time from 1 sec to avoid hitting gap rate limit for the first
2396 // gap message.
2397 gu::datetime::SimClock::init(gu::datetime::Sec);
2398 gu_log_max_level = GU_LOG_DEBUG;
2399 TwoNodeFixture f;
2400 gcomm::Protolay::sync_param_cb_t spcb;
2401
2402 // Increase evs1 send windows to allow generating out of order messages.
2403 f.evs1.set_param("evs.send_window", "4", spcb);
2404 f.evs1.set_param("evs.user_send_window", "4", spcb);
2405 // Print all debug logging on node2 for test troubleshooting.
2406 f.evs2.set_param("evs.debug_log_mask", "0xffff", spcb);
2407 f.evs2.set_param("evs.info_log_mask", "0xff", spcb);
2408 // The retransmission request is done for delayed only if
2409 // auto evict is on.
2410 f.evs2.set_param("evs.auto_evict", "1", spcb);
2411 const char data[1] = { 0 };
2412 gcomm::Datagram dg(gu::SharedBuffer(new gu::Buffer(data, data + 1)));
2413 // Generate four messages from node1. The first one is ignored,
2414 // the rest are handled by node2 for generating gap messages.
2415 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2416 gcomm::Datagram* read_dg;
2417 gcomm::evs::Message um1;
2418 read_dg = get_msg(&f.tr1, &um1);
2419 ck_assert(read_dg != 0);
2420 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2421 gcomm::evs::Message um2;
2422 read_dg = get_msg(&f.tr1, &um2);
2423 ck_assert(read_dg != 0);
2424 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2425 gcomm::evs::Message um3;
2426 read_dg = get_msg(&f.tr1, &um3);
2427 ck_assert(read_dg != 0);
2428 f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE));
2429 gcomm::evs::Message um4;
2430 read_dg = get_msg(&f.tr1, &um4);
2431 ck_assert(read_dg != 0);
2432
2433 // Make node2 handle an out of order message and verify that gap is emitted
2434 f.evs2.handle_msg(um2);
2435 gcomm::evs::Message gm1;
2436 read_dg = get_msg(&f.tr2, &gm1);
2437 ck_assert(read_dg != 0);
2438 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2439 ck_assert(gm1.range_uuid() == f.uuid1);
2440 ck_assert(gm1.range().lu() == 0);
2441 ck_assert(gm1.range().hs() == 0);
2442 // The node2 will also send an user message to complete the sequence
2443 // number. Consume it.
2444 gcomm::evs::Message comp_um1;
2445 read_dg = get_msg(&f.tr2, &comp_um1);
2446 ck_assert(read_dg != 0);
2447 ck_assert(comp_um1.type() == gcomm::evs::Message::EVS_T_USER);
2448 ck_assert(comp_um1.seq() + comp_um1.seq_range() == 1);
2449 // No further messages should be emitted
2450 read_dg = get_msg(&f.tr2, &comp_um1);
2451 ck_assert(read_dg == 0);
2452
2453 // Move time forwards in 1 sec interval and make inactivity check
2454 // in between. No gap messages should be emitted.
2455 gu::datetime::SimClock::inc_time(gu::datetime::Sec);
2456 f.evs2.handle_inactivity_timer();
2457 gcomm::evs::Message gm_discard;
2458 read_dg = get_msg(&f.tr2, &gm_discard);
2459 ck_assert(read_dg == 0);
2460 // The clock is now advanced over retrans_period + delay margin. Next
2461 // call to handle_inactivity_timer() should fire the check. Gap message
2462 // is emitted.
2463 gu::datetime::SimClock::inc_time(gu::datetime::Sec);
2464 f.evs2.handle_inactivity_timer();
2465 read_dg = get_msg(&f.tr2, &gm1);
2466 ck_assert(read_dg != 0);
2467 ck_assert(gm1.type() == gcomm::evs::Message::EVS_T_GAP);
2468 // Now call handle_inactivity_timer() again, gap message should not
2469 // be emitted due to rate limit.
2470 f.evs2.handle_inactivity_timer();
2471 read_dg = get_msg(&f.tr2, &gm_discard);
2472 ck_assert(read_dg == 0);
2473 // Move clock forward 100msec, new gap should be now emitted.
2474 gu::datetime::SimClock::inc_time(100*gu::datetime::MSec);
2475 f.evs2.handle_inactivity_timer();
2476 gcomm::evs::Message gm2;
2477 read_dg = get_msg(&f.tr2, &gm2);
2478 ck_assert(read_dg != 0);
2479 ck_assert(gm2.type() == gcomm::evs::Message::EVS_T_GAP);
2480 log_info << "END test_gap_rate_limit_delayed";
2481
2482 gcomm::Datagram* tmp;
2483 while ((tmp = f.tr1.out())) delete tmp;
2484 while ((tmp = f.tr2.out())) delete tmp;
2485 }
2486 END_TEST
2487
START_TEST(test_out_queue_limit)2488 START_TEST(test_out_queue_limit)
2489 {
2490 TwoNodeFixture f;
2491
2492 std::vector<char> data(1 << 15);
2493 gcomm::Datagram dg(gu::SharedBuffer(
2494 new gu::Buffer(data.begin(), data.end())));
2495 // Default user send window is 2 and out queue limit is 1M,
2496 // so we can write 2 + 32 messages without blocking.
2497 for (size_t i(0); i < 34; ++i)
2498 {
2499 ck_assert(f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE)) == 0);
2500 }
2501 // The next write should fill the out_queue and return EAGAIN
2502 const char small_data[1] = { 0 };
2503 dg = gu::SharedBuffer(new gu::Buffer(small_data, small_data + 1));
2504 ck_assert(f.evs1.handle_down(dg, ProtoDownMeta(O_SAFE)) == EAGAIN);
2505
2506 gcomm::Datagram* tmp;
2507 while ((tmp = f.tr1.out())) delete tmp;
2508 }
2509 END_TEST
2510
evs2_suite()2511 Suite* evs2_suite()
2512 {
2513 Suite* s = suite_create("gcomm::evs");
2514 TCase* tc;
2515
2516 tc = tcase_create("test_range");
2517 tcase_add_test(tc, test_range);
2518 suite_add_tcase(s, tc);
2519
2520 tc = tcase_create("test_message");
2521 tcase_add_test(tc, test_message);
2522 suite_add_tcase(s, tc);
2523
2524 tc = tcase_create("test_input_map_insert");
2525 tcase_add_test(tc, test_input_map_insert);
2526 suite_add_tcase(s, tc);
2527
2528 tc = tcase_create("test_input_map_find");
2529 tcase_add_test(tc, test_input_map_find);
2530 suite_add_tcase(s, tc);
2531
2532 tc = tcase_create("test_input_map_safety");
2533 tcase_add_test(tc, test_input_map_safety);
2534 suite_add_tcase(s, tc);
2535
2536 tc = tcase_create("test_input_map_erase");
2537 tcase_add_test(tc, test_input_map_erase);
2538 suite_add_tcase(s, tc);
2539
2540 tc = tcase_create("test_input_map_overwrap");
2541 tcase_add_test(tc, test_input_map_overwrap);
2542 tcase_set_timeout(tc, 60);
2543 suite_add_tcase(s, tc);
2544
2545 tc = tcase_create("test_input_map_random_insert");
2546 tcase_add_test(tc, test_input_map_random_insert);
2547 suite_add_tcase(s, tc);
2548
2549 tc = tcase_create("test_input_map_gap_range_list");
2550 tcase_add_test(tc, test_input_map_gap_range_list);
2551 suite_add_tcase(s, tc);
2552
2553 tc = tcase_create("test_proto_single_join");
2554 tcase_add_test(tc, test_proto_single_join);
2555 suite_add_tcase(s, tc);
2556
2557 tc = tcase_create("test_proto_double_join");
2558 tcase_add_test(tc, test_proto_double_join);
2559 suite_add_tcase(s, tc);
2560
2561 tc = tcase_create("test_proto_join_n");
2562 tcase_add_test(tc, test_proto_join_n);
2563 suite_add_tcase(s, tc);
2564
2565 tc = tcase_create("test_proto_join_n_w_user_msg");
2566 tcase_add_test(tc, test_proto_join_n_w_user_msg);
2567 suite_add_tcase(s, tc);
2568
2569 tc = tcase_create("test_proto_join_n_lossy");
2570 tcase_add_test(tc, test_proto_join_n_lossy);
2571 suite_add_tcase(s, tc);
2572
2573 tc = tcase_create("test_proto_join_n_lossy_w_user_msg");
2574 tcase_add_test(tc, test_proto_join_n_lossy_w_user_msg);
2575 suite_add_tcase(s, tc);
2576
2577 tc = tcase_create("test_proto_leave_n");
2578 tcase_add_test(tc, test_proto_leave_n);
2579 tcase_set_timeout(tc, 20);
2580 suite_add_tcase(s, tc);
2581
2582 tc = tcase_create("test_proto_leave_n_w_user_msg");
2583 tcase_add_test(tc, test_proto_leave_n_w_user_msg);
2584 tcase_set_timeout(tc, 20);
2585 suite_add_tcase(s, tc);
2586
2587 tc = tcase_create("test_proto_leave_n_lossy");
2588 tcase_add_test(tc, test_proto_leave_n_lossy);
2589 tcase_set_timeout(tc, 25);
2590 suite_add_tcase(s, tc);
2591
2592 tc = tcase_create("test_proto_leave_n_lossy_w_user_msg");
2593 tcase_add_test(tc, test_proto_leave_n_lossy_w_user_msg);
2594 tcase_set_timeout(tc, 25);
2595 suite_add_tcase(s, tc);
2596
2597 tc = tcase_create("test_proto_split_merge");
2598 tcase_add_test(tc, test_proto_split_merge);
2599 tcase_set_timeout(tc, 20);
2600 suite_add_tcase(s, tc);
2601
2602 tc = tcase_create("test_proto_split_merge_lossy");
2603 tcase_add_test(tc, test_proto_split_merge_lossy);
2604 tcase_set_timeout(tc, 20);
2605 suite_add_tcase(s, tc);
2606
2607 tc = tcase_create("test_proto_split_merge_w_user_msg");
2608 tcase_add_test(tc, test_proto_split_merge_w_user_msg);
2609 tcase_set_timeout(tc, 60);
2610 suite_add_tcase(s, tc);
2611
2612 tc = tcase_create("test_proto_split_merge_lossy_w_user_msg");
2613 tcase_add_test(tc, test_proto_split_merge_lossy_w_user_msg);
2614 tcase_set_timeout(tc, 60);
2615 suite_add_tcase(s, tc);
2616
2617 tc = tcase_create("test_proto_stop_cont");
2618 tcase_add_test(tc, test_proto_stop_cont);
2619 tcase_set_timeout(tc, 10);
2620 suite_add_tcase(s, tc);
2621
2622 tc = tcase_create("test_proto_split_two");
2623 tcase_add_test(tc, test_proto_split_two);
2624 suite_add_tcase(s, tc);
2625
2626 tc = tcase_create("test_aggreg");
2627 tcase_add_test(tc, test_aggreg);
2628 suite_add_tcase(s, tc);
2629
2630 tc = tcase_create("test_proto_arbitrate");
2631 tcase_add_test(tc, test_proto_arbitrate);
2632 suite_add_tcase(s, tc);
2633
2634 tc = tcase_create("test_trac_538");
2635 tcase_add_test(tc, test_trac_538);
2636 tcase_set_timeout(tc, 15);
2637 suite_add_tcase(s, tc);
2638
2639 tc = tcase_create("test_trac_552");
2640 tcase_add_test(tc, test_trac_552);
2641 tcase_set_timeout(tc, 15);
2642 suite_add_tcase(s, tc);
2643
2644 tc = tcase_create("test_trac_607");
2645 tcase_add_test(tc, test_trac_607);
2646 tcase_set_timeout(tc, 15);
2647 suite_add_tcase(s, tc);
2648
2649 tc = tcase_create("test_trac_724");
2650 tcase_add_test(tc, test_trac_724);
2651 tcase_set_timeout(tc, 15);
2652 suite_add_tcase(s, tc);
2653
2654 tc = tcase_create("test_trac_760");
2655 tcase_add_test(tc, test_trac_760);
2656 tcase_set_timeout(tc, 15);
2657 suite_add_tcase(s, tc);
2658
2659 tc = tcase_create("test_gh_41");
2660 tcase_add_test(tc, test_gh_41);
2661 tcase_set_timeout(tc, 15);
2662 suite_add_tcase(s, tc);
2663
2664 tc = tcase_create("test_gh_37");
2665 tcase_add_test(tc, test_gh_37);
2666 tcase_set_timeout(tc, 15);
2667 suite_add_tcase(s, tc);
2668
2669 tc = tcase_create("test_gh_40");
2670 tcase_add_test(tc, test_gh_40);
2671 tcase_set_timeout(tc, 5);
2672 suite_add_tcase(s, tc);
2673
2674 tc = tcase_create("test_gh_100");
2675 tcase_add_test(tc, test_gh_100);
2676 suite_add_tcase(s, tc);
2677
2678 tc = tcase_create("test_evs_protocol_upgrade");
2679 tcase_add_test(tc, test_evs_protocol_upgrade);
2680 suite_add_tcase(s, tc);
2681
2682 tc = tcase_create("test_gal_521");
2683 tcase_add_test(tc, test_gal_521);
2684 suite_add_tcase(s, tc);
2685
2686 tc = tcase_create("test_gap_rate_limit");
2687 tcase_add_test(tc, test_gap_rate_limit);
2688 suite_add_tcase(s, tc);
2689
2690 tc = tcase_create("test_gap_rate_limit_delayed");
2691 tcase_add_test(tc, test_gap_rate_limit_delayed);
2692 suite_add_tcase(s, tc);
2693
2694 tc = tcase_create("test_out_queue_limit");
2695 tcase_add_test(tc, test_out_queue_limit);
2696 suite_add_tcase(s, tc);
2697
2698 return s;
2699 }
2700