1 /*
2 * Copyright (c) 2005 MontaVista Software, Inc.
3 * Copyright (c) 2006-2012 Red Hat, Inc.
4 *
5 * All rights reserved.
6 *
7 * Author: Steven Dake (sdake@redhat.com)
8 *
9 * This software licensed under BSD license, the text of which follows:
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions are met:
13 *
14 * - Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * - Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation
18 * and/or other materials provided with the distribution.
19 * - Neither the name of the MontaVista Software, Inc. nor the names of its
20 * contributors may be used to endorse or promote products derived from this
21 * software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33 * THE POSSIBILITY OF SUCH DAMAGE.
34 */
35
36 #include <config.h>
37
38 #include <assert.h>
39 #include <pthread.h>
40 #include <sys/mman.h>
41 #include <sys/types.h>
42 #include <sys/stat.h>
43 #include <sys/socket.h>
44 #include <netdb.h>
45 #include <sys/un.h>
46 #include <sys/ioctl.h>
47 #include <sys/param.h>
48 #include <netinet/in.h>
49 #include <arpa/inet.h>
50 #include <unistd.h>
51 #include <fcntl.h>
52 #include <stdlib.h>
53 #include <stdio.h>
54 #include <errno.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <limits.h>
60
61 #include <corosync/sq.h>
62 #include <corosync/list.h>
63 #include <corosync/swab.h>
64 #include <qb/qbdefs.h>
65 #include <qb/qbloop.h>
66 #define LOGSYS_UTILS_ONLY 1
67 #include <corosync/logsys.h>
68
69 #include "totemnet.h"
70 #include "totemrrp.h"
71
72 void rrp_deliver_fn (
73 void *context,
74 const void *msg,
75 unsigned int msg_len);
76
77 void rrp_iface_change_fn (
78 void *context,
79 const struct totem_ip_address *iface_addr);
80
81 struct totemrrp_instance;
82 struct passive_instance {
83 struct totemrrp_instance *rrp_instance;
84 unsigned int *faulty;
85 unsigned int *token_recv_count;
86 unsigned int *mcast_recv_count;
87 unsigned char token[15000];
88 unsigned int token_len;
89 qb_loop_timer_handle timer_expired_token;
90 qb_loop_timer_handle timer_problem_decrementer;
91 void *totemrrp_context;
92 unsigned int token_xmit_iface;
93 unsigned int msg_xmit_iface;
94 };
95
96 struct active_instance {
97 struct totemrrp_instance *rrp_instance;
98 unsigned int *faulty;
99 unsigned int *last_token_recv;
100 unsigned int *counter_problems;
101 unsigned char token[15000];
102 unsigned int token_len;
103 unsigned int last_token_seq;
104 qb_loop_timer_handle timer_expired_token;
105 qb_loop_timer_handle timer_problem_decrementer;
106 void *totemrrp_context;
107 };
108
109 struct rrp_algo {
110 const char *name;
111
112 void * (*initialize) (
113 struct totemrrp_instance *rrp_instance,
114 int interface_count);
115
116 void (*mcast_recv) (
117 struct totemrrp_instance *instance,
118 unsigned int iface_no,
119 void *context,
120 const void *msg,
121 unsigned int msg_len);
122
123 void (*mcast_noflush_send) (
124 struct totemrrp_instance *instance,
125 const void *msg,
126 unsigned int msg_len);
127
128 void (*mcast_flush_send) (
129 struct totemrrp_instance *instance,
130 const void *msg,
131 unsigned int msg_len);
132
133 void (*token_recv) (
134 struct totemrrp_instance *instance,
135 unsigned int iface_no,
136 void *context,
137 const void *msg,
138 unsigned int msg_len,
139 unsigned int token_seqid);
140
141 void (*token_send) (
142 struct totemrrp_instance *instance,
143 const void *msg,
144 unsigned int msg_len);
145
146 void (*recv_flush) (
147 struct totemrrp_instance *instance);
148
149 void (*send_flush) (
150 struct totemrrp_instance *instance);
151
152 void (*iface_check) (
153 struct totemrrp_instance *instance);
154
155 void (*processor_count_set) (
156 struct totemrrp_instance *instance,
157 unsigned int processor_count);
158
159 void (*token_target_set) (
160 struct totemrrp_instance *instance,
161 struct totem_ip_address *token_target,
162 unsigned int iface_no);
163
164 void (*ring_reenable) (
165 struct totemrrp_instance *instance,
166 unsigned int iface_no);
167
168 int (*mcast_recv_empty) (
169 struct totemrrp_instance *instance);
170
171 int (*member_add) (
172 struct totemrrp_instance *instance,
173 const struct totem_ip_address *member,
174 unsigned int iface_no);
175
176 int (*member_remove) (
177 struct totemrrp_instance *instance,
178 const struct totem_ip_address *member,
179 unsigned int iface_no);
180
181 void (*membership_changed) (
182 struct totemrrp_instance *instance,
183 enum totem_configuration_type configuration_type,
184 const struct srp_addr *member_list, size_t member_list_entries,
185 const struct srp_addr *left_list, size_t left_list_entries,
186 const struct srp_addr *joined_list, size_t joined_list_entries,
187 const struct memb_ring_id *ring_id);
188 };
189
190 #define STATUS_STR_LEN 512
191 struct totemrrp_instance {
192 qb_loop_t *poll_handle;
193
194 struct totem_interface *interfaces;
195
196 struct rrp_algo *rrp_algo;
197
198 void *context;
199
200 char *status[INTERFACE_MAX];
201
202 void (*totemrrp_deliver_fn) (
203 void *context,
204 const void *msg,
205 unsigned int msg_len);
206
207 void (*totemrrp_iface_change_fn) (
208 void *context,
209 const struct totem_ip_address *iface_addr,
210 unsigned int iface_no);
211
212 void (*totemrrp_token_seqid_get) (
213 const void *msg,
214 unsigned int *seqid,
215 unsigned int *token_is);
216
217 void (*totemrrp_target_set_completed) (
218 void *context);
219
220 unsigned int (*totemrrp_msgs_missing) (void);
221
222 /*
223 * Function and data used to log messages
224 */
225 int totemrrp_log_level_security;
226
227 int totemrrp_log_level_error;
228
229 int totemrrp_log_level_warning;
230
231 int totemrrp_log_level_notice;
232
233 int totemrrp_log_level_debug;
234
235 int totemrrp_subsys_id;
236
237 void (*totemrrp_log_printf) (
238 int level,
239 int subsys,
240 const char *function,
241 const char *file,
242 int line,
243 const char *format, ...)__attribute__((format(printf, 6, 7)));
244
245 void **net_handles;
246
247 void *rrp_algo_instance;
248
249 int interface_count;
250
251 int processor_count;
252
253 int my_nodeid;
254
255 struct totem_config *totem_config;
256
257 void *deliver_fn_context[INTERFACE_MAX];
258
259 qb_loop_timer_handle timer_active_test_ring_timeout[INTERFACE_MAX];
260
261 totemrrp_stats_t stats;
262 };
263
264 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
265 unsigned int iface_no, int is_faulty);
266
267 /*
268 * None Replication Forward Declerations
269 */
270 static void none_mcast_recv (
271 struct totemrrp_instance *instance,
272 unsigned int iface_no,
273 void *context,
274 const void *msg,
275 unsigned int msg_len);
276
277 static void none_mcast_noflush_send (
278 struct totemrrp_instance *instance,
279 const void *msg,
280 unsigned int msg_len);
281
282 static void none_mcast_flush_send (
283 struct totemrrp_instance *instance,
284 const void *msg,
285 unsigned int msg_len);
286
287 static void none_token_recv (
288 struct totemrrp_instance *instance,
289 unsigned int iface_no,
290 void *context,
291 const void *msg,
292 unsigned int msg_len,
293 unsigned int token_seqid);
294
295 static void none_token_send (
296 struct totemrrp_instance *instance,
297 const void *msg,
298 unsigned int msg_len);
299
300 static void none_recv_flush (
301 struct totemrrp_instance *instance);
302
303 static void none_send_flush (
304 struct totemrrp_instance *instance);
305
306 static void none_iface_check (
307 struct totemrrp_instance *instance);
308
309 static void none_processor_count_set (
310 struct totemrrp_instance *instance,
311 unsigned int processor_count_set);
312
313 static void none_token_target_set (
314 struct totemrrp_instance *instance,
315 struct totem_ip_address *token_target,
316 unsigned int iface_no);
317
318 static void none_ring_reenable (
319 struct totemrrp_instance *instance,
320 unsigned int iface_no);
321
322 static int none_mcast_recv_empty (
323 struct totemrrp_instance *instance);
324
325 static int none_member_add (
326 struct totemrrp_instance *instance,
327 const struct totem_ip_address *member,
328 unsigned int iface_no);
329
330 static int none_member_remove (
331 struct totemrrp_instance *instance,
332 const struct totem_ip_address *member,
333 unsigned int iface_no);
334
335 static void none_membership_changed (
336 struct totemrrp_instance *instance,
337 enum totem_configuration_type configuration_type,
338 const struct srp_addr *member_list, size_t member_list_entries,
339 const struct srp_addr *left_list, size_t left_list_entries,
340 const struct srp_addr *joined_list, size_t joined_list_entries,
341 const struct memb_ring_id *ring_id);
342
343 /*
344 * Passive Replication Forward Declerations
345 */
346 static void *passive_instance_initialize (
347 struct totemrrp_instance *rrp_instance,
348 int interface_count);
349
350 static void passive_mcast_recv (
351 struct totemrrp_instance *instance,
352 unsigned int iface_no,
353 void *context,
354 const void *msg,
355 unsigned int msg_len);
356
357 static void passive_mcast_noflush_send (
358 struct totemrrp_instance *instance,
359 const void *msg,
360 unsigned int msg_len);
361
362 static void passive_mcast_flush_send (
363 struct totemrrp_instance *instance,
364 const void *msg,
365 unsigned int msg_len);
366
367 static void passive_monitor (
368 struct totemrrp_instance *rrp_instance,
369 unsigned int iface_no,
370 int is_token_recv_count);
371
372 static void passive_token_recv (
373 struct totemrrp_instance *instance,
374 unsigned int iface_no,
375 void *context,
376 const void *msg,
377 unsigned int msg_len,
378 unsigned int token_seqid);
379
380 static void passive_token_send (
381 struct totemrrp_instance *instance,
382 const void *msg,
383 unsigned int msg_len);
384
385 static void passive_recv_flush (
386 struct totemrrp_instance *instance);
387
388 static void passive_send_flush (
389 struct totemrrp_instance *instance);
390
391 static void passive_iface_check (
392 struct totemrrp_instance *instance);
393
394 static void passive_processor_count_set (
395 struct totemrrp_instance *instance,
396 unsigned int processor_count_set);
397
398 static void passive_token_target_set (
399 struct totemrrp_instance *instance,
400 struct totem_ip_address *token_target,
401 unsigned int iface_no);
402
403 static void passive_ring_reenable (
404 struct totemrrp_instance *instance,
405 unsigned int iface_no);
406
407 static int passive_mcast_recv_empty (
408 struct totemrrp_instance *instance);
409
410 static int passive_member_add (
411 struct totemrrp_instance *instance,
412 const struct totem_ip_address *member,
413 unsigned int iface_no);
414
415 static int passive_member_remove (
416 struct totemrrp_instance *instance,
417 const struct totem_ip_address *member,
418 unsigned int iface_no);
419
420 static void passive_membership_changed (
421 struct totemrrp_instance *instance,
422 enum totem_configuration_type configuration_type,
423 const struct srp_addr *member_list, size_t member_list_entries,
424 const struct srp_addr *left_list, size_t left_list_entries,
425 const struct srp_addr *joined_list, size_t joined_list_entries,
426 const struct memb_ring_id *ring_id);
427
428 /*
429 * Active Replication Forward Definitions
430 */
431 static void *active_instance_initialize (
432 struct totemrrp_instance *rrp_instance,
433 int interface_count);
434
435 static void active_mcast_recv (
436 struct totemrrp_instance *instance,
437 unsigned int iface_no,
438 void *context,
439 const void *msg,
440 unsigned int msg_len);
441
442 static void active_mcast_noflush_send (
443 struct totemrrp_instance *instance,
444 const void *msg,
445 unsigned int msg_len);
446
447 static void active_mcast_flush_send (
448 struct totemrrp_instance *instance,
449 const void *msg,
450 unsigned int msg_len);
451
452 static void active_token_recv (
453 struct totemrrp_instance *instance,
454 unsigned int iface_no,
455 void *context,
456 const void *msg,
457 unsigned int msg_len,
458 unsigned int token_seqid);
459
460 static void active_token_send (
461 struct totemrrp_instance *instance,
462 const void *msg,
463 unsigned int msg_len);
464
465 static void active_recv_flush (
466 struct totemrrp_instance *instance);
467
468 static void active_send_flush (
469 struct totemrrp_instance *instance);
470
471 static void active_iface_check (
472 struct totemrrp_instance *instance);
473
474 static void active_processor_count_set (
475 struct totemrrp_instance *instance,
476 unsigned int processor_count_set);
477
478 static void active_token_target_set (
479 struct totemrrp_instance *instance,
480 struct totem_ip_address *token_target,
481 unsigned int iface_no);
482
483 static void active_ring_reenable (
484 struct totemrrp_instance *instance,
485 unsigned int iface_no);
486
487 static int active_mcast_recv_empty (
488 struct totemrrp_instance *instance);
489
490 static int active_member_add (
491 struct totemrrp_instance *instance,
492 const struct totem_ip_address *member,
493 unsigned int iface_no);
494
495 static int active_member_remove (
496 struct totemrrp_instance *instance,
497 const struct totem_ip_address *member,
498 unsigned int iface_no);
499
500 static void active_membership_changed (
501 struct totemrrp_instance *instance,
502 enum totem_configuration_type configuration_type,
503 const struct srp_addr *member_list, size_t member_list_entries,
504 const struct srp_addr *left_list, size_t left_list_entries,
505 const struct srp_addr *joined_list, size_t joined_list_entries,
506 const struct memb_ring_id *ring_id);
507
508 static void active_timer_expired_token_start (
509 struct active_instance *active_instance);
510
511 static void active_timer_expired_token_cancel (
512 struct active_instance *active_instance);
513
514 static void active_timer_problem_decrementer_start (
515 struct active_instance *active_instance);
516
517 static void active_timer_problem_decrementer_cancel (
518 struct active_instance *active_instance);
519
520 /*
521 * 0-5 reserved for totemsrp.c
522 */
523 #define MESSAGE_TYPE_RING_TEST_ACTIVE 6
524 #define MESSAGE_TYPE_RING_TEST_ACTIVATE 7
525
526 #define ENDIAN_LOCAL 0xff22
527
528 /*
529 * Rollover handling:
530 *
531 * ARR_SEQNO_START_TOKEN is the starting sequence number of last seen sequence
532 * for a token for active redundand ring. This should remain zero, unless testing
533 * overflow in which case 07fffff00 or 0xffffff00 are good starting values.
534 * It should be same as on defined in totemsrp.c
535 */
536
537 #define ARR_SEQNO_START_TOKEN 0x0
538
539 /*
540 * These can be used ot test different rollover points
541 * #define ARR_SEQNO_START_MSG 0xfffffe00
542 */
543
544 /*
545 * Threshold value when recv_count for passive rrp should be adjusted.
546 * Set this value to some smaller for testing of adjusting proper
547 * functionality. Also keep in mind that this value must be smaller
548 * then rrp_problem_count_threshold
549 */
550 #define PASSIVE_RECV_COUNT_THRESHOLD (INT_MAX / 2)
551
552 struct message_header {
553 char type;
554 char encapsulated;
555 unsigned short endian_detector;
556 int ring_number;
557 int nodeid_activator;
558 } __attribute__((packed));
559
560 struct deliver_fn_context {
561 struct totemrrp_instance *instance;
562 void *context;
563 int iface_no;
564 };
565
566 struct rrp_algo none_algo = {
567 .name = "none",
568 .initialize = NULL,
569 .mcast_recv = none_mcast_recv,
570 .mcast_noflush_send = none_mcast_noflush_send,
571 .mcast_flush_send = none_mcast_flush_send,
572 .token_recv = none_token_recv,
573 .token_send = none_token_send,
574 .recv_flush = none_recv_flush,
575 .send_flush = none_send_flush,
576 .iface_check = none_iface_check,
577 .processor_count_set = none_processor_count_set,
578 .token_target_set = none_token_target_set,
579 .ring_reenable = none_ring_reenable,
580 .mcast_recv_empty = none_mcast_recv_empty,
581 .member_add = none_member_add,
582 .member_remove = none_member_remove,
583 .membership_changed = none_membership_changed
584 };
585
586 struct rrp_algo passive_algo = {
587 .name = "passive",
588 .initialize = passive_instance_initialize,
589 .mcast_recv = passive_mcast_recv,
590 .mcast_noflush_send = passive_mcast_noflush_send,
591 .mcast_flush_send = passive_mcast_flush_send,
592 .token_recv = passive_token_recv,
593 .token_send = passive_token_send,
594 .recv_flush = passive_recv_flush,
595 .send_flush = passive_send_flush,
596 .iface_check = passive_iface_check,
597 .processor_count_set = passive_processor_count_set,
598 .token_target_set = passive_token_target_set,
599 .ring_reenable = passive_ring_reenable,
600 .mcast_recv_empty = passive_mcast_recv_empty,
601 .member_add = passive_member_add,
602 .member_remove = passive_member_remove,
603 .membership_changed = passive_membership_changed
604 };
605
606 struct rrp_algo active_algo = {
607 .name = "active",
608 .initialize = active_instance_initialize,
609 .mcast_recv = active_mcast_recv,
610 .mcast_noflush_send = active_mcast_noflush_send,
611 .mcast_flush_send = active_mcast_flush_send,
612 .token_recv = active_token_recv,
613 .token_send = active_token_send,
614 .recv_flush = active_recv_flush,
615 .send_flush = active_send_flush,
616 .iface_check = active_iface_check,
617 .processor_count_set = active_processor_count_set,
618 .token_target_set = active_token_target_set,
619 .ring_reenable = active_ring_reenable,
620 .mcast_recv_empty = active_mcast_recv_empty,
621 .member_add = active_member_add,
622 .member_remove = active_member_remove,
623 .membership_changed = active_membership_changed
624 };
625
626 struct rrp_algo *rrp_algos[] = {
627 &none_algo,
628 &passive_algo,
629 &active_algo
630 };
631
632 #define RRP_ALGOS_COUNT 3
633
634 #define log_printf(level, format, args...) \
635 do { \
636 rrp_instance->totemrrp_log_printf ( \
637 level, rrp_instance->totemrrp_subsys_id, \
638 __FUNCTION__, __FILE__, __LINE__, \
639 format, ##args); \
640 } while (0);
641
stats_set_interface_faulty(struct totemrrp_instance * rrp_instance,unsigned int iface_no,int is_faulty)642 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
643 unsigned int iface_no, int is_faulty)
644 {
645 rrp_instance->stats.faulty[iface_no] = (is_faulty ? 1 : 0);
646 }
647
test_active_msg_endian_convert(const struct message_header * in,struct message_header * out)648 static void test_active_msg_endian_convert(const struct message_header *in, struct message_header *out)
649 {
650 out->type = in->type;
651 out->encapsulated = in->encapsulated;
652 out->endian_detector = ENDIAN_LOCAL;
653 out->ring_number = swab32 (in->ring_number);
654 out->nodeid_activator = swab32(in->nodeid_activator);
655 }
656
timer_function_test_ring_timeout(void * context)657 static void timer_function_test_ring_timeout (void *context)
658 {
659 struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
660 struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
661 unsigned int *faulty = NULL;
662 int iface_no = deliver_fn_context->iface_no;
663 struct message_header msg = {
664 .type = MESSAGE_TYPE_RING_TEST_ACTIVE,
665 .endian_detector = ENDIAN_LOCAL,
666 };
667
668 if (strcmp(rrp_instance->totem_config->rrp_mode, "active") == 0)
669 faulty = ((struct active_instance *)(rrp_instance->rrp_algo_instance))->faulty;
670 if (strcmp(rrp_instance->totem_config->rrp_mode, "passive") == 0)
671 faulty = ((struct passive_instance *)(rrp_instance->rrp_algo_instance))->faulty;
672
673 assert (faulty != NULL);
674
675 if (faulty[iface_no] == 1) {
676 msg.ring_number = iface_no;
677 msg.nodeid_activator = rrp_instance->my_nodeid;
678 totemnet_token_send (
679 rrp_instance->net_handles[iface_no],
680 &msg, sizeof (struct message_header));
681 qb_loop_timer_add (rrp_instance->poll_handle,
682 QB_LOOP_MED,
683 rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
684 (void *)deliver_fn_context,
685 timer_function_test_ring_timeout,
686 &rrp_instance->timer_active_test_ring_timeout[iface_no]);
687 }
688 }
689
690 /*
691 * None Replication Implementation
692 */
693
none_mcast_recv(struct totemrrp_instance * rrp_instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len)694 static void none_mcast_recv (
695 struct totemrrp_instance *rrp_instance,
696 unsigned int iface_no,
697 void *context,
698 const void *msg,
699 unsigned int msg_len)
700 {
701 rrp_instance->totemrrp_deliver_fn (
702 context,
703 msg,
704 msg_len);
705 }
706
none_mcast_flush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)707 static void none_mcast_flush_send (
708 struct totemrrp_instance *instance,
709 const void *msg,
710 unsigned int msg_len)
711 {
712 totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
713 }
714
none_mcast_noflush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)715 static void none_mcast_noflush_send (
716 struct totemrrp_instance *instance,
717 const void *msg,
718 unsigned int msg_len)
719 {
720 totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
721 }
722
none_token_recv(struct totemrrp_instance * rrp_instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len,unsigned int token_seq)723 static void none_token_recv (
724 struct totemrrp_instance *rrp_instance,
725 unsigned int iface_no,
726 void *context,
727 const void *msg,
728 unsigned int msg_len,
729 unsigned int token_seq)
730 {
731 rrp_instance->totemrrp_deliver_fn (
732 context,
733 msg,
734 msg_len);
735 }
736
none_token_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)737 static void none_token_send (
738 struct totemrrp_instance *instance,
739 const void *msg,
740 unsigned int msg_len)
741 {
742 totemnet_token_send (
743 instance->net_handles[0],
744 msg, msg_len);
745 }
746
none_recv_flush(struct totemrrp_instance * instance)747 static void none_recv_flush (struct totemrrp_instance *instance)
748 {
749 totemnet_recv_flush (instance->net_handles[0]);
750 }
751
none_send_flush(struct totemrrp_instance * instance)752 static void none_send_flush (struct totemrrp_instance *instance)
753 {
754 totemnet_send_flush (instance->net_handles[0]);
755 }
756
none_iface_check(struct totemrrp_instance * instance)757 static void none_iface_check (struct totemrrp_instance *instance)
758 {
759 totemnet_iface_check (instance->net_handles[0]);
760 }
761
none_processor_count_set(struct totemrrp_instance * instance,unsigned int processor_count)762 static void none_processor_count_set (
763 struct totemrrp_instance *instance,
764 unsigned int processor_count)
765 {
766 totemnet_processor_count_set (instance->net_handles[0],
767 processor_count);
768 }
769
none_token_target_set(struct totemrrp_instance * instance,struct totem_ip_address * token_target,unsigned int iface_no)770 static void none_token_target_set (
771 struct totemrrp_instance *instance,
772 struct totem_ip_address *token_target,
773 unsigned int iface_no)
774 {
775 totemnet_token_target_set (instance->net_handles[0], token_target);
776 }
777
none_ring_reenable(struct totemrrp_instance * instance,unsigned int iface_no)778 static void none_ring_reenable (
779 struct totemrrp_instance *instance,
780 unsigned int iface_no)
781 {
782 /*
783 * No operation
784 */
785 }
786
none_mcast_recv_empty(struct totemrrp_instance * instance)787 static int none_mcast_recv_empty (
788 struct totemrrp_instance *instance)
789 {
790 int res;
791
792 res = totemnet_recv_mcast_empty (instance->net_handles[0]);
793
794 return (res);
795 }
796
none_member_add(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)797 static int none_member_add (
798 struct totemrrp_instance *instance,
799 const struct totem_ip_address *member,
800 unsigned int iface_no)
801 {
802 int res;
803 res = totemnet_member_add (instance->net_handles[0], member);
804 return (res);
805 }
806
none_member_remove(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)807 static int none_member_remove (
808 struct totemrrp_instance *instance,
809 const struct totem_ip_address *member,
810 unsigned int iface_no)
811 {
812 int res;
813 res = totemnet_member_remove (instance->net_handles[0], member);
814 return (res);
815 }
816
none_membership_changed(struct totemrrp_instance * rrp_instance,enum totem_configuration_type configuration_type,const struct srp_addr * member_list,size_t member_list_entries,const struct srp_addr * left_list,size_t left_list_entries,const struct srp_addr * joined_list,size_t joined_list_entries,const struct memb_ring_id * ring_id)817 static void none_membership_changed (
818 struct totemrrp_instance *rrp_instance,
819 enum totem_configuration_type configuration_type,
820 const struct srp_addr *member_list, size_t member_list_entries,
821 const struct srp_addr *left_list, size_t left_list_entries,
822 const struct srp_addr *joined_list, size_t joined_list_entries,
823 const struct memb_ring_id *ring_id)
824 {
825 int i;
826
827 for (i = 0; i < left_list_entries; i++) {
828 if (left_list->no_addrs < 1 ||
829 (left_list[i].addr[0].family != AF_INET && left_list[i].addr[0].family != AF_INET6)) {
830 log_printf(rrp_instance->totemrrp_log_level_error,
831 "Membership left list contains incorrect address. "
832 "This is sign of misconfiguration between nodes!");
833 } else {
834 totemnet_member_set_active(rrp_instance->net_handles[0],
835 &left_list[i].addr[0], 0);
836 }
837 }
838
839 for (i = 0; i < joined_list_entries; i++) {
840 if (joined_list->no_addrs < 1 ||
841 (joined_list[i].addr[0].family != AF_INET && joined_list[i].addr[0].family != AF_INET6)) {
842 log_printf(rrp_instance->totemrrp_log_level_error,
843 "Membership join list contains incorrect address. "
844 "This is sign of misconfiguration between nodes!");
845 } else {
846 totemnet_member_set_active(rrp_instance->net_handles[0],
847 &joined_list[i].addr[0], 1);
848 }
849 }
850 }
851
852 /*
853 * Passive Replication Implementation
854 */
passive_instance_initialize(struct totemrrp_instance * rrp_instance,int interface_count)855 void *passive_instance_initialize (
856 struct totemrrp_instance *rrp_instance,
857 int interface_count)
858 {
859 struct passive_instance *instance;
860 int i;
861
862 instance = malloc (sizeof (struct passive_instance));
863 if (instance == 0) {
864 goto error_exit;
865 }
866 memset (instance, 0, sizeof (struct passive_instance));
867
868 instance->faulty = malloc (sizeof (int) * interface_count);
869 if (instance->faulty == 0) {
870 free (instance);
871 instance = 0;
872 goto error_exit;
873 }
874 memset (instance->faulty, 0, sizeof (int) * interface_count);
875
876 for (i = 0; i < interface_count; i++) {
877 stats_set_interface_faulty (rrp_instance, i, 0);
878 }
879
880 instance->token_recv_count = malloc (sizeof (int) * interface_count);
881 if (instance->token_recv_count == 0) {
882 free (instance->faulty);
883 free (instance);
884 instance = 0;
885 goto error_exit;
886 }
887 memset (instance->token_recv_count, 0, sizeof (int) * interface_count);
888
889 instance->mcast_recv_count = malloc (sizeof (int) * interface_count);
890 if (instance->mcast_recv_count == 0) {
891 free (instance->token_recv_count);
892 free (instance->faulty);
893 free (instance);
894 instance = 0;
895 goto error_exit;
896 }
897 memset (instance->mcast_recv_count, 0, sizeof (int) * interface_count);
898
899 error_exit:
900 return ((void *)instance);
901 }
902
timer_function_passive_token_expired(void * context)903 static void timer_function_passive_token_expired (void *context)
904 {
905 struct passive_instance *passive_instance = (struct passive_instance *)context;
906 struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
907
908 rrp_instance->totemrrp_deliver_fn (
909 passive_instance->totemrrp_context,
910 passive_instance->token,
911 passive_instance->token_len);
912 }
913
914 /* TODO
915 static void timer_function_passive_problem_decrementer (void *context)
916 {
917 // struct passive_instance *passive_instance = (struct passive_instance *)context;
918 // struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
919
920 }
921 */
922
923
passive_timer_expired_token_start(struct passive_instance * passive_instance)924 static void passive_timer_expired_token_start (
925 struct passive_instance *passive_instance)
926 {
927 qb_loop_timer_add (
928 passive_instance->rrp_instance->poll_handle,
929 QB_LOOP_MED,
930 passive_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
931 (void *)passive_instance,
932 timer_function_passive_token_expired,
933 &passive_instance->timer_expired_token);
934 }
935
passive_timer_expired_token_cancel(struct passive_instance * passive_instance)936 static void passive_timer_expired_token_cancel (
937 struct passive_instance *passive_instance)
938 {
939 qb_loop_timer_del (
940 passive_instance->rrp_instance->poll_handle,
941 passive_instance->timer_expired_token);
942 }
943
944 /*
945 static void passive_timer_problem_decrementer_start (
946 struct passive_instance *passive_instance)
947 {
948 qb_loop_timer_add (
949 QB_LOOP_MED,
950 passive_instance->rrp_instance->poll_handle,
951 passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
952 (void *)passive_instance,
953 timer_function_passive_problem_decrementer,
954 &passive_instance->timer_problem_decrementer);
955 }
956
957 static void passive_timer_problem_decrementer_cancel (
958 struct passive_instance *passive_instance)
959 {
960 qb_loop_timer_del (
961 passive_instance->rrp_instance->poll_handle,
962 passive_instance->timer_problem_decrementer);
963 }
964 */
965
966 /*
967 * Monitor function implementation from rrp paper.
968 * rrp_instance is passive rrp instance, iface_no is interface with received messgae/token and
969 * is_token_recv_count is boolean variable which donates if message is token (>1) or regular
970 * message (= 0)
971 */
passive_monitor(struct totemrrp_instance * rrp_instance,unsigned int iface_no,int is_token_recv_count)972 static void passive_monitor (
973 struct totemrrp_instance *rrp_instance,
974 unsigned int iface_no,
975 int is_token_recv_count)
976 {
977 struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
978 unsigned int *recv_count;
979 unsigned int max;
980 unsigned int i;
981 unsigned int min_all, min_active;
982 unsigned int threshold;
983
984 /*
985 * Monitor for failures
986 */
987 if (is_token_recv_count) {
988 recv_count = passive_instance->token_recv_count;
989 threshold = rrp_instance->totem_config->rrp_problem_count_threshold;
990 } else {
991 recv_count = passive_instance->mcast_recv_count;
992 threshold = rrp_instance->totem_config->rrp_problem_count_mcast_threshold;
993 }
994
995 recv_count[iface_no] += 1;
996
997 max = 0;
998 for (i = 0; i < rrp_instance->interface_count; i++) {
999 if (max < recv_count[i]) {
1000 max = recv_count[i];
1001 }
1002 }
1003
1004 /*
1005 * Max is larger than threshold -> start adjusting process
1006 */
1007 if (max > PASSIVE_RECV_COUNT_THRESHOLD) {
1008 min_all = min_active = recv_count[iface_no];
1009
1010 for (i = 0; i < rrp_instance->interface_count; i++) {
1011 if (recv_count[i] < min_all) {
1012 min_all = recv_count[i];
1013 }
1014
1015 if (passive_instance->faulty[i] == 0 &&
1016 recv_count[i] < min_active) {
1017 min_active = recv_count[i];
1018 }
1019 }
1020
1021 if (min_all > 0) {
1022 /*
1023 * There is one or more faulty device with recv_count > 0
1024 */
1025 for (i = 0; i < rrp_instance->interface_count; i++) {
1026 recv_count[i] -= min_all;
1027 }
1028 } else {
1029 /*
1030 * No faulty device with recv_count > 0, adjust only active
1031 * devices
1032 */
1033 for (i = 0; i < rrp_instance->interface_count; i++) {
1034 if (passive_instance->faulty[i] == 0) {
1035 recv_count[i] -= min_active;
1036 }
1037 }
1038 }
1039
1040 /*
1041 * Find again max
1042 */
1043 max = 0;
1044
1045 for (i = 0; i < rrp_instance->interface_count; i++) {
1046 if (max < recv_count[i]) {
1047 max = recv_count[i];
1048 }
1049 }
1050 }
1051
1052 for (i = 0; i < rrp_instance->interface_count; i++) {
1053 if ((passive_instance->faulty[i] == 0) &&
1054 (max - recv_count[i] > threshold)) {
1055 passive_instance->faulty[i] = 1;
1056
1057 qb_loop_timer_add (rrp_instance->poll_handle,
1058 QB_LOOP_MED,
1059 rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
1060 rrp_instance->deliver_fn_context[i],
1061 timer_function_test_ring_timeout,
1062 &rrp_instance->timer_active_test_ring_timeout[i]);
1063
1064 stats_set_interface_faulty (rrp_instance, i, passive_instance->faulty[i]);
1065
1066 snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1067 "Marking ringid %u interface %s FAULTY",
1068 i,
1069 totemnet_iface_print (rrp_instance->net_handles[i]));
1070 log_printf (
1071 rrp_instance->totemrrp_log_level_error,
1072 "%s",
1073 rrp_instance->status[i]);
1074 }
1075 }
1076 }
1077
passive_mcast_recv(struct totemrrp_instance * rrp_instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len)1078 static void passive_mcast_recv (
1079 struct totemrrp_instance *rrp_instance,
1080 unsigned int iface_no,
1081 void *context,
1082 const void *msg,
1083 unsigned int msg_len)
1084 {
1085 struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1086
1087 rrp_instance->totemrrp_deliver_fn (
1088 context,
1089 msg,
1090 msg_len);
1091
1092 if (rrp_instance->totemrrp_msgs_missing() == 0 &&
1093 passive_instance->timer_expired_token) {
1094 /*
1095 * Delivers the last token
1096 */
1097 rrp_instance->totemrrp_deliver_fn (
1098 passive_instance->totemrrp_context,
1099 passive_instance->token,
1100 passive_instance->token_len);
1101 passive_timer_expired_token_cancel (passive_instance);
1102 }
1103
1104 passive_monitor (rrp_instance, iface_no, 0);
1105 }
1106
passive_mcast_flush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1107 static void passive_mcast_flush_send (
1108 struct totemrrp_instance *instance,
1109 const void *msg,
1110 unsigned int msg_len)
1111 {
1112 struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1113 int i = 0;
1114
1115 do {
1116 passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1117 i++;
1118 } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1119
1120 if (i > instance->interface_count) {
1121 /*
1122 * All interfaces are faulty. It's still needed to send mcast
1123 * message to local host so use first interface.
1124 */
1125 passive_instance->msg_xmit_iface = 0;
1126 }
1127
1128 totemnet_mcast_flush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1129 }
1130
passive_mcast_noflush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1131 static void passive_mcast_noflush_send (
1132 struct totemrrp_instance *instance,
1133 const void *msg,
1134 unsigned int msg_len)
1135 {
1136 struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1137 int i = 0;
1138
1139 do {
1140 passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1141 i++;
1142 } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1143
1144
1145 if (i > instance->interface_count) {
1146 /*
1147 * All interfaces are faulty. It's still needed to send mcast
1148 * message to local host so use first interface.
1149 */
1150 passive_instance->msg_xmit_iface = 0;
1151 }
1152
1153 totemnet_mcast_noflush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1154 }
1155
passive_token_recv(struct totemrrp_instance * rrp_instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len,unsigned int token_seq)1156 static void passive_token_recv (
1157 struct totemrrp_instance *rrp_instance,
1158 unsigned int iface_no,
1159 void *context,
1160 const void *msg,
1161 unsigned int msg_len,
1162 unsigned int token_seq)
1163 {
1164 struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1165
1166 passive_instance->totemrrp_context = context; // this should be in totemrrp_instance ? TODO
1167
1168 if (rrp_instance->totemrrp_msgs_missing() == 0) {
1169 rrp_instance->totemrrp_deliver_fn (
1170 context,
1171 msg,
1172 msg_len);
1173 } else {
1174 memcpy (passive_instance->token, msg, msg_len);
1175 passive_timer_expired_token_start (passive_instance);
1176
1177 }
1178
1179 passive_monitor (rrp_instance, iface_no, 1);
1180 }
1181
passive_token_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1182 static void passive_token_send (
1183 struct totemrrp_instance *instance,
1184 const void *msg,
1185 unsigned int msg_len)
1186 {
1187 struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1188 int i = 0;
1189
1190 do {
1191 passive_instance->token_xmit_iface = (passive_instance->token_xmit_iface + 1) % instance->interface_count;
1192 i++;
1193 } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->token_xmit_iface] == 1));
1194
1195 if (i > instance->interface_count) {
1196 /*
1197 * All interfaces are faulty. It's still needed to send token
1198 * message to (potentionally) local host so use first interface.
1199 */
1200 passive_instance->msg_xmit_iface = 0;
1201 }
1202
1203 totemnet_token_send (
1204 instance->net_handles[passive_instance->token_xmit_iface],
1205 msg, msg_len);
1206 }
1207
passive_recv_flush(struct totemrrp_instance * instance)1208 static void passive_recv_flush (struct totemrrp_instance *instance)
1209 {
1210 struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1211 unsigned int i;
1212
1213 for (i = 0; i < instance->interface_count; i++) {
1214 if (rrp_algo_instance->faulty[i] == 0) {
1215
1216 totemnet_recv_flush (instance->net_handles[i]);
1217 }
1218 }
1219 }
1220
passive_send_flush(struct totemrrp_instance * instance)1221 static void passive_send_flush (struct totemrrp_instance *instance)
1222 {
1223 struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1224 unsigned int i;
1225
1226 for (i = 0; i < instance->interface_count; i++) {
1227 if (rrp_algo_instance->faulty[i] == 0) {
1228
1229 totemnet_send_flush (instance->net_handles[i]);
1230 }
1231 }
1232 }
1233
passive_iface_check(struct totemrrp_instance * instance)1234 static void passive_iface_check (struct totemrrp_instance *instance)
1235 {
1236 struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1237 unsigned int i;
1238
1239 for (i = 0; i < instance->interface_count; i++) {
1240 if (rrp_algo_instance->faulty[i] == 0) {
1241
1242 totemnet_iface_check (instance->net_handles[i]);
1243 }
1244 }
1245 }
1246
passive_processor_count_set(struct totemrrp_instance * instance,unsigned int processor_count)1247 static void passive_processor_count_set (
1248 struct totemrrp_instance *instance,
1249 unsigned int processor_count)
1250 {
1251 struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1252 unsigned int i;
1253
1254 for (i = 0; i < instance->interface_count; i++) {
1255 if (rrp_algo_instance->faulty[i] == 0) {
1256
1257 totemnet_processor_count_set (instance->net_handles[i],
1258 processor_count);
1259 }
1260 }
1261 }
1262
passive_token_target_set(struct totemrrp_instance * instance,struct totem_ip_address * token_target,unsigned int iface_no)1263 static void passive_token_target_set (
1264 struct totemrrp_instance *instance,
1265 struct totem_ip_address *token_target,
1266 unsigned int iface_no)
1267 {
1268 totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1269 }
1270
passive_mcast_recv_empty(struct totemrrp_instance * instance)1271 static int passive_mcast_recv_empty (
1272 struct totemrrp_instance *instance)
1273 {
1274 int res;
1275 int msgs_emptied = 0;
1276 int i;
1277
1278 for (i = 0; i < instance->interface_count; i++) {
1279 res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1280 if (res == -1) {
1281 return (-1);
1282 }
1283 if (res == 1) {
1284 msgs_emptied = 1;
1285 }
1286 }
1287
1288 return (msgs_emptied);
1289 }
1290
passive_member_add(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)1291 static int passive_member_add (
1292 struct totemrrp_instance *instance,
1293 const struct totem_ip_address *member,
1294 unsigned int iface_no)
1295 {
1296 int res;
1297 res = totemnet_member_add (instance->net_handles[iface_no], member);
1298 return (res);
1299 }
1300
passive_member_remove(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)1301 static int passive_member_remove (
1302 struct totemrrp_instance *instance,
1303 const struct totem_ip_address *member,
1304 unsigned int iface_no)
1305 {
1306 int res;
1307 res = totemnet_member_remove (instance->net_handles[iface_no], member);
1308 return (res);
1309 }
1310
passive_membership_changed(struct totemrrp_instance * rrp_instance,enum totem_configuration_type configuration_type,const struct srp_addr * member_list,size_t member_list_entries,const struct srp_addr * left_list,size_t left_list_entries,const struct srp_addr * joined_list,size_t joined_list_entries,const struct memb_ring_id * ring_id)1311 static void passive_membership_changed (
1312 struct totemrrp_instance *rrp_instance,
1313 enum totem_configuration_type configuration_type,
1314 const struct srp_addr *member_list, size_t member_list_entries,
1315 const struct srp_addr *left_list, size_t left_list_entries,
1316 const struct srp_addr *joined_list, size_t joined_list_entries,
1317 const struct memb_ring_id *ring_id)
1318 {
1319 int i;
1320 int interface;
1321
1322 for (interface = 0; interface < rrp_instance->interface_count; interface++) {
1323 for (i = 0; i < left_list_entries; i++) {
1324 if (left_list->no_addrs < interface + 1 ||
1325 (left_list[i].addr[interface].family != AF_INET &&
1326 left_list[i].addr[interface].family != AF_INET6)) {
1327 log_printf(rrp_instance->totemrrp_log_level_error,
1328 "Membership left list contains incorrect address. "
1329 "This is sign of misconfiguration between nodes!");
1330 } else {
1331 totemnet_member_set_active(rrp_instance->net_handles[interface],
1332 &left_list[i].addr[interface], 0);
1333 }
1334 }
1335
1336 for (i = 0; i < joined_list_entries; i++) {
1337 if (joined_list->no_addrs < interface + 1 ||
1338 (joined_list[i].addr[interface].family != AF_INET &&
1339 joined_list[i].addr[interface].family != AF_INET6)) {
1340 log_printf(rrp_instance->totemrrp_log_level_error,
1341 "Membership join list contains incorrect address. "
1342 "This is sign of misconfiguration between nodes!");
1343 } else {
1344 totemnet_member_set_active(rrp_instance->net_handles[interface],
1345 &joined_list[i].addr[interface], 1);
1346 }
1347 }
1348 }
1349 }
1350
passive_ring_reenable(struct totemrrp_instance * instance,unsigned int iface_no)1351 static void passive_ring_reenable (
1352 struct totemrrp_instance *instance,
1353 unsigned int iface_no)
1354 {
1355 struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1356 int i;
1357
1358 memset (rrp_algo_instance->mcast_recv_count, 0, sizeof (unsigned int) *
1359 instance->interface_count);
1360 memset (rrp_algo_instance->token_recv_count, 0, sizeof (unsigned int) *
1361 instance->interface_count);
1362
1363 if (iface_no == instance->interface_count) {
1364 memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1365 instance->interface_count);
1366 for (i = 0; i < instance->interface_count; i++) {
1367 stats_set_interface_faulty (instance, i, 0);
1368 }
1369 } else {
1370 rrp_algo_instance->faulty[iface_no] = 0;
1371 stats_set_interface_faulty (instance, iface_no, 0);
1372 }
1373 }
1374
1375 /*
1376 * Active Replication Implementation
1377 */
active_instance_initialize(struct totemrrp_instance * rrp_instance,int interface_count)1378 void *active_instance_initialize (
1379 struct totemrrp_instance *rrp_instance,
1380 int interface_count)
1381 {
1382 struct active_instance *instance;
1383 int i;
1384
1385 instance = malloc (sizeof (struct active_instance));
1386 if (instance == 0) {
1387 goto error_exit;
1388 }
1389 memset (instance, 0, sizeof (struct active_instance));
1390
1391 instance->faulty = malloc (sizeof (int) * interface_count);
1392 if (instance->faulty == 0) {
1393 free (instance);
1394 instance = 0;
1395 goto error_exit;
1396 }
1397 memset (instance->faulty, 0, sizeof (unsigned int) * interface_count);
1398
1399 for (i = 0; i < interface_count; i++) {
1400 stats_set_interface_faulty (rrp_instance, i, 0);
1401 }
1402
1403 instance->last_token_recv = malloc (sizeof (int) * interface_count);
1404 if (instance->last_token_recv == 0) {
1405 free (instance->faulty);
1406 free (instance);
1407 instance = 0;
1408 goto error_exit;
1409 }
1410 memset (instance->last_token_recv, 0, sizeof (unsigned int) * interface_count);
1411
1412 instance->counter_problems = malloc (sizeof (int) * interface_count);
1413 if (instance->counter_problems == 0) {
1414 free (instance->last_token_recv);
1415 free (instance->faulty);
1416 free (instance);
1417 instance = 0;
1418 goto error_exit;
1419 }
1420 memset (instance->counter_problems, 0, sizeof (unsigned int) * interface_count);
1421
1422 instance->timer_expired_token = 0;
1423
1424 instance->timer_problem_decrementer = 0;
1425
1426 instance->rrp_instance = rrp_instance;
1427
1428 instance->last_token_seq = ARR_SEQNO_START_TOKEN - 1;
1429
1430 error_exit:
1431 return ((void *)instance);
1432 }
timer_function_active_problem_decrementer(void * context)1433 static void timer_function_active_problem_decrementer (void *context)
1434 {
1435 struct active_instance *active_instance = (struct active_instance *)context;
1436 struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1437 unsigned int problem_found = 0;
1438 unsigned int i;
1439
1440 for (i = 0; i < rrp_instance->interface_count; i++) {
1441 if (active_instance->counter_problems[i] > 0) {
1442 problem_found = 1;
1443 active_instance->counter_problems[i] -= 1;
1444 if (active_instance->counter_problems[i] == 0) {
1445 snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1446 "ring %d active with no faults", i);
1447 } else {
1448 snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1449 "Decrementing problem counter for iface %s to [%d of %d]",
1450 totemnet_iface_print (rrp_instance->net_handles[i]),
1451 active_instance->counter_problems[i],
1452 rrp_instance->totem_config->rrp_problem_count_threshold);
1453 }
1454 log_printf (
1455 rrp_instance->totemrrp_log_level_warning,
1456 "%s",
1457 rrp_instance->status[i]);
1458 }
1459 }
1460 if (problem_found) {
1461 active_timer_problem_decrementer_start (active_instance);
1462 } else {
1463 active_instance->timer_problem_decrementer = 0;
1464 }
1465 }
1466
timer_function_active_token_expired(void * context)1467 static void timer_function_active_token_expired (void *context)
1468 {
1469 struct active_instance *active_instance = (struct active_instance *)context;
1470 struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1471 unsigned int i;
1472
1473 for (i = 0; i < rrp_instance->interface_count; i++) {
1474 if (active_instance->last_token_recv[i] == 0) {
1475 active_instance->counter_problems[i] += 1;
1476
1477 if (active_instance->timer_problem_decrementer == 0) {
1478 active_timer_problem_decrementer_start (active_instance);
1479 }
1480 snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1481 "Incrementing problem counter for seqid %d iface %s to [%d of %d]",
1482 active_instance->last_token_seq,
1483 totemnet_iface_print (rrp_instance->net_handles[i]),
1484 active_instance->counter_problems[i],
1485 rrp_instance->totem_config->rrp_problem_count_threshold);
1486 log_printf (
1487 rrp_instance->totemrrp_log_level_warning,
1488 "%s",
1489 rrp_instance->status[i]);
1490 }
1491 }
1492 for (i = 0; i < rrp_instance->interface_count; i++) {
1493 if (active_instance->counter_problems[i] >= rrp_instance->totem_config->rrp_problem_count_threshold &&
1494 active_instance->faulty[i] == 0) {
1495 active_instance->faulty[i] = 1;
1496
1497 qb_loop_timer_add (rrp_instance->poll_handle,
1498 QB_LOOP_MED,
1499 rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
1500 rrp_instance->deliver_fn_context[i],
1501 timer_function_test_ring_timeout,
1502 &rrp_instance->timer_active_test_ring_timeout[i]);
1503
1504 stats_set_interface_faulty (rrp_instance, i, active_instance->faulty[i]);
1505
1506 snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1507 "Marking seqid %d ringid %u interface %s FAULTY",
1508 active_instance->last_token_seq,
1509 i,
1510 totemnet_iface_print (rrp_instance->net_handles[i]));
1511 log_printf (
1512 rrp_instance->totemrrp_log_level_error,
1513 "%s",
1514 rrp_instance->status[i]);
1515 active_timer_problem_decrementer_cancel (active_instance);
1516 }
1517 }
1518
1519 rrp_instance->totemrrp_deliver_fn (
1520 active_instance->totemrrp_context,
1521 active_instance->token,
1522 active_instance->token_len);
1523 }
1524
active_timer_expired_token_start(struct active_instance * active_instance)1525 static void active_timer_expired_token_start (
1526 struct active_instance *active_instance)
1527 {
1528 qb_loop_timer_add (
1529 active_instance->rrp_instance->poll_handle,
1530 QB_LOOP_MED,
1531 active_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
1532 (void *)active_instance,
1533 timer_function_active_token_expired,
1534 &active_instance->timer_expired_token);
1535 }
1536
active_timer_expired_token_cancel(struct active_instance * active_instance)1537 static void active_timer_expired_token_cancel (
1538 struct active_instance *active_instance)
1539 {
1540 qb_loop_timer_del (
1541 active_instance->rrp_instance->poll_handle,
1542 active_instance->timer_expired_token);
1543 }
1544
active_timer_problem_decrementer_start(struct active_instance * active_instance)1545 static void active_timer_problem_decrementer_start (
1546 struct active_instance *active_instance)
1547 {
1548 qb_loop_timer_add (
1549 active_instance->rrp_instance->poll_handle,
1550 QB_LOOP_MED,
1551 active_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
1552 (void *)active_instance,
1553 timer_function_active_problem_decrementer,
1554 &active_instance->timer_problem_decrementer);
1555 }
1556
active_timer_problem_decrementer_cancel(struct active_instance * active_instance)1557 static void active_timer_problem_decrementer_cancel (
1558 struct active_instance *active_instance)
1559 {
1560 qb_loop_timer_del (
1561 active_instance->rrp_instance->poll_handle,
1562 active_instance->timer_problem_decrementer);
1563 active_instance->timer_problem_decrementer = 0;
1564 }
1565
1566
1567 /*
1568 * active replication
1569 */
active_mcast_recv(struct totemrrp_instance * instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len)1570 static void active_mcast_recv (
1571 struct totemrrp_instance *instance,
1572 unsigned int iface_no,
1573 void *context,
1574 const void *msg,
1575 unsigned int msg_len)
1576 {
1577 instance->totemrrp_deliver_fn (
1578 context,
1579 msg,
1580 msg_len);
1581 }
1582
active_mcast_flush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1583 static void active_mcast_flush_send (
1584 struct totemrrp_instance *instance,
1585 const void *msg,
1586 unsigned int msg_len)
1587 {
1588 int i;
1589 int msg_sent;
1590 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1591
1592 msg_sent = 0;
1593
1594 for (i = 0; i < instance->interface_count; i++) {
1595 if (rrp_algo_instance->faulty[i] == 0) {
1596 msg_sent = 1;
1597 totemnet_mcast_flush_send (instance->net_handles[i], msg, msg_len);
1598 }
1599 }
1600
1601 if (!msg_sent) {
1602 /*
1603 * All interfaces are faulty. It's still needed to send mcast
1604 * message to local host so use first interface.
1605 */
1606 totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
1607 }
1608 }
1609
active_mcast_noflush_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1610 static void active_mcast_noflush_send (
1611 struct totemrrp_instance *instance,
1612 const void *msg,
1613 unsigned int msg_len)
1614 {
1615 int i;
1616 int msg_sent;
1617 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1618
1619 msg_sent = 0;
1620
1621 for (i = 0; i < instance->interface_count; i++) {
1622 if (rrp_algo_instance->faulty[i] == 0) {
1623 msg_sent = 1;
1624 totemnet_mcast_noflush_send (instance->net_handles[i], msg, msg_len);
1625 }
1626 }
1627
1628 if (!msg_sent) {
1629 /*
1630 * All interfaces are faulty. It's still needed to send mcast
1631 * message to local host so use first interface.
1632 */
1633 totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
1634 }
1635 }
1636
active_token_recv(struct totemrrp_instance * rrp_instance,unsigned int iface_no,void * context,const void * msg,unsigned int msg_len,unsigned int token_seq)1637 static void active_token_recv (
1638 struct totemrrp_instance *rrp_instance,
1639 unsigned int iface_no,
1640 void *context,
1641 const void *msg,
1642 unsigned int msg_len,
1643 unsigned int token_seq)
1644 {
1645 int i;
1646 struct active_instance *active_instance = (struct active_instance *)rrp_instance->rrp_algo_instance;
1647
1648 active_instance->totemrrp_context = context;
1649 if (sq_lt_compare (active_instance->last_token_seq, token_seq)) {
1650 memcpy (active_instance->token, msg, msg_len);
1651 active_instance->token_len = msg_len;
1652 for (i = 0; i < rrp_instance->interface_count; i++) {
1653 active_instance->last_token_recv[i] = 0;
1654 }
1655
1656 active_instance->last_token_recv[iface_no] = 1;
1657 active_timer_expired_token_start (active_instance);
1658 }
1659
1660 /*
1661 * This doesn't follow spec because the spec assumes we will know
1662 * when token resets occur.
1663 */
1664 active_instance->last_token_seq = token_seq;
1665
1666 if (token_seq == active_instance->last_token_seq) {
1667 active_instance->last_token_recv[iface_no] = 1;
1668 for (i = 0; i < rrp_instance->interface_count; i++) {
1669 if ((active_instance->last_token_recv[i] == 0) &&
1670 active_instance->faulty[i] == 0) {
1671 return; /* don't deliver token */
1672 }
1673 }
1674 active_timer_expired_token_cancel (active_instance);
1675
1676 rrp_instance->totemrrp_deliver_fn (
1677 context,
1678 msg,
1679 msg_len);
1680 }
1681 }
1682
active_token_send(struct totemrrp_instance * instance,const void * msg,unsigned int msg_len)1683 static void active_token_send (
1684 struct totemrrp_instance *instance,
1685 const void *msg,
1686 unsigned int msg_len)
1687 {
1688 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1689 int i;
1690 int msg_sent;
1691
1692 msg_sent = 0;
1693
1694 for (i = 0; i < instance->interface_count; i++) {
1695 if (rrp_algo_instance->faulty[i] == 0) {
1696 msg_sent = 1;
1697 totemnet_token_send (
1698 instance->net_handles[i],
1699 msg, msg_len);
1700
1701 }
1702 }
1703
1704 if (!msg_sent) {
1705 /*
1706 * All interfaces are faulty. It's still needed to send token
1707 * message to (potentionally) local host so use first interface.
1708 */
1709 totemnet_token_send (
1710 instance->net_handles[0],
1711 msg, msg_len);
1712
1713 }
1714 }
1715
active_recv_flush(struct totemrrp_instance * instance)1716 static void active_recv_flush (struct totemrrp_instance *instance)
1717 {
1718 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1719 unsigned int i;
1720
1721 for (i = 0; i < instance->interface_count; i++) {
1722 if (rrp_algo_instance->faulty[i] == 0) {
1723
1724 totemnet_recv_flush (instance->net_handles[i]);
1725 }
1726 }
1727 }
1728
active_send_flush(struct totemrrp_instance * instance)1729 static void active_send_flush (struct totemrrp_instance *instance)
1730 {
1731 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1732 unsigned int i;
1733
1734 for (i = 0; i < instance->interface_count; i++) {
1735 if (rrp_algo_instance->faulty[i] == 0) {
1736
1737 totemnet_send_flush (instance->net_handles[i]);
1738 }
1739 }
1740 }
1741
active_member_add(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)1742 static int active_member_add (
1743 struct totemrrp_instance *instance,
1744 const struct totem_ip_address *member,
1745 unsigned int iface_no)
1746 {
1747 int res;
1748 res = totemnet_member_add (instance->net_handles[iface_no], member);
1749 return (res);
1750 }
1751
active_member_remove(struct totemrrp_instance * instance,const struct totem_ip_address * member,unsigned int iface_no)1752 static int active_member_remove (
1753 struct totemrrp_instance *instance,
1754 const struct totem_ip_address *member,
1755 unsigned int iface_no)
1756 {
1757 int res;
1758 res = totemnet_member_remove (instance->net_handles[iface_no], member);
1759 return (res);
1760 }
1761
active_membership_changed(struct totemrrp_instance * rrp_instance,enum totem_configuration_type configuration_type,const struct srp_addr * member_list,size_t member_list_entries,const struct srp_addr * left_list,size_t left_list_entries,const struct srp_addr * joined_list,size_t joined_list_entries,const struct memb_ring_id * ring_id)1762 static void active_membership_changed (
1763 struct totemrrp_instance *rrp_instance,
1764 enum totem_configuration_type configuration_type,
1765 const struct srp_addr *member_list, size_t member_list_entries,
1766 const struct srp_addr *left_list, size_t left_list_entries,
1767 const struct srp_addr *joined_list, size_t joined_list_entries,
1768 const struct memb_ring_id *ring_id)
1769 {
1770 int i;
1771 int interface;
1772
1773 for (interface = 0; interface < rrp_instance->interface_count; interface++) {
1774 for (i = 0; i < left_list_entries; i++) {
1775 if (left_list->no_addrs < interface + 1 ||
1776 (left_list[i].addr[interface].family != AF_INET &&
1777 left_list[i].addr[interface].family != AF_INET6)) {
1778 log_printf(rrp_instance->totemrrp_log_level_error,
1779 "Membership left list contains incorrect address. "
1780 "This is sign of misconfiguration between nodes!");
1781 } else {
1782 totemnet_member_set_active(rrp_instance->net_handles[interface],
1783 &left_list[i].addr[interface], 0);
1784 }
1785 }
1786
1787 for (i = 0; i < joined_list_entries; i++) {
1788 if (joined_list->no_addrs < interface + 1 ||
1789 (joined_list[i].addr[interface].family != AF_INET &&
1790 joined_list[i].addr[interface].family != AF_INET6)) {
1791 log_printf(rrp_instance->totemrrp_log_level_error,
1792 "Membership join list contains incorrect address. "
1793 "This is sign of misconfiguration between nodes!");
1794 } else {
1795 totemnet_member_set_active(rrp_instance->net_handles[interface],
1796 &joined_list[i].addr[interface], 1);
1797 }
1798 }
1799 }
1800 }
1801
active_iface_check(struct totemrrp_instance * instance)1802 static void active_iface_check (struct totemrrp_instance *instance)
1803 {
1804 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1805 unsigned int i;
1806
1807 for (i = 0; i < instance->interface_count; i++) {
1808 if (rrp_algo_instance->faulty[i] == 0) {
1809
1810 totemnet_iface_check (instance->net_handles[i]);
1811 }
1812 }
1813 }
1814
active_processor_count_set(struct totemrrp_instance * instance,unsigned int processor_count)1815 static void active_processor_count_set (
1816 struct totemrrp_instance *instance,
1817 unsigned int processor_count)
1818 {
1819 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1820 unsigned int i;
1821
1822 for (i = 0; i < instance->interface_count; i++) {
1823 if (rrp_algo_instance->faulty[i] == 0) {
1824
1825 totemnet_processor_count_set (instance->net_handles[i],
1826 processor_count);
1827 }
1828 }
1829 }
1830
active_token_target_set(struct totemrrp_instance * instance,struct totem_ip_address * token_target,unsigned int iface_no)1831 static void active_token_target_set (
1832 struct totemrrp_instance *instance,
1833 struct totem_ip_address *token_target,
1834 unsigned int iface_no)
1835 {
1836 totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1837 }
1838
active_mcast_recv_empty(struct totemrrp_instance * instance)1839 static int active_mcast_recv_empty (
1840 struct totemrrp_instance *instance)
1841 {
1842 int res;
1843 int msgs_emptied = 0;
1844 int i;
1845
1846 for (i = 0; i < instance->interface_count; i++) {
1847 res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1848 if (res == -1) {
1849 return (-1);
1850 }
1851 if (res == 1) {
1852 msgs_emptied = 1;
1853 }
1854 }
1855
1856 return (msgs_emptied);
1857 }
1858
active_ring_reenable(struct totemrrp_instance * instance,unsigned int iface_no)1859 static void active_ring_reenable (
1860 struct totemrrp_instance *instance,
1861 unsigned int iface_no)
1862 {
1863 struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1864 int i;
1865
1866 if (iface_no == instance->interface_count) {
1867 memset (rrp_algo_instance->last_token_recv, 0, sizeof (unsigned int) *
1868 instance->interface_count);
1869 memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1870 instance->interface_count);
1871 memset (rrp_algo_instance->counter_problems, 0, sizeof (unsigned int) *
1872 instance->interface_count);
1873
1874 for (i = 0; i < instance->interface_count; i++) {
1875 stats_set_interface_faulty (instance, i, 0);
1876 }
1877 } else {
1878 rrp_algo_instance->last_token_recv[iface_no] = 0;
1879 rrp_algo_instance->faulty[iface_no] = 0;
1880 rrp_algo_instance->counter_problems[iface_no] = 0;
1881
1882 stats_set_interface_faulty (instance, iface_no, 0);
1883 }
1884 }
1885
totemrrp_instance_initialize(struct totemrrp_instance * instance)1886 static void totemrrp_instance_initialize (struct totemrrp_instance *instance)
1887 {
1888 memset (instance, 0, sizeof (struct totemrrp_instance));
1889 }
1890
totemrrp_algorithm_set(struct totem_config * totem_config,struct totemrrp_instance * instance)1891 static int totemrrp_algorithm_set (
1892 struct totem_config *totem_config,
1893 struct totemrrp_instance *instance)
1894 {
1895 unsigned int res = -1;
1896 unsigned int i;
1897
1898 for (i = 0; i < RRP_ALGOS_COUNT; i++) {
1899 if (strcmp (totem_config->rrp_mode, rrp_algos[i]->name) == 0) {
1900 instance->rrp_algo = rrp_algos[i];
1901 if (rrp_algos[i]->initialize) {
1902 instance->rrp_algo_instance = rrp_algos[i]->initialize (
1903 instance,
1904 totem_config->interface_count);
1905 }
1906 res = 0;
1907 break;
1908 }
1909 }
1910 for (i = 0; i < totem_config->interface_count; i++) {
1911 instance->status[i] = malloc (STATUS_STR_LEN+1);
1912 snprintf (instance->status[i], STATUS_STR_LEN,
1913 "ring %d active with no faults", i);
1914 }
1915 return (res);
1916 }
1917
rrp_deliver_fn(void * context,const void * msg,unsigned int msg_len)1918 void rrp_deliver_fn (
1919 void *context,
1920 const void *msg,
1921 unsigned int msg_len)
1922 {
1923 unsigned int token_seqid;
1924 unsigned int token_is;
1925
1926 struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1927 struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
1928 const struct message_header *hdr = msg;
1929 struct message_header tmp_msg, activate_msg;
1930
1931 memset(&tmp_msg, 0, sizeof(struct message_header));
1932 memset(&activate_msg, 0, sizeof(struct message_header));
1933
1934 rrp_instance->totemrrp_token_seqid_get (
1935 msg,
1936 &token_seqid,
1937 &token_is);
1938
1939 if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVE) {
1940 log_printf (
1941 rrp_instance->totemrrp_log_level_debug,
1942 "received message requesting test of ring now active");
1943
1944 if (hdr->endian_detector != ENDIAN_LOCAL) {
1945 test_active_msg_endian_convert(hdr, &tmp_msg);
1946 hdr = &tmp_msg;
1947 }
1948
1949 if (hdr->nodeid_activator == rrp_instance->my_nodeid) {
1950 /*
1951 * Send an activate message
1952 */
1953 activate_msg.type = MESSAGE_TYPE_RING_TEST_ACTIVATE;
1954 activate_msg.endian_detector = ENDIAN_LOCAL;
1955 activate_msg.ring_number = hdr->ring_number;
1956 activate_msg.nodeid_activator = rrp_instance->my_nodeid;
1957 totemnet_token_send (
1958 rrp_instance->net_handles[deliver_fn_context->iface_no],
1959 &activate_msg, sizeof (struct message_header));
1960 } else {
1961 /*
1962 * Send a ring test message
1963 */
1964 totemnet_token_send (
1965 rrp_instance->net_handles[deliver_fn_context->iface_no],
1966 msg, msg_len);
1967 }
1968 } else
1969 if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVATE) {
1970
1971 if (hdr->endian_detector != ENDIAN_LOCAL) {
1972 test_active_msg_endian_convert(hdr, &tmp_msg);
1973 hdr = &tmp_msg;
1974 }
1975
1976 log_printf (
1977 rrp_instance->totemrrp_log_level_debug,
1978 "Received ring test activate message for ring %d sent by node %u",
1979 hdr->ring_number,
1980 hdr->nodeid_activator);
1981
1982 if (rrp_instance->stats.faulty[deliver_fn_context->iface_no]) {
1983 log_printf (rrp_instance->totemrrp_log_level_notice,
1984 "Automatically recovered ring %d", hdr->ring_number);
1985 }
1986
1987 totemrrp_ring_reenable (rrp_instance, deliver_fn_context->iface_no);
1988 if (hdr->nodeid_activator != rrp_instance->my_nodeid) {
1989 totemnet_token_send (
1990 rrp_instance->net_handles[deliver_fn_context->iface_no],
1991 msg, msg_len);
1992 }
1993 } else
1994 if (token_is) {
1995 /*
1996 * Deliver to the token receiver for this rrp algorithm
1997 */
1998 rrp_instance->rrp_algo->token_recv (
1999 rrp_instance,
2000 deliver_fn_context->iface_no,
2001 deliver_fn_context->context,
2002 msg,
2003 msg_len,
2004 token_seqid);
2005 } else {
2006 /*
2007 * Deliver to the mcast receiver for this rrp algorithm
2008 */
2009 rrp_instance->rrp_algo->mcast_recv (
2010 rrp_instance,
2011 deliver_fn_context->iface_no,
2012 deliver_fn_context->context,
2013 msg,
2014 msg_len);
2015 }
2016 }
2017
rrp_iface_change_fn(void * context,const struct totem_ip_address * iface_addr)2018 void rrp_iface_change_fn (
2019 void *context,
2020 const struct totem_ip_address *iface_addr)
2021 {
2022 struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
2023
2024 deliver_fn_context->instance->my_nodeid = iface_addr->nodeid;
2025 deliver_fn_context->instance->totemrrp_iface_change_fn (
2026 deliver_fn_context->context,
2027 iface_addr,
2028 deliver_fn_context->iface_no);
2029 }
2030
totemrrp_finalize(void * rrp_context)2031 int totemrrp_finalize (
2032 void *rrp_context)
2033 {
2034 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2035 int i;
2036
2037 for (i = 0; i < instance->interface_count; i++) {
2038 totemnet_finalize (instance->net_handles[i]);
2039 }
2040 free (instance->net_handles);
2041 free (instance);
2042 return (0);
2043 }
2044
rrp_target_set_completed(void * context)2045 static void rrp_target_set_completed (void *context)
2046 {
2047 struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
2048
2049 deliver_fn_context->instance->totemrrp_target_set_completed (deliver_fn_context->context);
2050 }
2051
2052 /*
2053 * Totem Redundant Ring interface
2054 * depends on poll abstraction, POSIX, IPV4
2055 */
2056
2057 /*
2058 * Create an instance
2059 */
totemrrp_initialize(qb_loop_t * poll_handle,void ** rrp_context,struct totem_config * totem_config,totemsrp_stats_t * stats,void * context,void (* deliver_fn)(void * context,const void * msg,unsigned int msg_len),void (* iface_change_fn)(void * context,const struct totem_ip_address * iface_addr,unsigned int iface_no),void (* token_seqid_get)(const void * msg,unsigned int * seqid,unsigned int * token_is),unsigned int (* msgs_missing)(void),void (* target_set_completed)(void * context))2060 int totemrrp_initialize (
2061 qb_loop_t *poll_handle,
2062 void **rrp_context,
2063 struct totem_config *totem_config,
2064 totemsrp_stats_t *stats,
2065 void *context,
2066
2067 void (*deliver_fn) (
2068 void *context,
2069 const void *msg,
2070 unsigned int msg_len),
2071
2072 void (*iface_change_fn) (
2073 void *context,
2074 const struct totem_ip_address *iface_addr,
2075 unsigned int iface_no),
2076
2077 void (*token_seqid_get) (
2078 const void *msg,
2079 unsigned int *seqid,
2080 unsigned int *token_is),
2081
2082 unsigned int (*msgs_missing) (void),
2083
2084 void (*target_set_completed) (void *context))
2085 {
2086 struct totemrrp_instance *instance;
2087 unsigned int res;
2088 int i;
2089
2090 instance = malloc (sizeof (struct totemrrp_instance));
2091 if (instance == 0) {
2092 return (-1);
2093 }
2094
2095 totemrrp_instance_initialize (instance);
2096
2097 instance->totem_config = totem_config;
2098 stats->rrp = &instance->stats;
2099 instance->stats.interface_count = totem_config->interface_count;
2100 instance->stats.faulty = calloc(instance->stats.interface_count, sizeof(uint8_t));
2101
2102 res = totemrrp_algorithm_set (
2103 instance->totem_config,
2104 instance);
2105 if (res == -1) {
2106 goto error_destroy;
2107 }
2108
2109 /*
2110 * Configure logging
2111 */
2112 instance->totemrrp_log_level_security = totem_config->totem_logging_configuration.log_level_security;
2113 instance->totemrrp_log_level_error = totem_config->totem_logging_configuration.log_level_error;
2114 instance->totemrrp_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
2115 instance->totemrrp_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
2116 instance->totemrrp_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
2117 instance->totemrrp_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
2118 instance->totemrrp_log_printf = totem_config->totem_logging_configuration.log_printf;
2119
2120 instance->interfaces = totem_config->interfaces;
2121
2122 instance->poll_handle = poll_handle;
2123
2124 instance->totemrrp_deliver_fn = deliver_fn;
2125
2126 instance->totemrrp_iface_change_fn = iface_change_fn;
2127
2128 instance->totemrrp_token_seqid_get = token_seqid_get;
2129
2130 instance->totemrrp_target_set_completed = target_set_completed;
2131
2132 instance->totemrrp_msgs_missing = msgs_missing;
2133
2134 instance->interface_count = totem_config->interface_count;
2135
2136 instance->net_handles = malloc (sizeof (void *) * totem_config->interface_count);
2137
2138 instance->context = context;
2139
2140 instance->poll_handle = poll_handle;
2141
2142
2143 for (i = 0; i < totem_config->interface_count; i++) {
2144 struct deliver_fn_context *deliver_fn_context;
2145
2146 deliver_fn_context = malloc (sizeof (struct deliver_fn_context));
2147 assert (deliver_fn_context);
2148 deliver_fn_context->instance = instance;
2149 deliver_fn_context->context = context;
2150 deliver_fn_context->iface_no = i;
2151 instance->deliver_fn_context[i] = (void *)deliver_fn_context;
2152
2153 res = totemnet_initialize (
2154 poll_handle,
2155 &instance->net_handles[i],
2156 totem_config,
2157 stats,
2158 i,
2159 (void *)deliver_fn_context,
2160 rrp_deliver_fn,
2161 rrp_iface_change_fn,
2162 rrp_target_set_completed);
2163
2164 if (res == -1) {
2165 goto error_destroy;
2166 }
2167
2168 totemnet_net_mtu_adjust (instance->net_handles[i], totem_config);
2169 }
2170
2171 *rrp_context = instance;
2172 return (0);
2173
2174 error_destroy:
2175 free (instance);
2176 return (res);
2177 }
2178
totemrrp_buffer_alloc(void * rrp_context)2179 void *totemrrp_buffer_alloc (void *rrp_context)
2180 {
2181 struct totemrrp_instance *instance = rrp_context;
2182 assert (instance != NULL);
2183 return totemnet_buffer_alloc (instance->net_handles[0]);
2184 }
2185
totemrrp_buffer_release(void * rrp_context,void * ptr)2186 void totemrrp_buffer_release (void *rrp_context, void *ptr)
2187 {
2188 struct totemrrp_instance *instance = rrp_context;
2189 assert (instance != NULL);
2190 totemnet_buffer_release (instance->net_handles[0], ptr);
2191 }
2192
totemrrp_processor_count_set(void * rrp_context,unsigned int processor_count)2193 int totemrrp_processor_count_set (
2194 void *rrp_context,
2195 unsigned int processor_count)
2196 {
2197 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2198 instance->rrp_algo->processor_count_set (instance, processor_count);
2199
2200 instance->processor_count = processor_count;
2201
2202 return (0);
2203 }
2204
totemrrp_token_target_set(void * rrp_context,struct totem_ip_address * addr,unsigned int iface_no)2205 int totemrrp_token_target_set (
2206 void *rrp_context,
2207 struct totem_ip_address *addr,
2208 unsigned int iface_no)
2209 {
2210 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2211 instance->rrp_algo->token_target_set (instance, addr, iface_no);
2212
2213 return (0);
2214 }
totemrrp_recv_flush(void * rrp_context)2215 int totemrrp_recv_flush (void *rrp_context)
2216 {
2217 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2218
2219 instance->rrp_algo->recv_flush (instance);
2220
2221 return (0);
2222 }
2223
totemrrp_send_flush(void * rrp_context)2224 int totemrrp_send_flush (void *rrp_context)
2225 {
2226 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2227 instance->rrp_algo->send_flush (instance);
2228
2229 return (0);
2230 }
2231
totemrrp_token_send(void * rrp_context,const void * msg,unsigned int msg_len)2232 int totemrrp_token_send (
2233 void *rrp_context,
2234 const void *msg,
2235 unsigned int msg_len)
2236 {
2237 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2238 instance->rrp_algo->token_send (instance, msg, msg_len);
2239
2240 return (0);
2241 }
2242
totemrrp_mcast_flush_send(void * rrp_context,const void * msg,unsigned int msg_len)2243 int totemrrp_mcast_flush_send (
2244 void *rrp_context,
2245 const void *msg,
2246 unsigned int msg_len)
2247 {
2248 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2249 int res = 0;
2250
2251 // TODO this needs to return the result
2252 instance->rrp_algo->mcast_flush_send (instance, msg, msg_len);
2253
2254 return (res);
2255 }
2256
totemrrp_mcast_noflush_send(void * rrp_context,const void * msg,unsigned int msg_len)2257 int totemrrp_mcast_noflush_send (
2258 void *rrp_context,
2259 const void *msg,
2260 unsigned int msg_len)
2261 {
2262 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2263 /*
2264 * merge detects go out through mcast_flush_send so it is safe to
2265 * flush these messages if we are only one processor. This avoids
2266 * an encryption/hmac and decryption/hmac
2267 */
2268 if (instance->processor_count > 1) {
2269
2270 // TODO this needs to return the result
2271 instance->rrp_algo->mcast_noflush_send (instance, msg, msg_len);
2272 }
2273
2274 return (0);
2275 }
2276
totemrrp_iface_check(void * rrp_context)2277 int totemrrp_iface_check (void *rrp_context)
2278 {
2279 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2280
2281 instance->rrp_algo->iface_check (instance);
2282
2283 return (0);
2284 }
2285
totemrrp_ifaces_get(void * rrp_context,char *** status,unsigned int * iface_count)2286 int totemrrp_ifaces_get (
2287 void *rrp_context,
2288 char ***status,
2289 unsigned int *iface_count)
2290 {
2291 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2292 *status = instance->status;
2293
2294 if (iface_count) {
2295 *iface_count = instance->interface_count;
2296 }
2297
2298 return (0);
2299 }
2300
totemrrp_crypto_set(void * rrp_context,const char * cipher_type,const char * hash_type)2301 int totemrrp_crypto_set (
2302 void *rrp_context,
2303 const char *cipher_type,
2304 const char *hash_type)
2305 {
2306 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2307 int res;
2308
2309 res = totemnet_crypto_set(instance->net_handles[0], cipher_type, hash_type);
2310
2311 return (res);
2312 }
2313
2314
2315 /*
2316 * iface_no indicates the interface number [0, ..., interface_count-1] of the
2317 * specific ring which will be reenabled. We specify iface_no == interface_count
2318 * means reenabling all the rings.
2319 */
totemrrp_ring_reenable(void * rrp_context,unsigned int iface_no)2320 int totemrrp_ring_reenable (
2321 void *rrp_context,
2322 unsigned int iface_no)
2323 {
2324 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2325 int res = 0;
2326 unsigned int i;
2327
2328 instance->rrp_algo->ring_reenable (instance, iface_no);
2329
2330 if (iface_no == instance->interface_count) {
2331 for (i = 0; i < instance->interface_count; i++) {
2332 snprintf (instance->status[i], STATUS_STR_LEN,
2333 "ring %d active with no faults", i);
2334 }
2335 } else {
2336 snprintf (instance->status[iface_no], STATUS_STR_LEN,
2337 "ring %d active with no faults", iface_no);
2338 }
2339
2340 return (res);
2341 }
2342
totemrrp_mcast_recv_empty(void * rrp_context)2343 extern int totemrrp_mcast_recv_empty (
2344 void *rrp_context)
2345 {
2346 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2347 int res;
2348
2349 res = instance->rrp_algo->mcast_recv_empty (instance);
2350
2351 return (res);
2352 }
2353
totemrrp_member_add(void * rrp_context,const struct totem_ip_address * member,int iface_no)2354 int totemrrp_member_add (
2355 void *rrp_context,
2356 const struct totem_ip_address *member,
2357 int iface_no)
2358 {
2359 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2360 int res;
2361
2362 res = instance->rrp_algo->member_add (instance, member, iface_no);
2363
2364 return (res);
2365 }
2366
totemrrp_member_remove(void * rrp_context,const struct totem_ip_address * member,int iface_no)2367 int totemrrp_member_remove (
2368 void *rrp_context,
2369 const struct totem_ip_address *member,
2370 int iface_no)
2371 {
2372 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2373 int res;
2374
2375 res = instance->rrp_algo->member_remove (instance, member, iface_no);
2376
2377 return (res);
2378 }
2379
totemrrp_membership_changed(void * rrp_context,enum totem_configuration_type configuration_type,const struct srp_addr * member_list,size_t member_list_entries,const struct srp_addr * left_list,size_t left_list_entries,const struct srp_addr * joined_list,size_t joined_list_entries,const struct memb_ring_id * ring_id)2380 void totemrrp_membership_changed (
2381 void *rrp_context,
2382 enum totem_configuration_type configuration_type,
2383 const struct srp_addr *member_list, size_t member_list_entries,
2384 const struct srp_addr *left_list, size_t left_list_entries,
2385 const struct srp_addr *joined_list, size_t joined_list_entries,
2386 const struct memb_ring_id *ring_id)
2387 {
2388 struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2389
2390 instance->rrp_algo->membership_changed (instance,
2391 configuration_type,
2392 member_list, member_list_entries,
2393 left_list, left_list_entries,
2394 joined_list, joined_list_entries,
2395 ring_id);
2396 }
2397