1 /*
2 * Copyright (c) 2017 Red Hat, Inc.
3 *
4 * All rights reserved.
5 *
6 * Author: Jan Friesse (jfriesse@redhat.com)
7 *
8 * This software licensed under BSD license, the text of which follows:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the Red Hat, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include "qdevice-log.h"
36 #include "qdevice-net-algorithm.h"
37 #include "qdevice-net-cast-vote-timer.h"
38 #include "qdevice-net-heuristics.h"
39 #include "qdevice-net-send.h"
40 #include "qdevice-net-votequorum.h"
41
42 enum tlv_heuristics
qdevice_net_heuristics_exec_result_to_tlv(enum qdevice_heuristics_exec_result exec_result)43 qdevice_net_heuristics_exec_result_to_tlv(enum qdevice_heuristics_exec_result exec_result)
44 {
45 enum tlv_heuristics res;
46
47 switch (exec_result) {
48 case QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED: res = TLV_HEURISTICS_UNDEFINED; break;
49 case QDEVICE_HEURISTICS_EXEC_RESULT_PASS: res = TLV_HEURISTICS_PASS; break;
50 case QDEVICE_HEURISTICS_EXEC_RESULT_FAIL: res = TLV_HEURISTICS_FAIL; break;
51 default:
52 qdevice_log(LOG_ERR, "qdevice_net_heuristics_exec_result_to_tlv: Unhandled "
53 "heuristics exec result %s",
54 qdevice_heuristics_exec_result_to_str(exec_result));
55 exit(1);
56 break;
57 }
58
59 return (res);
60 }
61
62 static int
qdevice_net_regular_heuristics_exec_result_callback(void * heuristics_instance_ptr,uint32_t seq_number,enum qdevice_heuristics_exec_result exec_result)63 qdevice_net_regular_heuristics_exec_result_callback(void *heuristics_instance_ptr,
64 uint32_t seq_number, enum qdevice_heuristics_exec_result exec_result)
65 {
66 struct qdevice_heuristics_instance *heuristics_instance;
67 struct qdevice_instance *instance;
68 struct qdevice_net_instance *net_instance;
69 int send_msg;
70 enum tlv_vote vote;
71 enum tlv_heuristics heuristics;
72
73 heuristics_instance = (struct qdevice_heuristics_instance *)heuristics_instance_ptr;
74 instance = heuristics_instance->qdevice_instance_ptr;
75 net_instance = instance->model_data;
76
77 if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
78 qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
79 qdevice_log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");
80
81 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
82 net_instance->schedule_disconnect = 1;
83
84 return (0);
85 }
86
87 heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);
88
89 if (exec_result == QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED) {
90 /*
91 * Can happen when user disables heuristics during runtime
92 */
93 return (0);
94 }
95
96 if (net_instance->latest_heuristics_result != heuristics) {
97 qdevice_log(LOG_ERR, "Heuristics result changed from %s to %s",
98 tlv_heuristics_to_str(net_instance->latest_heuristics_result),
99 tlv_heuristics_to_str(heuristics));
100
101 if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
102 /*
103 * Not connected to qnetd
104 */
105 send_msg = 0;
106 } else {
107 send_msg = 1;
108 }
109
110 vote = TLV_VOTE_NO_CHANGE;
111
112 if (qdevice_net_algorithm_heuristics_change(net_instance, &heuristics, &send_msg,
113 &vote) == -1) {
114 qdevice_log(LOG_ERR, "Algorithm returned error. Disconnecting.");
115
116 net_instance->disconnect_reason =
117 QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
118 net_instance->schedule_disconnect = 1;
119
120 return (0);
121 } else {
122 qdevice_log(LOG_DEBUG, "Algorithm decided to %s message with heuristics result "
123 "%s and result vote is %s", (send_msg ? "send" : "not send"),
124 tlv_heuristics_to_str(heuristics), tlv_vote_to_str(vote));
125 }
126
127 if (send_msg) {
128 if (heuristics == TLV_HEURISTICS_UNDEFINED) {
129 qdevice_log(LOG_ERR, "Inconsistent algorithm result. "
130 "It's not possible to send message with undefined heuristics. "
131 "Disconnecting.");
132
133 net_instance->disconnect_reason =
134 QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
135 net_instance->schedule_disconnect = 1;
136
137 return (0);
138 }
139
140 if (!net_instance->server_supports_heuristics) {
141 qdevice_log(LOG_ERR, "Server doesn't support heuristics. "
142 "Disconnecting.");
143
144 net_instance->disconnect_reason =
145 QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_OPT;
146 net_instance->schedule_disconnect = 1;
147
148 return (0);
149 }
150
151 if (qdevice_net_send_heuristics_change(net_instance, heuristics) != 0) {
152 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
153 net_instance->schedule_disconnect = 1;
154
155 return (0);
156 }
157 }
158
159 if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
160 qdevice_log(LOG_CRIT, "qdevice_net_heuristics_exec_result_callback "
161 "Can't update cast vote timer");
162
163 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
164 net_instance->schedule_disconnect = 1;
165
166 return (0);
167 }
168 }
169
170 net_instance->latest_regular_heuristics_result = heuristics;
171 net_instance->latest_heuristics_result = heuristics;
172
173 if (qdevice_net_heuristics_schedule_timer(net_instance) != 0) {
174 return (0);
175 }
176
177 return (0);
178 }
179
180 static int
qdevice_net_connect_heuristics_exec_result_callback(void * heuristics_instance_ptr,uint32_t seq_number,enum qdevice_heuristics_exec_result exec_result)181 qdevice_net_connect_heuristics_exec_result_callback(void *heuristics_instance_ptr,
182 uint32_t seq_number, enum qdevice_heuristics_exec_result exec_result)
183 {
184 struct qdevice_heuristics_instance *heuristics_instance;
185 struct qdevice_instance *instance;
186 struct qdevice_net_instance *net_instance;
187 enum tlv_vote vote;
188 enum tlv_heuristics heuristics;
189 int send_config_node_list;
190 int send_membership_node_list;
191 int send_quorum_node_list;
192 struct tlv_ring_id tlv_rid;
193 enum tlv_quorate quorate;
194
195 heuristics_instance = (struct qdevice_heuristics_instance *)heuristics_instance_ptr;
196 instance = heuristics_instance->qdevice_instance_ptr;
197 net_instance = instance->model_data;
198
199
200 if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
201 qdevice_net_connect_heuristics_exec_result_callback, 0) != 0) {
202 qdevice_log(LOG_ERR, "Can't deactivate net connect heuristics exec callback notifier");
203
204 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
205 net_instance->schedule_disconnect = 1;
206
207 return (0);
208 }
209
210 heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);
211
212 send_config_node_list = 1;
213 send_membership_node_list = 1;
214 send_quorum_node_list = 1;
215 vote = TLV_VOTE_WAIT_FOR_REPLY;
216
217 if (qdevice_net_algorithm_connected(net_instance, &heuristics, &send_config_node_list,
218 &send_membership_node_list, &send_quorum_node_list, &vote) != 0) {
219 qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
220 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
221 return (0);
222 } else {
223 qdevice_log(LOG_DEBUG, "Algorithm decided to %s config node list, %s membership "
224 "node list, %s quorum node list, heuristics is %s and result vote is %s",
225 (send_config_node_list ? "send" : "not send"),
226 (send_membership_node_list ? "send" : "not send"),
227 (send_quorum_node_list ? "send" : "not send"),
228 tlv_heuristics_to_str(heuristics),
229 tlv_vote_to_str(vote));
230 }
231
232 /*
233 * Now we can finally really send node list, votequorum node list and update timer
234 */
235 if (send_config_node_list) {
236 if (qdevice_net_send_config_node_list(net_instance,
237 &instance->config_node_list,
238 instance->config_node_list_version_set,
239 instance->config_node_list_version, 1) != 0) {
240 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
241 return (0);
242 }
243 }
244
245 if (send_membership_node_list) {
246 qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid,
247 &instance->vq_node_list_ring_id);
248
249 if (qdevice_net_send_membership_node_list(net_instance, &tlv_rid,
250 instance->vq_node_list_entries,
251 instance->vq_node_list,
252 heuristics) != 0) {
253 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
254 return (0);
255 }
256 }
257
258 if (send_quorum_node_list) {
259 quorate = (instance->vq_quorum_quorate ?
260 TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE);
261
262 if (qdevice_net_send_quorum_node_list(net_instance,
263 quorate,
264 instance->vq_quorum_node_list_entries,
265 instance->vq_quorum_node_list) != 0) {
266 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
267 return (0);
268 }
269 }
270
271 if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
272 qdevice_log(LOG_CRIT, "qdevice_net_msg_received_set_option_reply fatal error. "
273 " Can't update cast vote timer vote");
274 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
275 }
276
277 net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
278 net_instance->connected_since_time = time(NULL);
279
280 net_instance->latest_connect_heuristics_result = heuristics;
281 net_instance->latest_heuristics_result = heuristics;
282
283 return (0);
284 }
285
286 static int
qdevice_net_heuristics_timer_callback(void * data1,void * data2)287 qdevice_net_heuristics_timer_callback(void *data1, void *data2)
288 {
289 struct qdevice_net_instance *net_instance;
290 struct qdevice_heuristics_instance *heuristics_instance;
291
292 net_instance = (struct qdevice_net_instance *)data1;
293 heuristics_instance = &net_instance->qdevice_instance_ptr->heuristics_instance;
294
295 if (qdevice_heuristics_waiting_for_result(heuristics_instance)) {
296 qdevice_log(LOG_DEBUG, "Not executing regular heuristics because other heuristics is already running.");
297
298 return (1);
299 }
300
301 net_instance->regular_heuristics_timer = NULL;
302
303 qdevice_log(LOG_DEBUG, "Executing regular heuristics.");
304
305 if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
306 qdevice_net_regular_heuristics_exec_result_callback, 1) != 0) {
307 qdevice_log(LOG_ERR, "Can't activate net regular heuristics exec callback notifier");
308
309 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
310 net_instance->schedule_disconnect = 1;
311
312 return (0);
313 }
314
315 if (qdevice_heuristics_exec(heuristics_instance,
316 net_instance->qdevice_instance_ptr->sync_in_progress) != 0) {
317 qdevice_log(LOG_ERR, "Can't execute regular heuristics.");
318
319 net_instance->schedule_disconnect = 1;
320 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;
321
322 return (0);
323 }
324
325 /*
326 * Do not schedule this callback again. It's going to be scheduled in the
327 * qdevice_net_heuristics_exec_result_callback
328 */
329 return (0);
330 }
331
332 int
qdevice_net_heuristics_stop_timer(struct qdevice_net_instance * net_instance)333 qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance)
334 {
335 struct qdevice_instance *instance;
336 struct qdevice_heuristics_instance *heuristics_instance;
337
338 instance = net_instance->qdevice_instance_ptr;
339 heuristics_instance = &instance->heuristics_instance;
340
341 if (net_instance->regular_heuristics_timer != NULL) {
342 qdevice_log(LOG_DEBUG, "Regular heuristics timer stopped");
343
344 timer_list_delete(&net_instance->main_timer_list, net_instance->regular_heuristics_timer);
345 net_instance->regular_heuristics_timer = NULL;
346
347 if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
348 qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
349 qdevice_log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");
350
351 net_instance->disconnect_reason =
352 QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
353 net_instance->schedule_disconnect = 1;
354 return (-1);
355 }
356 }
357
358 return (0);
359 }
360
361 int
qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance * net_instance)362 qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance)
363 {
364 uint32_t interval;
365 struct qdevice_instance *instance;
366 struct qdevice_heuristics_instance *heuristics_instance;
367
368 instance = net_instance->qdevice_instance_ptr;
369 heuristics_instance = &instance->heuristics_instance;
370
371 if (heuristics_instance->mode != QDEVICE_HEURISTICS_MODE_ENABLED) {
372 qdevice_log(LOG_DEBUG, "Not scheduling heuristics timer because mode is not enabled");
373
374 if (qdevice_net_heuristics_stop_timer(net_instance) != 0) {
375 return (-1);
376 }
377
378 return (0);
379 }
380
381 if (net_instance->regular_heuristics_timer != NULL) {
382 qdevice_log(LOG_DEBUG, "Not scheduling heuristics timer because it is already scheduled");
383
384 return (0);
385 }
386
387 interval = heuristics_instance->interval;
388
389 qdevice_log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval);
390
391 net_instance->regular_heuristics_timer = timer_list_add(&net_instance->main_timer_list,
392 interval,
393 qdevice_net_heuristics_timer_callback,
394 (void *)net_instance, NULL);
395
396 if (net_instance->regular_heuristics_timer == NULL) {
397 qdevice_log(LOG_ERR, "Can't schedule regular heuristics.");
398
399 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HEURISTICS_TIMER;
400 net_instance->schedule_disconnect = 1;
401 return (-1);
402 }
403
404 return (0);
405 }
406
407 int
qdevice_net_heuristics_init(struct qdevice_net_instance * net_instance)408 qdevice_net_heuristics_init(struct qdevice_net_instance *net_instance)
409 {
410
411 if (qdevice_heuristics_result_notifier_list_add(
412 &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
413 qdevice_net_regular_heuristics_exec_result_callback) == NULL) {
414 qdevice_log(LOG_ERR, "Can't add net regular heuristics exec callback into notifier");
415
416 return (-1);
417 }
418
419 if (qdevice_heuristics_result_notifier_list_add(
420 &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
421 qdevice_net_connect_heuristics_exec_result_callback) == NULL) {
422 qdevice_log(LOG_ERR, "Can't add net connect heuristics exec callback into notifier");
423
424 return (-1);
425 }
426
427 return (0);
428 }
429
430 int
qdevice_net_heuristics_exec_after_connect(struct qdevice_net_instance * net_instance)431 qdevice_net_heuristics_exec_after_connect(struct qdevice_net_instance *net_instance)
432 {
433 struct qdevice_instance *instance;
434 struct qdevice_heuristics_instance *heuristics_instance;
435
436 instance = net_instance->qdevice_instance_ptr;
437 heuristics_instance = &instance->heuristics_instance;
438
439 qdevice_log(LOG_DEBUG, "Executing after-connect heuristics.");
440
441 if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
442 qdevice_net_connect_heuristics_exec_result_callback, 1) != 0) {
443 qdevice_log(LOG_ERR, "Can't activate net connect heuristics exec callback notifier");
444
445 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
446 net_instance->schedule_disconnect = 1;
447
448 return (-1);
449 }
450
451 if (qdevice_heuristics_exec(heuristics_instance,
452 instance->sync_in_progress) != 0) {
453 qdevice_log(LOG_ERR, "Can't execute connect heuristics.");
454
455 net_instance->schedule_disconnect = 1;
456 net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;
457
458 return (-1);
459 }
460
461 return (0);
462 }
463