1 /*
2  * Copyright (c) 2015-2016 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Jan Friesse (jfriesse@redhat.com)
7  *
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  *   this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  *   this list of conditions and the following disclaimer in the documentation
17  *   and/or other materials provided with the distribution.
18  * - Neither the name of the Red Hat, Inc. nor the names of its
19  *   contributors may be used to endorse or promote products derived from this
20  *   software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <poll.h>
36 
37 #include "qdevice-config.h"
38 #include "qdevice-log.h"
39 #include "qdevice-votequorum.h"
40 #include "qdevice-model.h"
41 #include "utils.h"
42 
43 static void
qdevice_votequorum_quorum_notify_callback(votequorum_handle_t votequorum_handle,uint64_t context,uint32_t quorate,uint32_t node_list_entries,votequorum_node_t node_list[])44 qdevice_votequorum_quorum_notify_callback(votequorum_handle_t votequorum_handle,
45     uint64_t context, uint32_t quorate,
46     uint32_t node_list_entries, votequorum_node_t node_list[])
47 {
48 	struct qdevice_instance *instance;
49 	uint32_t u32;
50 
51 	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
52 		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
53 		exit(1);
54 	}
55 
56 	instance->sync_in_progress = 0;
57 
58 	qdevice_log(LOG_DEBUG, "Votequorum quorum notify callback:");
59 	qdevice_log(LOG_DEBUG, "  Quorate = %u", quorate);
60 
61 	qdevice_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
62 	for (u32 = 0; u32 < node_list_entries; u32++) {
63 		qdevice_log(LOG_DEBUG, "    %"PRIu32" nodeid = "UTILS_PRI_NODE_ID", state = %"PRIu32,
64 		    u32, node_list[u32].nodeid, node_list[u32].state);
65 	}
66 
67 	if (qdevice_model_votequorum_quorum_notify(instance, quorate, node_list_entries,
68 	    node_list) != 0) {
69 		qdevice_log(LOG_DEBUG, "qdevice_model_votequorum_quorum_notify returned error -> exit");
70 		exit(2);
71 	}
72 
73 	instance->vq_quorum_quorate = quorate;
74 	instance->vq_quorum_node_list_entries = node_list_entries;
75 
76 	free(instance->vq_quorum_node_list);
77 	instance->vq_quorum_node_list = malloc(sizeof(*node_list) * node_list_entries);
78 	if (instance->vq_quorum_node_list == NULL) {
79 		qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory");
80 		exit(1);
81 	}
82 	memcpy(instance->vq_quorum_node_list, node_list, sizeof(*node_list) * node_list_entries);
83 }
84 
85 static int
qdevice_votequorum_heuristics_exec_result_callback(void * heuristics_instance_ptr,uint32_t seq_number,enum qdevice_heuristics_exec_result exec_result)86 qdevice_votequorum_heuristics_exec_result_callback(
87     void *heuristics_instance_ptr,
88     uint32_t seq_number, enum qdevice_heuristics_exec_result exec_result)
89 {
90 	struct qdevice_heuristics_instance *heuristics_instance;
91 	struct qdevice_instance *instance;
92 
93 	heuristics_instance = (struct qdevice_heuristics_instance *)heuristics_instance_ptr;
94 	instance = heuristics_instance->qdevice_instance_ptr;
95 
96 	if (qdevice_heuristics_result_notifier_list_set_active(
97 	    &instance->heuristics_instance.exec_result_notifier_list,
98 	    qdevice_votequorum_heuristics_exec_result_callback, 0) != 0) {
99 		qdevice_log(LOG_CRIT, "Can't deactivate votequrorum heuristics exec callback notifier");
100 		exit(2);
101 	}
102 
103 	qdevice_log(LOG_DEBUG, "Votequorum heuristics exec result callback:");
104 	qdevice_log(LOG_DEBUG, "  seq_number = %"PRIu32", exec_result = %s",
105 	    seq_number, qdevice_heuristics_exec_result_to_str(exec_result));
106 
107 	if (qdevice_model_votequorum_node_list_heuristics_notify(instance, instance->vq_node_list_ring_id,
108 	    instance->vq_node_list_entries, instance->vq_node_list, exec_result) != 0) {
109 		qdevice_log(LOG_DEBUG, "qdevice_votequorum_node_list_heuristics_notify_callback returned error -> exit");
110 		exit(2);
111 	}
112 
113 	instance->vq_node_list_initial_heuristics_finished = 1;
114 	instance->vq_node_list_heuristics_result = exec_result;
115 
116 	return (0);
117 }
118 
119 static void
qdevice_votequorum_node_list_notify_callback(votequorum_handle_t votequorum_handle,uint64_t context,votequorum_ring_id_t votequorum_ring_id,uint32_t node_list_entries,uint32_t node_list[])120 qdevice_votequorum_node_list_notify_callback(votequorum_handle_t votequorum_handle,
121     uint64_t context, votequorum_ring_id_t votequorum_ring_id,
122     uint32_t node_list_entries, uint32_t node_list[])
123 {
124 	struct qdevice_instance *instance;
125 	uint32_t u32;
126 
127 	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
128 		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
129 		exit(1);
130 	}
131 
132 	instance->sync_in_progress = 1;
133 	memcpy(&instance->vq_poll_ring_id, &votequorum_ring_id, sizeof(votequorum_ring_id));
134 
135 	qdevice_log(LOG_DEBUG, "Votequorum nodelist notify callback:");
136 	qdevice_log(LOG_DEBUG, "  Ring_id = ("UTILS_PRI_RING_ID")",
137 	    votequorum_ring_id.nodeid, votequorum_ring_id.seq);
138 
139 	qdevice_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
140 	for (u32 = 0; u32 < node_list_entries; u32++) {
141 		qdevice_log(LOG_DEBUG, "    %"PRIu32" nodeid = "UTILS_PRI_NODE_ID,
142 		    u32, node_list[u32]);
143 	}
144 
145 	if (qdevice_model_votequorum_node_list_notify(instance, votequorum_ring_id, node_list_entries,
146 	    node_list) != 0) {
147 		qdevice_log(LOG_DEBUG, "qdevice_votequorum_node_list_notify_callback returned error -> exit");
148 		exit(2);
149 	}
150 
151 	if (qdevice_heuristics_result_notifier_list_set_active(
152 	    &instance->heuristics_instance.exec_result_notifier_list,
153 	    qdevice_votequorum_heuristics_exec_result_callback, 1) != 0) {
154 		qdevice_log(LOG_CRIT, "Can't activate votequrorum heuristics exec callback notifier");
155 		exit(2);
156 	}
157 
158 	if (qdevice_heuristics_exec(&instance->heuristics_instance, instance->sync_in_progress) != 0) {
159 		qdevice_log(LOG_CRIT, "Can't start heuristics -> exit");
160 		exit(2);
161 	}
162 
163 	instance->vq_node_list_initial_ring_id_set = 1;
164 	memcpy(&instance->vq_node_list_ring_id, &votequorum_ring_id, sizeof(votequorum_ring_id));
165 	instance->vq_node_list_entries = node_list_entries;
166 	free(instance->vq_node_list);
167 	instance->vq_node_list = malloc(sizeof(*node_list) * node_list_entries);
168 	if (instance->vq_node_list == NULL) {
169 		qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory");
170 		exit(1);
171 	}
172 	memcpy(instance->vq_node_list, node_list, sizeof(*node_list) * node_list_entries);
173 }
174 
175 static void
qdevice_votequorum_expected_votes_notify_callback(votequorum_handle_t votequorum_handle,uint64_t context,uint32_t expected_votes)176 qdevice_votequorum_expected_votes_notify_callback(votequorum_handle_t votequorum_handle,
177     uint64_t context, uint32_t expected_votes)
178 {
179 	struct qdevice_instance *instance;
180 
181 	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
182 		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
183 		exit(1);
184 	}
185 
186 	qdevice_log(LOG_DEBUG, "Votequorum expected_votes notify callback:");
187 	qdevice_log(LOG_DEBUG, "  Expected_votes: "UTILS_PRI_EXPECTED_VOTES, expected_votes);
188 
189 	if (qdevice_model_votequorum_expected_votes_notify(instance, expected_votes) != 0) {
190 		qdevice_log(LOG_DEBUG, "qdevice_votequorum_expected_votes_notify_callback returned error -> exit");
191 		exit(2);
192 	}
193 
194 	instance->vq_expected_votes = expected_votes;
195 }
196 
197 void
qdevice_votequorum_init(struct qdevice_instance * instance)198 qdevice_votequorum_init(struct qdevice_instance *instance)
199 {
200 	votequorum_callbacks_t votequorum_callbacks;
201 	votequorum_handle_t votequorum_handle;
202 	cs_error_t res;
203 	int no_retries;
204 	struct votequorum_info vq_info;
205 
206 	memset(&votequorum_callbacks, 0, sizeof(votequorum_callbacks));
207 
208 	votequorum_callbacks.votequorum_quorum_notify_fn =
209 	    qdevice_votequorum_quorum_notify_callback;
210 
211 	votequorum_callbacks.votequorum_nodelist_notify_fn =
212 	    qdevice_votequorum_node_list_notify_callback;
213 
214 	votequorum_callbacks.votequorum_expectedvotes_notify_fn =
215 	    qdevice_votequorum_expected_votes_notify_callback;
216 
217 	no_retries = 0;
218 
219 	while ((res = votequorum_initialize(&votequorum_handle,
220 	    &votequorum_callbacks)) == CS_ERR_TRY_AGAIN &&
221 	    no_retries++ < instance->advanced_settings->max_cs_try_again) {
222 		(void)poll(NULL, 0, 1000);
223 	}
224 
225 	if (res != CS_OK) {
226 		qdevice_log(LOG_CRIT, "Failed to initialize the votequorum API. Error %s", cs_strerror(res));
227 		exit(1);
228 	}
229 
230 	if ((res = votequorum_qdevice_register(votequorum_handle,
231 	    instance->advanced_settings->votequorum_device_name)) != CS_OK) {
232 		qdevice_log(LOG_CRIT, "Can't register votequorum device. Error %s", cs_strerror(res));
233 		exit(1);
234 	}
235 
236 	if ((res = votequorum_context_set(votequorum_handle, (void *)instance)) != CS_OK) {
237 		qdevice_log(LOG_CRIT, "Can't set votequorum context. Error %s", cs_strerror(res));
238 		exit(1);
239 	}
240 
241 	if ((res = votequorum_getinfo(votequorum_handle, VOTEQUORUM_QDEVICE_NODEID,
242 	    &vq_info)) != CS_OK) {
243 		qdevice_log(LOG_CRIT, "Can't get votequorum information. Error %s", cs_strerror(res));
244 		exit(1);
245 	}
246 	instance->vq_expected_votes = vq_info.node_expected_votes;
247 
248 	instance->votequorum_handle = votequorum_handle;
249 
250 	votequorum_fd_get(votequorum_handle, &instance->votequorum_poll_fd);
251 
252 	if ((res = votequorum_trackstart(instance->votequorum_handle, 0,
253 	    CS_TRACK_CHANGES)) != CS_OK) {
254 		qdevice_log(LOG_CRIT, "Can't start tracking votequorum changes. Error %s",
255 		    cs_strerror(res));
256 		exit(1);
257 	}
258 
259 	if (qdevice_heuristics_result_notifier_list_add(&instance->heuristics_instance.exec_result_notifier_list,
260 	    qdevice_votequorum_heuristics_exec_result_callback) == NULL) {
261 		qdevice_log(LOG_CRIT, "Can't add votequrorum heuristics exec callback into notifier");
262 		exit(1);
263 	}
264 }
265 
266 void
qdevice_votequorum_destroy(struct qdevice_instance * instance)267 qdevice_votequorum_destroy(struct qdevice_instance *instance)
268 {
269 	cs_error_t res;
270 
271 	free(instance->vq_quorum_node_list); instance->vq_quorum_node_list = NULL;
272 	free(instance->vq_node_list); instance->vq_node_list = NULL;
273 
274 	res = votequorum_trackstop(instance->votequorum_handle);
275 	if (res != CS_OK) {
276 		qdevice_log(LOG_WARNING, "Can't start tracking votequorum changes. Error %s",
277 		    cs_strerror(res));
278 	}
279 
280 	res = votequorum_qdevice_unregister(instance->votequorum_handle,
281 		instance->advanced_settings->votequorum_device_name);
282 
283 	if (res != CS_OK) {
284 		qdevice_log(LOG_WARNING, "Unable to unregister votequorum device. Error %s", cs_strerror(res));
285 	}
286 
287 	res = votequorum_finalize(instance->votequorum_handle);
288 	if (res != CS_OK) {
289 		qdevice_log(LOG_WARNING, "Unable to finalize votequorum. Error %s", cs_strerror(res));
290 	}
291 }
292 
293 int
qdevice_votequorum_wait_for_ring_id(struct qdevice_instance * instance)294 qdevice_votequorum_wait_for_ring_id(struct qdevice_instance *instance)
295 {
296 	int no_retries;
297 
298 	no_retries = 0;
299 
300 	while (qdevice_votequorum_dispatch(instance) != -1 &&
301 	    no_retries++ < instance->advanced_settings->max_cs_try_again &&
302 	    !instance->vq_node_list_initial_ring_id_set) {
303 		(void)poll(NULL, 0, 1000);
304 	}
305 
306 	if (!instance->vq_node_list_initial_ring_id_set) {
307 		qdevice_log(LOG_CRIT, "Can't get initial votequorum membership information.");
308 		return (-1);
309 	}
310 
311 	return (0);
312 }
313 
314 int
qdevice_votequorum_dispatch(struct qdevice_instance * instance)315 qdevice_votequorum_dispatch(struct qdevice_instance *instance)
316 {
317 	cs_error_t res;
318 
319 	res = votequorum_dispatch(instance->votequorum_handle, CS_DISPATCH_ALL);
320 
321 	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
322 		qdevice_log(LOG_ERR, "Can't dispatch votequorum messages");
323 
324 		return (-1);
325 	}
326 
327 	return (0);
328 }
329 
330 int
qdevice_votequorum_poll(struct qdevice_instance * instance,int cast_vote)331 qdevice_votequorum_poll(struct qdevice_instance *instance, int cast_vote)
332 {
333 	cs_error_t res;
334 
335 	instance->vq_last_poll = time(NULL);
336 	instance->vq_last_poll_cast_vote = cast_vote;
337 
338 	res = votequorum_qdevice_poll(instance->votequorum_handle,
339 	    instance->advanced_settings->votequorum_device_name, cast_vote,
340 	    instance->vq_poll_ring_id);
341 
342 	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
343 		if (res == CS_ERR_MESSAGE_ERROR) {
344 			qdevice_log(LOG_INFO, "qdevice_votequorum_poll called with old ring id");
345 		} else {
346 			qdevice_log(LOG_CRIT, "Can't call votequorum_qdevice_poll. Error %s",
347 			    cs_strerror(res));
348 
349 			return (-1);
350 		}
351 	}
352 
353 	return (0);
354 }
355 
356 int
qdevice_votequorum_master_wins(struct qdevice_instance * instance,int allow)357 qdevice_votequorum_master_wins(struct qdevice_instance *instance, int allow)
358 {
359 	cs_error_t res;
360 	int final_allow;
361 
362 	final_allow = allow;
363 
364 	if (instance->advanced_settings->master_wins ==
365 	    QDEVICE_ADVANCED_SETTINGS_MASTER_WINS_FORCE_OFF && allow) {
366 		qdevice_log(LOG_WARNING, "Allow of master wins is requested, but user forcibly "
367 		    "disallowed it. Keeping master wins disallowed.");
368 
369 		final_allow = 0;
370 	}
371 
372 	if (instance->advanced_settings->master_wins ==
373 	    QDEVICE_ADVANCED_SETTINGS_MASTER_WINS_FORCE_ON && !allow) {
374 		qdevice_log(LOG_WARNING, "Disallow of master wins is requested, but user forcibly "
375 		    "allowed it. Keeping master wins allowed.");
376 
377 		final_allow = 1;
378 	}
379 
380 	res = votequorum_qdevice_master_wins(instance->votequorum_handle,
381 	    instance->advanced_settings->votequorum_device_name, final_allow);
382 
383 	if (res != CS_OK) {
384 		qdevice_log(LOG_CRIT, "Can't set master wins. Error %s", cs_strerror(res));
385 
386 		return (-1);
387 	}
388 
389 	return (0);
390 }
391