1 /* -*-pgsql-c-*- */
2 /*
3 * $Header$
4 *
5 * pgpool: a language independent connection pool server for PostgreSQL
6 * written by Tatsuo Ishii
7 *
8 * Copyright (c) 2003-2018 PgPool Global Development Group
9 *
10 * Permission to use, copy, modify, and distribute this software and
11 * its documentation for any purpose and without fee is hereby
12 * granted, provided that the above copyright notice appear in all
13 * copies and that both that copyright notice and this permission
14 * notice appear in supporting documentation, and that the name of the
15 * author not be used in advertising or publicity pertaining to
16 * distribution of the software without specific, written prior
17 * permission. The author makes no representations about the
18 * suitability of this software for any purpose. It is provided "as
19 * is" without express or implied warranty.
20 *
21 * recovery.c: online recovery process
22 *
23 */
24
25 #include "config.h"
26
27 #include <unistd.h>
28 #include <string.h>
29 #include "utils/elog.h"
30
31 #include "pool.h"
32 #include "pool_config.h"
33
34 #include "libpq-fe.h"
35
36 #include "watchdog/wd_ipc_commands.h"
37
38 #define WAIT_RETRY_COUNT (pool_config->recovery_timeout / 3)
39
40 #define FIRST_STAGE 0
41 #define SECOND_STAGE 1
42
43 static void exec_checkpoint(PGconn *conn);
44 static void exec_recovery(PGconn *conn, BackendInfo * master_backend, BackendInfo * recovery_backend, char stage, int recovery_node);
45 static void exec_remote_start(PGconn *conn, BackendInfo * backend);
46 static PGconn *connect_backend_libpq(BackendInfo * backend);
47 static void check_postmaster_started(BackendInfo * backend);
48
49 static char recovery_command[1024];
50
51 extern volatile sig_atomic_t pcp_worker_wakeup_request;
52
53 /*
54 * Start online recovery.
55 * "recovery_node" is the node to be recovered.
56 * Master or primary node is chosen in this function.
57 */
58 void
start_recovery(int recovery_node)59 start_recovery(int recovery_node)
60 {
61 int node_id;
62 BackendInfo *backend;
63 BackendInfo *recovery_backend;
64 PGconn *conn;
65 int failback_wait_count;
66 #define FAILBACK_WAIT_MAX_RETRY 5 /* 5 seconds should be enough for failback
67 * operation */
68
69 ereport(LOG,
70 (errmsg("starting recovering node %d", recovery_node)));
71
72 if ((recovery_node < 0) || (recovery_node >= pool_config->backend_desc->num_backends))
73 ereport(ERROR,
74 (errmsg("node recovery failed, node id: %d is not valid", recovery_node)));
75
76 if (*(my_backend_status[(recovery_node)]) == CON_UNUSED)
77 ereport(ERROR,
78 (errmsg("node recovery failed, node id: %d is unused", recovery_node)));
79
80 if (VALID_BACKEND(recovery_node))
81 ereport(ERROR,
82 (errmsg("node recovery failed, node id: %d is alive", recovery_node)));
83
84 /* select master/primary node */
85 node_id = MASTER_SLAVE ? PRIMARY_NODE_ID : REAL_MASTER_NODE_ID;
86 backend = &pool_config->backend_desc->backend_info[node_id];
87
88 /* get node info to be recovered */
89 recovery_backend = &pool_config->backend_desc->backend_info[recovery_node];
90
91 conn = connect_backend_libpq(backend);
92 if (conn == NULL)
93 ereport(ERROR,
94 (errmsg("node recovery failed, unable to connect to master node: %d ", node_id)));
95
96 PG_TRY();
97 {
98 /* 1st stage */
99 if (REPLICATION)
100 {
101 exec_checkpoint(conn);
102 ereport(LOG,
103 (errmsg("node recovery, CHECKPOINT in the 1st stage done")));
104 }
105
106 exec_recovery(conn, backend, recovery_backend, FIRST_STAGE, recovery_node);
107
108 ereport(LOG,
109 (errmsg("node recovery, 1st stage is done")));
110
111 if (REPLICATION)
112 {
113 ereport(LOG,
114 (errmsg("node recovery, starting 2nd stage")));
115
116 /* 2nd stage */
117 *InRecovery = RECOVERY_ONLINE;
118 if (pool_config->use_watchdog)
119 {
120 /* announce start recovery */
121 if (COMMAND_OK != wd_start_recovery())
122 ereport(ERROR,
123 (errmsg("node recovery failed, failed to send start recovery packet")));
124 }
125
126 if (wait_connection_closed() != 0)
127 ereport(ERROR,
128 (errmsg("node recovery failed, waiting connection closed in the other pgpools timeout")));
129
130 ereport(LOG,
131 (errmsg("node recovery, all connections from clients have been closed")));
132
133 exec_checkpoint(conn);
134
135 ereport(LOG,
136 (errmsg("node recovery"),
137 errdetail("CHECKPOINT in the 2nd stage done")));
138
139 exec_recovery(conn, backend, recovery_backend, SECOND_STAGE, recovery_node);
140 }
141
142 exec_remote_start(conn, recovery_backend);
143
144 check_postmaster_started(recovery_backend);
145
146 ereport(LOG,
147 (errmsg("node recovery, node: %d restarted", recovery_node)));
148
149 /*
150 * reset failover completion flag. this is necessary since previous
151 * failover/failback will set the flag to 1.
152 */
153 pcp_worker_wakeup_request = 0;
154
155 /* send failback request to pgpool parent */
156 send_failback_request(recovery_node, false, REQ_DETAIL_CONFIRMED);
157
158 /* wait for failback */
159 failback_wait_count = 0;
160 while (!pcp_worker_wakeup_request)
161 {
162 struct timeval t = {1, 0};
163
164 /* polling SIGUSR2 signal every 1 sec */
165 select(0, NULL, NULL, NULL, &t);
166 failback_wait_count++;
167 if (failback_wait_count >= FAILBACK_WAIT_MAX_RETRY)
168 {
169 ereport(LOG,
170 (errmsg("node recovery"),
171 errdetail("waiting for wake up request is timeout(%d seconds)",
172 FAILBACK_WAIT_MAX_RETRY)));
173
174 break;
175 }
176 }
177 pcp_worker_wakeup_request = 0;
178 }
179 PG_CATCH();
180 {
181 PQfinish(conn);
182 PG_RE_THROW();
183 }
184 PG_END_TRY();
185
186 PQfinish(conn);
187
188 ereport(LOG,
189 (errmsg("recovery done")));
190 }
191
192 /*
193 * Notice all children finishing recovery.
194 */
195 void
finish_recovery(void)196 finish_recovery(void)
197 {
198 /* announce end recovery */
199 if (pool_config->use_watchdog && *InRecovery != RECOVERY_INIT)
200 {
201 wd_end_recovery();
202 }
203
204 *InRecovery = RECOVERY_INIT;
205 pool_signal_parent(SIGUSR2);
206 }
207
208 /*
209 * Execute CHECKPOINT
210 */
211 static void
exec_checkpoint(PGconn * conn)212 exec_checkpoint(PGconn *conn)
213 {
214 PGresult *result;
215
216 ereport(DEBUG1,
217 (errmsg("recovery execute checkpoint, start checkpoint")));
218
219 result = PQexec(conn, "CHECKPOINT");
220 if (PQresultStatus(result) != PGRES_COMMAND_OK)
221 ereport(ERROR,
222 (errmsg("executing recovery, execute CHECKPOINT failed")));
223 PQclear(result);
224
225 ereport(DEBUG1,
226 (errmsg("recovery execute checkpoint, finish checkpoint")));
227 }
228
229 /*
230 * Call pgpool_recovery() function.
231 */
232 static void
exec_recovery(PGconn * conn,BackendInfo * master_backend,BackendInfo * recovery_backend,char stage,int recovery_node)233 exec_recovery(PGconn *conn, BackendInfo * master_backend, BackendInfo * recovery_backend, char stage, int recovery_node)
234 {
235 PGresult *result;
236 char *hostname;
237 char *script;
238
239 if (strlen(recovery_backend->backend_hostname) == 0 || *(recovery_backend->backend_hostname) == '/')
240 hostname = "localhost";
241 else
242 hostname = recovery_backend->backend_hostname;
243
244 script = (stage == FIRST_STAGE) ?
245 pool_config->recovery_1st_stage_command : pool_config->recovery_2nd_stage_command;
246
247 if (script == NULL || strlen(script) == 0)
248 {
249 /* do not execute script */
250 return;
251 }
252
253 /*
254 * Execute recovery command
255 */
256 snprintf(recovery_command,
257 sizeof(recovery_command),
258 "SELECT pgpool_recovery('%s', '%s', '%s', '%d', %d)",
259 script,
260 hostname,
261 recovery_backend->backend_data_directory,
262 master_backend->backend_port,
263 recovery_node
264 );
265
266 ereport(LOG,
267 (errmsg("executing recovery"),
268 errdetail("starting recovery command: \"%s\"", recovery_command)));
269
270 ereport(LOG,
271 (errmsg("executing recovery"),
272 errdetail("disabling statement_timeout")));
273
274 result = PQexec(conn, "SET statement_timeout To 0");
275 if (PQresultStatus(result) != PGRES_COMMAND_OK)
276 ereport(ERROR,
277 (errmsg("executing recovery, SET STATEMENT_TIMEOUT failed at \"%s\"",
278 (stage == FIRST_STAGE) ? "1st stage" : "2nd stage")));
279
280 PQclear(result);
281
282 ereport(DEBUG1,
283 (errmsg("executing recovery, start recovery")));
284
285 result = PQexec(conn, recovery_command);
286 if (PQresultStatus(result) != PGRES_TUPLES_OK)
287 ereport(ERROR,
288 (errmsg("executing recovery, execution of command failed at \"%s\"",
289 (stage == FIRST_STAGE) ? "1st stage" : "2nd stage"),
290 errdetail("command:\"%s\"", script)));
291
292 PQclear(result);
293
294 ereport(DEBUG1,
295 (errmsg("executing recovery, finish recovery")));
296 }
297
298 /*
299 * Call pgpool_remote_start() function.
300 */
301 static void
exec_remote_start(PGconn * conn,BackendInfo * backend)302 exec_remote_start(PGconn *conn, BackendInfo * backend)
303 {
304 PGresult *result;
305 char *hostname;
306
307 if (strlen(backend->backend_hostname) == 0 || *(backend->backend_hostname) == '/')
308 hostname = "localhost";
309 else
310 hostname = backend->backend_hostname;
311
312 snprintf(recovery_command, sizeof(recovery_command),
313 "SELECT pgpool_remote_start('%s', '%s')",
314 hostname,
315 backend->backend_data_directory);
316
317 ereport(DEBUG1,
318 (errmsg("executing remote start"),
319 errdetail("start pgpool_remote_start")));
320
321 result = PQexec(conn, recovery_command);
322 if (PQresultStatus(result) != PGRES_TUPLES_OK)
323 ereport(ERROR,
324 (errmsg("executing remote start failed with error: \"%s\"", PQresultErrorMessage(result))));
325
326 PQclear(result);
327
328 ereport(DEBUG1,
329 (errmsg("executing remote start"),
330 errdetail("finish pgpool_remote_start")));
331 }
332
333 /*
334 * Check postmaster is started.
335 */
336 static void
check_postmaster_started(BackendInfo * backend)337 check_postmaster_started(BackendInfo * backend)
338 {
339 int i = 0;
340 char port_str[16];
341 PGconn *conn;
342 char *dbname;
343 char *password = get_pgpool_config_user_password(pool_config->recovery_user,
344 pool_config->recovery_password);
345
346 snprintf(port_str, sizeof(port_str), "%d", backend->backend_port);
347
348 /*
349 * First we try with "postgres" database.
350 */
351 dbname = "postgres";
352
353 do
354 {
355 ConnStatusType r;
356
357 ereport(LOG,
358 (errmsg("checking if postmaster is started"),
359 errdetail("trying to connect to postmaster on hostname:%s database:%s user:%s (retry %d times)",
360 backend->backend_hostname, dbname, pool_config->recovery_user, i)));
361
362 conn = PQsetdbLogin(backend->backend_hostname,
363 port_str,
364 NULL,
365 NULL,
366 dbname,
367 pool_config->recovery_user,
368 password ? password : NULL);
369
370 r = PQstatus(conn);
371 PQfinish(conn);
372 if (r == CONNECTION_OK)
373 {
374 if (password)
375 pfree(password);
376 return;
377 }
378 ereport(LOG,
379 (errmsg("checking if postmaster is started"),
380 errdetail("failed to connect to postmaster on hostname:%s database:%s user:%s",
381 backend->backend_hostname, dbname, pool_config->recovery_user)));
382
383 sleep(3);
384 } while (i++ < 3); /* XXX Hard coded retry (9 seconds) */
385
386 /*
387 * Retry with "template1" database.
388 */
389 dbname = "template1";
390 i = 0;
391
392 do
393 {
394 ConnStatusType r;
395
396 ereport(LOG,
397 (errmsg("checking if postmaster is started"),
398 errdetail("trying to connect to postmaster on hostname:%s database:%s user:%s (retry %d times)",
399 backend->backend_hostname, dbname, pool_config->recovery_user, i)));
400
401 conn = PQsetdbLogin(backend->backend_hostname,
402 port_str,
403 NULL,
404 NULL,
405 dbname,
406 pool_config->recovery_user,
407 password ? password : NULL);
408
409 r = PQstatus(conn);
410 PQfinish(conn);
411 if (r == CONNECTION_OK)
412 {
413 if (password)
414 pfree(password);
415 return;
416 }
417
418 ereport(LOG,
419 (errmsg("checking if postmaster is started"),
420 errdetail("failed to connect to postmaster on hostname:%s database:%s user:%s",
421 backend->backend_hostname, dbname, pool_config->recovery_user)));
422
423 if (WAIT_RETRY_COUNT != 0)
424 sleep(3);
425 } while (i++ < WAIT_RETRY_COUNT);
426
427 if (password)
428 pfree(password);
429
430 ereport(ERROR,
431 (errmsg("recovery is checking if postmaster is started"),
432 errdetail("postmaster on hostname:\"%s\" database:\"%s\" user:\"%s\" failed to start in %d second",
433 backend->backend_hostname, dbname, pool_config->recovery_user, pool_config->recovery_timeout)));
434 }
435
436 static PGconn *
connect_backend_libpq(BackendInfo * backend)437 connect_backend_libpq(BackendInfo * backend)
438 {
439 char port_str[16];
440 PGconn *conn;
441 char *password = get_pgpool_config_user_password(pool_config->recovery_user,
442 pool_config->recovery_password);
443
444 snprintf(port_str, sizeof(port_str),
445 "%d", backend->backend_port);
446 conn = PQsetdbLogin(backend->backend_hostname,
447 port_str,
448 NULL,
449 NULL,
450 "template1",
451 pool_config->recovery_user,
452 password ? password : "");
453
454 if (password)
455 pfree(password);
456
457 if (PQstatus(conn) != CONNECTION_OK)
458 {
459 PQfinish(conn);
460 return NULL;
461 }
462 return conn;
463 }
464
465 /*
466 * Wait all connections are closed.
467 */
468 int
wait_connection_closed(void)469 wait_connection_closed(void)
470 {
471 int i = 0;
472
473 do
474 {
475
476 if (Req_info->conn_counter == 0)
477 return 0;
478
479 if (WAIT_RETRY_COUNT != 0)
480 sleep(3);
481
482 } while (i++ < WAIT_RETRY_COUNT);
483 ereport(LOG,
484 (errmsg("wait_connection_closed: existing connections did not close in %d sec.", pool_config->recovery_timeout)));
485 return ensure_conn_counter_validity();
486 }
487
ensure_conn_counter_validity(void)488 int ensure_conn_counter_validity(void)
489 {
490 /*
491 * recovery_timeout was expired. Before returning with failure status,
492 * let's check if this is caused by the malformed conn_counter. If a child
493 * process abnormally exits (killed by SIGKILL or SEGFAULT, for example),
494 * then conn_counter is not decremented at process exit, thus it will
495 * never be returning to 0. This could be detected by checking if
496 * client_idle_limit_in_recovery is enabled and less value than
497 * recovery_timeout because all clients must be kicked out by the time
498 * when client_idle_limit_in_recovery is expired. If so, we should reset
499 * conn_counter to 0 also.
500 *
501 * See bug 431 for more info.
502 */
503 if (pool_config->client_idle_limit_in_recovery == -1 ||
504 (pool_config->client_idle_limit_in_recovery > 0 &&
505 pool_config->recovery_timeout >= pool_config->client_idle_limit_in_recovery))
506 {
507 ereport(LOG,
508 (errmsg("wait_connection_closed: mulformed conn_counter (%d) detected. reset it to 0",
509 Req_info->conn_counter)));
510 Req_info->conn_counter = 0;
511 return 0;
512 }
513 return 1;
514 }
515