1 /***********************************************************************************************************************************
2 Restore Command
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include <string.h>
7 #include <sys/stat.h>
8 #include <time.h>
9 #include <unistd.h>
10
11 #include "command/restore/protocol.h"
12 #include "command/restore/restore.h"
13 #include "common/crypto/cipherBlock.h"
14 #include "common/debug.h"
15 #include "common/log.h"
16 #include "common/regExp.h"
17 #include "common/user.h"
18 #include "config/config.h"
19 #include "config/exec.h"
20 #include "info/infoBackup.h"
21 #include "info/manifest.h"
22 #include "postgres/interface.h"
23 #include "postgres/version.h"
24 #include "protocol/helper.h"
25 #include "protocol/parallel.h"
26 #include "storage/helper.h"
27 #include "storage/write.intern.h"
28 #include "version.h"
29
30 /***********************************************************************************************************************************
31 Recovery constants
32 ***********************************************************************************************************************************/
33 #define RESTORE_COMMAND "restore_command"
34 STRING_STATIC(RESTORE_COMMAND_STR, RESTORE_COMMAND);
35
36 #define RECOVERY_TARGET "recovery_target"
37 #define RECOVERY_TARGET_LSN "recovery_target_lsn"
38 #define RECOVERY_TARGET_NAME "recovery_target_name"
39 #define RECOVERY_TARGET_TIME "recovery_target_time"
40 #define RECOVERY_TARGET_XID "recovery_target_xid"
41
42 #define RECOVERY_TARGET_ACTION "recovery_target_action"
43
44 #define RECOVERY_TARGET_INCLUSIVE "recovery_target_inclusive"
45 #define RECOVERY_TARGET_TIMELINE "recovery_target_timeline"
46 #define PAUSE_AT_RECOVERY_TARGET "pause_at_recovery_target"
47 #define STANDBY_MODE "standby_mode"
48 STRING_STATIC(STANDBY_MODE_STR, STANDBY_MODE);
49
50 #define ARCHIVE_MODE "archive_mode"
51
52 /***********************************************************************************************************************************
53 Validate restore path
54 ***********************************************************************************************************************************/
55 static void
restorePathValidate(void)56 restorePathValidate(void)
57 {
58 FUNCTION_LOG_VOID(logLevelDebug);
59
60 MEM_CONTEXT_TEMP_BEGIN()
61 {
62 // PostgreSQL must not be running
63 if (storageExistsP(storagePg(), PG_FILE_POSTMASTERPID_STR))
64 {
65 THROW_FMT(
66 PgRunningError,
67 "unable to restore while PostgreSQL is running\n"
68 "HINT: presence of '" PG_FILE_POSTMASTERPID "' in '%s' indicates PostgreSQL is running.\n"
69 "HINT: remove '" PG_FILE_POSTMASTERPID "' only if PostgreSQL is not running.",
70 strZ(cfgOptionDisplay(cfgOptPgPath)));
71 }
72
73 // If the restore will be destructive attempt to verify that PGDATA looks like a valid PostgreSQL directory
74 if ((cfgOptionBool(cfgOptDelta) || cfgOptionBool(cfgOptForce)) &&
75 !storageExistsP(storagePg(), PG_FILE_PGVERSION_STR) && !storageExistsP(storagePg(), BACKUP_MANIFEST_FILE_STR))
76 {
77 LOG_WARN_FMT(
78 "--delta or --force specified but unable to find '" PG_FILE_PGVERSION "' or '" BACKUP_MANIFEST_FILE "' in '%s' to"
79 " confirm that this is a valid $PGDATA directory. --delta and --force have been disabled and if any files"
80 " exist in the destination directories the restore will be aborted.",
81 strZ(cfgOptionDisplay(cfgOptPgPath)));
82
83 // Disable delta and force so restore will fail if the directories are not empty
84 cfgOptionSet(cfgOptDelta, cfgSourceDefault, VARBOOL(false));
85 cfgOptionSet(cfgOptForce, cfgSourceDefault, VARBOOL(false));
86 }
87 }
88 MEM_CONTEXT_TEMP_END();
89
90 FUNCTION_LOG_RETURN_VOID();
91 }
92
93 /***********************************************************************************************************************************
94 Get epoch time from formatted string
95 ***********************************************************************************************************************************/
96 static time_t
getEpoch(const String * targetTime)97 getEpoch(const String *targetTime)
98 {
99 FUNCTION_LOG_BEGIN(logLevelDebug);
100 FUNCTION_LOG_PARAM(STRING, targetTime);
101 FUNCTION_LOG_END();
102
103 ASSERT(targetTime != NULL);
104
105 time_t result = 0;
106
107 MEM_CONTEXT_TEMP_BEGIN()
108 {
109 // Build the regex to accept formats: YYYY-MM-DD HH:MM:SS with optional msec (up to 6 digits and separated from minutes by
110 // a comma or period), optional timezone offset +/- HH or HHMM or HH:MM, where offset boundaries are UTC-12 to UTC+14
111 const String *expression = STRDEF(
112 "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}((\\,|\\.)[0-9]{1,6})?((\\+|\\-)[0-9]{2}(:?)([0-9]{2})?)?$");
113
114 RegExp *regExp = regExpNew(expression);
115
116 // If the target-recovery time matches the regular expression then validate it
117 if (regExpMatch(regExp, targetTime))
118 {
119 // Strip off the date and time and put the remainder into another string
120 String *datetime = strSubN(targetTime, 0, 19);
121
122 int dtYear = cvtZToInt(strZ(strSubN(datetime, 0, 4)));
123 int dtMonth = cvtZToInt(strZ(strSubN(datetime, 5, 2)));
124 int dtDay = cvtZToInt(strZ(strSubN(datetime, 8, 2)));
125 int dtHour = cvtZToInt(strZ(strSubN(datetime, 11, 2)));
126 int dtMinute = cvtZToInt(strZ(strSubN(datetime, 14, 2)));
127 int dtSecond = cvtZToInt(strZ(strSubN(datetime, 17, 2)));
128
129 // Confirm date and time parts are valid
130 datePartsValid(dtYear, dtMonth, dtDay);
131 timePartsValid(dtHour, dtMinute, dtSecond);
132
133 String *timeTargetZone = strSub(targetTime, 19);
134
135 // Find the + or - indicating a timezone offset was provided (there may be milliseconds before the timezone, so need to
136 // skip). If a timezone offset was not provided, then local time is assumed.
137 int idxSign = strChr(timeTargetZone, '+');
138
139 if (idxSign == -1)
140 idxSign = strChr(timeTargetZone, '-');
141
142 if (idxSign != -1)
143 {
144 String *timezoneOffset = strSub(timeTargetZone, (size_t)idxSign);
145
146 // Include the sign with the hour
147 int tzHour = cvtZToInt(strZ(strSubN(timezoneOffset, 0, 3)));
148 int tzMinute = 0;
149
150 // If minutes are included in timezone offset then extract the minutes based on whether a colon separates them from
151 // the hour
152 if (strSize(timezoneOffset) > 3)
153 tzMinute = cvtZToInt(strZ(strSubN(timezoneOffset, 3 + (strChr(timezoneOffset, ':') == -1 ? 0 : 1), 2)));
154
155 result = epochFromParts(dtYear, dtMonth, dtDay, dtHour, dtMinute, dtSecond, tzOffsetSeconds(tzHour, tzMinute));
156 }
157 // If there is no timezone offset, then assume it is local time
158 else
159 {
160 // Set tm_isdst to -1 to force mktime to consider if DST. For example, if system time is America/New_York then
161 // 2019-09-14 20:02:49 was a time in DST so the Epoch value should be 1568505769 (and not 1568509369 which would be
162 // 2019-09-14 21:02:49 - an hour too late)
163 result = mktime(
164 &(struct tm){.tm_sec = dtSecond, .tm_min = dtMinute, .tm_hour = dtHour, .tm_mday = dtDay, .tm_mon = dtMonth - 1,
165 .tm_year = dtYear - 1900, .tm_isdst = -1});
166 }
167 }
168 else
169 {
170 LOG_WARN_FMT(
171 "automatic backup set selection cannot be performed with provided time '%s', latest backup set will be used"
172 "\nHINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone (+/- HH or HHMM or HH:MM)"
173 " - if timezone is omitted, local time is assumed (for UTC use +00)",
174 strZ(targetTime));
175 }
176 }
177 MEM_CONTEXT_TEMP_END();
178
179 FUNCTION_LOG_RETURN(TIME, result);
180 }
181
182 /***********************************************************************************************************************************
183 Get the backup set to restore
184 ***********************************************************************************************************************************/
185 typedef struct RestoreBackupData
186 {
187 unsigned int repoIdx; // Internal repo index
188 CipherType repoCipherType; // Repo encryption type (0 = none)
189 const String *backupCipherPass; // Passphrase of backup files if repo is encrypted (else NULL)
190 const String *backupSet; // Backup set to restore
191 } RestoreBackupData;
192
193 #define FUNCTION_LOG_RESTORE_BACKUP_DATA_TYPE \
194 RestoreBackupData
195 #define FUNCTION_LOG_RESTORE_BACKUP_DATA_FORMAT(value, buffer, bufferSize) \
196 objToLog(&value, "RestoreBackupData", buffer, bufferSize)
197
198 // Helper function for restoreBackupSet
199 static RestoreBackupData
restoreBackupData(const String * backupLabel,unsigned int repoIdx,const String * backupCipherPass)200 restoreBackupData(const String *backupLabel, unsigned int repoIdx, const String *backupCipherPass)
201 {
202 ASSERT(backupLabel != NULL);
203
204 RestoreBackupData restoreBackup = {0};
205
206 MEM_CONTEXT_PRIOR_BEGIN()
207 {
208 restoreBackup.backupSet = strDup(backupLabel);
209 restoreBackup.repoIdx = repoIdx;
210 restoreBackup.repoCipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx);
211 restoreBackup.backupCipherPass = strDup(backupCipherPass);
212 }
213 MEM_CONTEXT_PRIOR_END();
214
215 return restoreBackup;
216 }
217
218 static RestoreBackupData
restoreBackupSet(void)219 restoreBackupSet(void)
220 {
221 FUNCTION_LOG_VOID(logLevelDebug);
222
223 RestoreBackupData result = {0};
224
225 MEM_CONTEXT_TEMP_BEGIN()
226 {
227 // Initialize the repo index
228 unsigned int repoIdxMin = 0;
229 unsigned int repoIdxMax = cfgOptionGroupIdxTotal(cfgOptGrpRepo) - 1;
230
231 // If the repo was specified then set index to the array location and max to loop only once
232 if (cfgOptionTest(cfgOptRepo))
233 {
234 repoIdxMin = cfgOptionGroupIdxDefault(cfgOptGrpRepo);
235 repoIdxMax = repoIdxMin;
236 }
237
238 // Initialize a backup candidate list
239 List *backupCandidateList = lstNewP(sizeof(RestoreBackupData));
240
241 const String *backupSetRequested = NULL;
242 time_t timeTargetEpoch = 0;
243
244 // If the set option was not provided by the user but a time to recover was set, then we will need to search for a backup
245 // set that satisfies the time condition, else we will use the backup provided
246 if (cfgOptionSource(cfgOptSet) == cfgSourceDefault)
247 {
248 if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_TIME)
249 timeTargetEpoch = getEpoch(cfgOptionStr(cfgOptTarget));
250 }
251 else
252 backupSetRequested = cfgOptionStr(cfgOptSet);
253
254 // Search through the repo list for a backup set to use for recovery
255 for (unsigned int repoIdx = repoIdxMin; repoIdx <= repoIdxMax; repoIdx++)
256 {
257 // Get the repo storage in case it is remote and encryption settings need to be pulled down
258 storageRepoIdx(repoIdx);
259
260 InfoBackup *infoBackup = NULL;
261
262 // Attempt to load backup.info
263 TRY_BEGIN()
264 {
265 infoBackup = infoBackupLoadFile(
266 storageRepoIdx(repoIdx), INFO_BACKUP_PATH_FILE_STR, cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx),
267 cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));
268 }
269 CATCH_ANY()
270 {
271 LOG_WARN_FMT(
272 "repo%u: [%s] %s", cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), errorTypeName(errorType()), errorMessage());
273 }
274 TRY_END();
275
276 // If unable to load the backup info file, then move on to next repo
277 if (infoBackup == NULL)
278 continue;
279
280 if (infoBackupDataTotal(infoBackup) == 0)
281 {
282 LOG_WARN_FMT(
283 "repo%u: [%s] no backup sets to restore", cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx),
284 errorTypeName(&BackupSetInvalidError));
285 continue;
286 }
287
288 // If a backup set was not specified, then see if a time to recover was requested
289 if (backupSetRequested == NULL)
290 {
291 // Get the latest backup
292 InfoBackupData latestBackup = infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1);
293
294 // If the recovery type is time, attempt to determine the backup set
295 if (timeTargetEpoch != 0)
296 {
297 bool found = false;
298
299 // Search current backups from newest to oldest
300 for (unsigned int keyIdx = infoBackupDataTotal(infoBackup) - 1; (int)keyIdx >= 0; keyIdx--)
301 {
302 // Get the backup data
303 InfoBackupData backupData = infoBackupData(infoBackup, keyIdx);
304
305 // If the end of the backup is before the target time, then select this backup
306 if (backupData.backupTimestampStop < timeTargetEpoch)
307 {
308 found = true;
309
310 result = restoreBackupData(
311 backupData.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
312 break;
313 }
314 }
315
316 // If a backup was found on this repo matching the criteria for time then exit, else determine if the latest
317 // backup from this repo might be used
318 if (found)
319 break;
320 else
321 {
322 // If a backup was not yet found then set the latest from this repo as the backup that might be used
323 RestoreBackupData candidate = restoreBackupData(
324 latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
325
326 lstAdd(backupCandidateList, &candidate);
327 }
328 }
329 else
330 {
331 // If the recovery type was not time (or time provided was not valid), then use the latest backup from this repo
332 result = restoreBackupData(latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
333 break;
334 }
335 }
336 // Otherwise check to see if the specified backup set is on this repo
337 else
338 {
339 for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++)
340 {
341 if (strEq(infoBackupData(infoBackup, backupIdx).backupLabel, backupSetRequested))
342 {
343 result = restoreBackupData(backupSetRequested, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
344 break;
345 }
346 }
347
348 // If the backup set is found, then exit, else continue to next repo
349 if (result.backupSet != NULL)
350 break;
351 }
352 }
353
354 // Still no backup set to use after checking all the repos required to be checked?
355 if (result.backupSet == NULL)
356 {
357 if (backupSetRequested != NULL)
358 THROW_FMT(BackupSetInvalidError, "backup set %s is not valid", strZ(backupSetRequested));
359 else if (timeTargetEpoch != 0 && lstSize(backupCandidateList) > 0)
360 {
361 // Since the repos were scanned in priority order, use the first candidate found
362 result = restoreBackupData(
363 ((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupSet,
364 ((RestoreBackupData *)lstGet(backupCandidateList, 0))->repoIdx,
365 ((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupCipherPass);
366
367 LOG_WARN_FMT(
368 "unable to find backup set with stop time less than '%s', repo%u: latest backup set will be used",
369 strZ(cfgOptionDisplay(cfgOptTarget)), cfgOptionGroupIdxToKey(cfgOptGrpRepo, result.repoIdx));
370 }
371 else
372 THROW(BackupSetInvalidError, "no backup set found to restore");
373 }
374 }
375 MEM_CONTEXT_TEMP_END();
376
377 FUNCTION_LOG_RETURN_STRUCT(result);
378 }
379
380 /***********************************************************************************************************************************
381 Validate the manifest
382 ***********************************************************************************************************************************/
383 static void
restoreManifestValidate(Manifest * manifest,const String * backupSet)384 restoreManifestValidate(Manifest *manifest, const String *backupSet)
385 {
386 FUNCTION_LOG_BEGIN(logLevelDebug);
387 FUNCTION_LOG_PARAM(MANIFEST, manifest);
388 FUNCTION_LOG_PARAM(STRING, backupSet);
389 FUNCTION_LOG_END();
390
391 ASSERT(manifest != NULL);
392 ASSERT(backupSet != NULL);
393
394 MEM_CONTEXT_TEMP_BEGIN()
395 {
396 // If there are no files in the manifest then something has gone horribly wrong
397 CHECK(manifestFileTotal(manifest) > 0);
398
399 // Sanity check to ensure the manifest has not been moved to a new directory
400 const ManifestData *data = manifestData(manifest);
401
402 if (!strEq(data->backupLabel, backupSet))
403 {
404 THROW_FMT(
405 FormatError,
406 "requested backup '%s' and manifest label '%s' do not match\n"
407 "HINT: this indicates some sort of corruption (at the very least paths have been renamed).",
408 strZ(backupSet), strZ(data->backupLabel));
409 }
410 }
411 MEM_CONTEXT_TEMP_END();
412
413 FUNCTION_LOG_RETURN_VOID();
414 }
415
416 /***********************************************************************************************************************************
417 Remap the manifest based on mappings provided by the user
418 ***********************************************************************************************************************************/
419 static void
restoreManifestMap(Manifest * manifest)420 restoreManifestMap(Manifest *manifest)
421 {
422 FUNCTION_LOG_BEGIN(logLevelDebug);
423 FUNCTION_LOG_PARAM(MANIFEST, manifest);
424 FUNCTION_LOG_END();
425
426 ASSERT(manifest != NULL);
427
428 MEM_CONTEXT_TEMP_BEGIN()
429 {
430 // Remap the data directory
431 // -------------------------------------------------------------------------------------------------------------------------
432 const String *pgPath = cfgOptionStr(cfgOptPgPath);
433 const ManifestTarget *targetBase = manifestTargetBase(manifest);
434
435 if (!strEq(targetBase->path, pgPath))
436 {
437 LOG_INFO_FMT("remap data directory to '%s'", strZ(pgPath));
438 manifestTargetUpdate(manifest, targetBase->name, pgPath, NULL);
439 }
440
441 // Remap tablespaces
442 // -------------------------------------------------------------------------------------------------------------------------
443 KeyValue *tablespaceMap = varKv(cfgOption(cfgOptTablespaceMap));
444 const String *tablespaceMapAllPath = cfgOptionStrNull(cfgOptTablespaceMapAll);
445
446 if (tablespaceMap != NULL || tablespaceMapAllPath != NULL)
447 {
448 StringList *tablespaceRemapped = strLstNew();
449
450 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
451 {
452 const ManifestTarget *target = manifestTarget(manifest, targetIdx);
453
454 // Is this a tablespace?
455 if (target->tablespaceId != 0)
456 {
457 const String *tablespacePath = NULL;
458
459 // Check for an individual mapping for this tablespace
460 if (tablespaceMap != NULL)
461 {
462 // Attempt to get the tablespace by name
463 const String *tablespacePathByName = varStr(kvGet(tablespaceMap, VARSTR(target->tablespaceName)));
464
465 if (tablespacePathByName != NULL)
466 strLstAdd(tablespaceRemapped, target->tablespaceName);
467
468 // Attempt to get the tablespace by id
469 const String *tablespacePathById = varStr(
470 kvGet(tablespaceMap, VARSTR(varStrForce(VARUINT(target->tablespaceId)))));
471
472 if (tablespacePathById != NULL)
473 strLstAdd(tablespaceRemapped, varStrForce(VARUINT(target->tablespaceId)));
474
475 // Error when both are set but the paths are different
476 if (tablespacePathByName != NULL && tablespacePathById != NULL && !
477 strEq(tablespacePathByName, tablespacePathById))
478 {
479 THROW_FMT(
480 TablespaceMapError, "tablespace remapped by name '%s' and id %u with different paths",
481 strZ(target->tablespaceName), target->tablespaceId);
482 }
483 // Else set the path by name
484 else if (tablespacePathByName != NULL)
485 {
486 tablespacePath = tablespacePathByName;
487 }
488 // Else set the path by id
489 else if (tablespacePathById != NULL)
490 tablespacePath = tablespacePathById;
491 }
492
493 // If not individual mapping check if all tablespaces are being remapped
494 if (tablespacePath == NULL && tablespaceMapAllPath != NULL)
495 tablespacePath = strNewFmt("%s/%s", strZ(tablespaceMapAllPath), strZ(target->tablespaceName));
496
497 // Remap tablespace if a mapping was found
498 if (tablespacePath != NULL)
499 {
500 LOG_INFO_FMT("map tablespace '%s' to '%s'", strZ(target->name), strZ(tablespacePath));
501
502 manifestTargetUpdate(manifest, target->name, tablespacePath, NULL);
503 manifestLinkUpdate(manifest, strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(target->name)), tablespacePath);
504 }
505 }
506 }
507
508 // Error on invalid tablespaces
509 if (tablespaceMap != NULL)
510 {
511 const VariantList *tablespaceMapList = kvKeyList(tablespaceMap);
512 strLstSort(tablespaceRemapped, sortOrderAsc);
513
514 for (unsigned int tablespaceMapIdx = 0; tablespaceMapIdx < varLstSize(tablespaceMapList); tablespaceMapIdx++)
515 {
516 const String *tablespace = varStr(varLstGet(tablespaceMapList, tablespaceMapIdx));
517
518 if (!strLstExists(tablespaceRemapped, tablespace))
519 THROW_FMT(TablespaceMapError, "unable to remap invalid tablespace '%s'", strZ(tablespace));
520 }
521 }
522
523 // Issue a warning message when remapping tablespaces in PostgreSQL < 9.2
524 if (manifestData(manifest)->pgVersion <= PG_VERSION_92)
525 LOG_WARN("update pg_tablespace.spclocation with new tablespace locations for PostgreSQL <= " PG_VERSION_92_STR);
526 }
527
528 // Remap links
529 // -------------------------------------------------------------------------------------------------------------------------
530 KeyValue *linkMap = varKv(cfgOption(cfgOptLinkMap));
531 bool linkAll = cfgOptionBool(cfgOptLinkAll);
532
533 StringList *linkRemapped = strLstNew();
534
535 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
536 {
537 const ManifestTarget *target = manifestTarget(manifest, targetIdx);
538
539 // Is this a link?
540 if (target->type == manifestTargetTypeLink && target->tablespaceId == 0)
541 {
542 const String *link = strSub(target->name, strSize(MANIFEST_TARGET_PGDATA_STR) + 1);
543 const String *linkPath = linkMap == NULL ? NULL : varStr(kvGet(linkMap, VARSTR(link)));
544
545 // Remap link if a mapping was found
546 if (linkPath != NULL)
547 {
548 LOG_INFO_FMT("map link '%s' to '%s'", strZ(link), strZ(linkPath));
549 manifestLinkUpdate(manifest, target->name, linkPath);
550
551 // If the link is a file separate the file name from the path to update the target
552 const String *linkFile = NULL;
553
554 if (target->file != NULL)
555 {
556 // The link destination must have at least one path component in addition to the file part. So '..' would
557 // not be a valid destination but '../file' or '/file' is.
558 if (strSize(strPath(linkPath)) == 0)
559 {
560 THROW_FMT(
561 LinkMapError, "'%s' is not long enough to be the destination for file link '%s'", strZ(linkPath),
562 strZ(link));
563 }
564
565 linkFile = strBase(linkPath);
566 linkPath = strPath(linkPath);
567 }
568
569 manifestTargetUpdate(manifest, target->name, linkPath, linkFile);
570
571 // Add to remapped list for later validation that all links were valid
572 strLstAdd(linkRemapped, link);
573 }
574 // If all links are not being restored then remove the target and link
575 else if (!linkAll)
576 {
577 if (target->file != NULL)
578 LOG_WARN_FMT("file link '%s' will be restored as a file at the same location", strZ(link));
579 else
580 {
581 LOG_WARN_FMT(
582 "contents of directory link '%s' will be restored in a directory at the same location", strZ(link));
583 }
584
585 manifestLinkRemove(manifest, target->name);
586 manifestTargetRemove(manifest, target->name);
587 targetIdx--;
588 }
589 }
590 }
591
592 // Error on invalid links
593 if (linkMap != NULL)
594 {
595 const VariantList *linkMapList = kvKeyList(linkMap);
596 strLstSort(linkRemapped, sortOrderAsc);
597
598 for (unsigned int linkMapIdx = 0; linkMapIdx < varLstSize(linkMapList); linkMapIdx++)
599 {
600 const String *link = varStr(varLstGet(linkMapList, linkMapIdx));
601
602 if (!strLstExists(linkRemapped, link))
603 THROW_FMT(LinkMapError, "unable to remap invalid link '%s'", strZ(link));
604 }
605 }
606 }
607 MEM_CONTEXT_TEMP_END();
608
609 FUNCTION_LOG_RETURN_VOID();
610 }
611
612 /***********************************************************************************************************************************
613 Check ownership of items in the manifest
614 ***********************************************************************************************************************************/
615 // Helper to get list of owners from a file/link/path list
616 #define RESTORE_MANIFEST_OWNER_GET(type) \
617 for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
618 { \
619 Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \
620 \
621 if (item->user == NULL) \
622 userNull = true; \
623 else \
624 strLstAddIfMissing(userList, item->user); \
625 \
626 if (item->group == NULL) \
627 groupNull = true; \
628 else \
629 strLstAddIfMissing(groupList, item->group); \
630 \
631 if (!userRoot()) \
632 { \
633 item->user = NULL; \
634 item->group = NULL; \
635 } \
636 }
637
638 // Helper to update an owner in a file/link/path list
639 #define RESTORE_MANIFEST_OWNER_NULL_UPDATE(type, user, group) \
640 for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
641 { \
642 Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \
643 \
644 if (item->user == NULL) \
645 item->user = user; \
646 \
647 if (item->group == NULL) \
648 item->group = group; \
649 }
650
651 // Helper to warn when an owner is missing and must be remapped
652 #define RESTORE_MANIFEST_OWNER_WARN(type) \
653 do \
654 { \
655 if (type##Null) \
656 LOG_WARN("unknown " #type " in backup manifest mapped to current " #type); \
657 \
658 for (unsigned int ownerIdx = 0; ownerIdx < strLstSize(type##List); ownerIdx++) \
659 { \
660 const String *owner = strLstGet(type##List, ownerIdx); \
661 \
662 if (type##Name() == NULL || !strEq(type##Name(), owner)) \
663 LOG_WARN_FMT("unknown " #type " '%s' in backup manifest mapped to current " #type, strZ(owner)); \
664 } \
665 } \
666 while (0)
667
668 static void
restoreManifestOwner(Manifest * manifest)669 restoreManifestOwner(Manifest *manifest)
670 {
671 FUNCTION_LOG_BEGIN(logLevelDebug);
672 FUNCTION_LOG_PARAM(MANIFEST, manifest);
673 FUNCTION_LOG_END();
674
675 ASSERT(manifest != NULL);
676
677 MEM_CONTEXT_TEMP_BEGIN()
678 {
679 // Build a list of users and groups in the manifest
680 // -------------------------------------------------------------------------------------------------------------------------
681 bool userNull = false;
682 StringList *userList = strLstNew();
683 bool groupNull = false;
684 StringList *groupList = strLstNew();
685
686 RESTORE_MANIFEST_OWNER_GET(File);
687 RESTORE_MANIFEST_OWNER_GET(Link);
688 RESTORE_MANIFEST_OWNER_GET(Path);
689
690 // Update users and groups in the manifest (this can only be done as root)
691 // -------------------------------------------------------------------------------------------------------------------------
692 if (userRoot())
693 {
694 // Get user/group info from data directory to use for invalid user/groups
695 StorageInfo pathInfo = storageInfoP(storagePg(), manifestTargetBase(manifest)->path);
696
697 // If user/group is null then set it to root
698 if (pathInfo.user == NULL) // {vm_covered}
699 pathInfo.user = userName(); // {vm_covered}
700
701 if (pathInfo.group == NULL) // {vm_covered}
702 pathInfo.group = groupName(); // {vm_covered}
703
704 if (userNull || groupNull)
705 {
706 if (userNull)
707 LOG_WARN_FMT("unknown user in backup manifest mapped to '%s'", strZ(pathInfo.user));
708
709 if (groupNull)
710 LOG_WARN_FMT("unknown group in backup manifest mapped to '%s'", strZ(pathInfo.group));
711
712 MEM_CONTEXT_PRIOR_BEGIN()
713 {
714 const String *user = strDup(pathInfo.user);
715 const String *group = strDup(pathInfo.group);
716
717 RESTORE_MANIFEST_OWNER_NULL_UPDATE(File, user, group)
718 RESTORE_MANIFEST_OWNER_NULL_UPDATE(Link, user, group)
719 RESTORE_MANIFEST_OWNER_NULL_UPDATE(Path, user, group)
720 }
721 MEM_CONTEXT_PRIOR_END();
722 }
723 }
724 // Else set owners to NULL. This means we won't make any attempt to update ownership and will just leave it as written by
725 // the current user/group. If there are existing files that are not owned by the current user/group then we will attempt
726 // to update them, which will generally cause an error, though some systems allow updates to the group ownership.
727 // -------------------------------------------------------------------------------------------------------------------------
728 else
729 {
730 RESTORE_MANIFEST_OWNER_WARN(user);
731 RESTORE_MANIFEST_OWNER_WARN(group);
732 }
733 }
734 MEM_CONTEXT_TEMP_END();
735
736 FUNCTION_LOG_RETURN_VOID();
737 }
738
739 /***********************************************************************************************************************************
740 Clean the data directory of any paths/files/links that are not in the manifest and create missing links/paths
741 ***********************************************************************************************************************************/
742 typedef struct RestoreCleanCallbackData
743 {
744 const Manifest *manifest; // Manifest to compare against
745 const ManifestTarget *target; // Current target being compared
746 const String *targetName; // Name to use when finding files/paths/links
747 const String *targetPath; // Path of target currently being compared
748 const String *subPath; // Subpath in target currently being compared
749 bool basePath; // Is this the base path?
750 bool exists; // Does the target path exist?
751 bool delta; // Is this a delta restore?
752 StringList *fileIgnore; // Files to ignore during clean
753 } RestoreCleanCallbackData;
754
755 // Helper to update ownership on a file/link/path
756 static void
restoreCleanOwnership(const String * pgPath,const String * manifestUserName,const String * manifestGroupName,uid_t actualUserId,gid_t actualGroupId,bool new)757 restoreCleanOwnership(
758 const String *pgPath, const String *manifestUserName, const String *manifestGroupName, uid_t actualUserId, gid_t actualGroupId,
759 bool new)
760 {
761 FUNCTION_TEST_BEGIN();
762 FUNCTION_TEST_PARAM(STRING, pgPath);
763 FUNCTION_TEST_PARAM(STRING, manifestUserName);
764 FUNCTION_TEST_PARAM(STRING, manifestGroupName);
765 FUNCTION_TEST_PARAM(UINT, actualUserId);
766 FUNCTION_TEST_PARAM(UINT, actualGroupId);
767 FUNCTION_TEST_PARAM(BOOL, new);
768 FUNCTION_TEST_END();
769
770 ASSERT(pgPath != NULL);
771
772 // Get the expected user id
773 uid_t expectedUserId = userId();
774
775 if (manifestUserName != NULL)
776 {
777 uid_t manifestUserId = userIdFromName(manifestUserName);
778
779 if (manifestUserId != (uid_t)-1)
780 expectedUserId = manifestUserId;
781 }
782
783 // Get the expected group id
784 gid_t expectedGroupId = groupId();
785
786 if (manifestGroupName != NULL)
787 {
788 uid_t manifestGroupId = groupIdFromName(manifestGroupName);
789
790 if (manifestGroupId != (uid_t)-1)
791 expectedGroupId = manifestGroupId;
792 }
793
794 // Update ownership if not as expected
795 if (actualUserId != expectedUserId || actualGroupId != expectedGroupId)
796 {
797 // If this is a newly created file/link/path then there's no need to log updated permissions
798 if (!new)
799 LOG_DETAIL_FMT("update ownership for '%s'", strZ(pgPath));
800
801 THROW_ON_SYS_ERROR_FMT(
802 lchown(strZ(pgPath), expectedUserId, expectedGroupId) == -1, FileOwnerError, "unable to set ownership for '%s'",
803 strZ(pgPath));
804 }
805
806 FUNCTION_TEST_RETURN_VOID();
807 }
808
809 // Helper to update mode on a file/path
810 static void
restoreCleanMode(const String * pgPath,mode_t manifestMode,const StorageInfo * info)811 restoreCleanMode(const String *pgPath, mode_t manifestMode, const StorageInfo *info)
812 {
813 FUNCTION_TEST_BEGIN();
814 FUNCTION_TEST_PARAM(STRING, pgPath);
815 FUNCTION_TEST_PARAM(MODE, manifestMode);
816 FUNCTION_TEST_PARAM(INFO, info);
817 FUNCTION_TEST_END();
818
819 ASSERT(pgPath != NULL);
820 ASSERT(info != NULL);
821
822 // Update mode if not as expected
823 if (manifestMode != info->mode)
824 {
825 LOG_DETAIL_FMT("update mode for '%s' to %04o", strZ(pgPath), manifestMode);
826
827 THROW_ON_SYS_ERROR_FMT(
828 chmod(strZ(pgPath), manifestMode) == -1, FileOwnerError, "unable to set mode for '%s'", strZ(pgPath));
829 }
830
831 FUNCTION_TEST_RETURN_VOID();
832 }
833
834 // storageInfoList() callback that cleans the paths
835 static void
restoreCleanInfoListCallback(void * data,const StorageInfo * info)836 restoreCleanInfoListCallback(void *data, const StorageInfo *info)
837 {
838 FUNCTION_TEST_BEGIN();
839 FUNCTION_TEST_PARAM_P(VOID, data);
840 FUNCTION_TEST_PARAM(STORAGE_INFO, info);
841 FUNCTION_TEST_END();
842
843 ASSERT(data != NULL);
844 ASSERT(info != NULL);
845
846 RestoreCleanCallbackData *cleanData = (RestoreCleanCallbackData *)data;
847
848 // Don't include backup.manifest or recovery.conf (when preserved) in the comparison or empty directory check
849 if (cleanData->basePath && info->type == storageTypeFile && strLstExists(cleanData->fileIgnore, info->name))
850 {
851 FUNCTION_TEST_RETURN_VOID();
852 return;
853 }
854
855 // Skip all . paths because they have already been cleaned on the previous level of recursion
856 if (strEq(info->name, DOT_STR))
857 {
858 FUNCTION_TEST_RETURN_VOID();
859 return;
860 }
861
862 // If this is not a delta then error because the directory is expected to be empty. Ignore the . path.
863 if (!cleanData->delta)
864 {
865 THROW_FMT(
866 PathNotEmptyError,
867 "unable to restore to path '%s' because it contains files\n"
868 "HINT: try using --delta if this is what you intended.",
869 strZ(cleanData->targetPath));
870 }
871
872 // Construct the name used to find this file/link/path in the manifest
873 const String *manifestName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info->name));
874
875 // Construct the path of this file/link/path in the PostgreSQL data directory
876 const String *pgPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info->name));
877
878 switch (info->type)
879 {
880 case storageTypeFile:
881 {
882 const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL);
883
884 if (manifestFile != NULL)
885 {
886 restoreCleanOwnership(pgPath, manifestFile->user, manifestFile->group, info->userId, info->groupId, false);
887 restoreCleanMode(pgPath, manifestFile->mode, info);
888 }
889 else
890 {
891 LOG_DETAIL_FMT("remove invalid file '%s'", strZ(pgPath));
892 storageRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true);
893 }
894
895 break;
896 }
897
898 case storageTypeLink:
899 {
900 const ManifestLink *manifestLink = manifestLinkFindDefault(cleanData->manifest, manifestName, NULL);
901
902 if (manifestLink != NULL)
903 {
904 if (!strEq(manifestLink->destination, info->linkDestination))
905 {
906 LOG_DETAIL_FMT("remove link '%s' because destination changed", strZ(pgPath));
907 storageRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true);
908 }
909 else
910 restoreCleanOwnership(pgPath, manifestLink->user, manifestLink->group, info->userId, info->groupId, false);
911 }
912 else
913 {
914 LOG_DETAIL_FMT("remove invalid link '%s'", strZ(pgPath));
915 storageRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true);
916 }
917
918 break;
919 }
920
921 case storageTypePath:
922 {
923 const ManifestPath *manifestPath = manifestPathFindDefault(cleanData->manifest, manifestName, NULL);
924
925 if (manifestPath != NULL)
926 {
927 // Check ownership/permissions
928 restoreCleanOwnership(pgPath, manifestPath->user, manifestPath->group, info->userId, info->groupId, false);
929 restoreCleanMode(pgPath, manifestPath->mode, info);
930
931 // Recurse into the path
932 RestoreCleanCallbackData cleanDataSub = *cleanData;
933 cleanDataSub.targetName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info->name));
934 cleanDataSub.targetPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info->name));
935 cleanDataSub.basePath = false;
936
937 storageInfoListP(
938 storageLocalWrite(), cleanDataSub.targetPath, restoreCleanInfoListCallback, &cleanDataSub,
939 .errorOnMissing = true, .sortOrder = sortOrderAsc);
940 }
941 else
942 {
943 LOG_DETAIL_FMT("remove invalid path '%s'", strZ(pgPath));
944 storagePathRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true, .recurse = true);
945 }
946
947 break;
948 }
949
950 // Special file types cannot exist in the manifest so just delete them
951 case storageTypeSpecial:
952 LOG_DETAIL_FMT("remove special file '%s'", strZ(pgPath));
953 storageRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true);
954 break;
955 }
956
957 FUNCTION_TEST_RETURN_VOID();
958 }
959
960 static void
restoreCleanBuild(Manifest * manifest)961 restoreCleanBuild(Manifest *manifest)
962 {
963 FUNCTION_LOG_BEGIN(logLevelDebug);
964 FUNCTION_LOG_PARAM(MANIFEST, manifest);
965 FUNCTION_LOG_END();
966
967 ASSERT(manifest != NULL);
968
969 MEM_CONTEXT_TEMP_BEGIN()
970 {
971 // Is this a delta restore?
972 bool delta = cfgOptionBool(cfgOptDelta) || cfgOptionBool(cfgOptForce);
973
974 // Allocate data for each target
975 RestoreCleanCallbackData *cleanDataList = memNew(sizeof(RestoreCleanCallbackData) * manifestTargetTotal(manifest));
976
977 // Step 1: Check permissions and validity (is the directory empty without delta?) if the target directory exists
978 // -------------------------------------------------------------------------------------------------------------------------
979 StringList *pathChecked = strLstNew();
980
981 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
982 {
983 RestoreCleanCallbackData *cleanData = &cleanDataList[targetIdx];
984
985 *cleanData = (RestoreCleanCallbackData)
986 {
987 .manifest = manifest,
988 .target = manifestTarget(manifest, targetIdx),
989 .delta = delta,
990 .fileIgnore = strLstNew(),
991 };
992
993 cleanData->targetName = cleanData->target->name;
994 cleanData->targetPath = manifestTargetPath(manifest, cleanData->target);
995 cleanData->basePath = strEq(cleanData->targetName, MANIFEST_TARGET_PGDATA_STR);
996
997 // Ignore backup.manifest while cleaning since it may exist from an prior incomplete restore
998 strLstAdd(cleanData->fileIgnore, BACKUP_MANIFEST_FILE_STR);
999
1000 // Also ignore recovery files when recovery type = preserve
1001 if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE)
1002 {
1003 // If recovery GUCs then three files must be preserved
1004 if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC)
1005 {
1006 strLstAdd(cleanData->fileIgnore, PG_FILE_POSTGRESQLAUTOCONF_STR);
1007 strLstAdd(cleanData->fileIgnore, PG_FILE_RECOVERYSIGNAL_STR);
1008 strLstAdd(cleanData->fileIgnore, PG_FILE_STANDBYSIGNAL_STR);
1009 }
1010 // Else just recovery.conf
1011 else
1012 strLstAdd(cleanData->fileIgnore, PG_FILE_RECOVERYCONF_STR);
1013 }
1014
1015 // If this is a tablespace append the tablespace identifier
1016 if (cleanData->target->type == manifestTargetTypeLink && cleanData->target->tablespaceId != 0)
1017 {
1018 const String *tablespaceId = pgTablespaceId(
1019 manifestData(manifest)->pgVersion, manifestData(manifest)->pgCatalogVersion);
1020
1021 // Only PostgreSQL >= 9.0 has tablespace indentifiers
1022 if (tablespaceId != NULL)
1023 {
1024 cleanData->targetName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(tablespaceId));
1025 cleanData->targetPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(tablespaceId));
1026 }
1027 }
1028
1029 strLstSort(cleanData->fileIgnore, sortOrderAsc);
1030
1031 // Check that the path exists. If not, there's no need to do any cleaning and we'll attempt to create it later.
1032 // Don't log check for the same path twice. There can be multiple links to files in the same path, but logging it more
1033 // than once makes the logs noisy and looks like a bug.
1034 if (!strLstExists(pathChecked, cleanData->targetPath))
1035 LOG_DETAIL_FMT("check '%s' exists", strZ(cleanData->targetPath));
1036
1037 StorageInfo info = storageInfoP(storageLocal(), cleanData->targetPath, .ignoreMissing = true, .followLink = true);
1038 strLstAdd(pathChecked, cleanData->targetPath);
1039
1040 if (info.exists)
1041 {
1042 // Make sure our uid will be able to write to this directory
1043 if (!userRoot() && userId() != info.userId)
1044 {
1045 THROW_FMT(
1046 PathOpenError, "unable to restore to path '%s' not owned by current user", strZ(cleanData->targetPath));
1047 }
1048
1049 if ((info.mode & 0700) != 0700)
1050 {
1051 THROW_FMT(
1052 PathOpenError, "unable to restore to path '%s' without rwx permissions", strZ(cleanData->targetPath));
1053 }
1054
1055 // If not a delta restore then check that the directories are empty, or if a file link, that the file doesn't exist
1056 if (!cleanData->delta)
1057 {
1058 if (cleanData->target->file == NULL)
1059 {
1060 storageInfoListP(
1061 storageLocal(), cleanData->targetPath, restoreCleanInfoListCallback, cleanData,
1062 .errorOnMissing = true);
1063 }
1064 else
1065 {
1066 const String *file = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(cleanData->target->file));
1067
1068 if (storageExistsP(storageLocal(), file))
1069 {
1070 THROW_FMT(
1071 FileExistsError,
1072 "unable to restore file '%s' because it already exists\n"
1073 "HINT: try using --delta if this is what you intended.",
1074 strZ(file));
1075 }
1076 }
1077
1078 // Now that we know there are no files in this target enable delta for processing in step 2
1079 cleanData->delta = true;
1080 }
1081
1082 // The target directory exists and is valid and will need to be cleaned
1083 cleanData->exists = true;
1084 }
1085 }
1086
1087 // Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be
1088 // created after paths are cleaned.
1089 if (manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP), NULL) != NULL &&
1090 manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP)
1091 {
1092 LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings");
1093 manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP));
1094 }
1095
1096 // Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs
1097 if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE &&
1098 manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF), NULL) != NULL)
1099 {
1100 LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve");
1101 manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF));
1102 }
1103
1104 // Step 2: Clean target directories
1105 // -------------------------------------------------------------------------------------------------------------------------
1106 // Delete the pg_control file (if it exists) so the cluster cannot be started if restore does not complete. Sync the path
1107 // so the file does not return, zombie-like, in the case of a host crash.
1108 if (storageExistsP(storagePg(), STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
1109 {
1110 LOG_DETAIL_FMT(
1111 "remove '" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL "' so cluster will not start if restore does not complete");
1112 storageRemoveP(storagePgWrite(), STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL));
1113 storagePathSyncP(storagePgWrite(), PG_PATH_GLOBAL_STR);
1114 }
1115
1116 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
1117 {
1118 RestoreCleanCallbackData *cleanData = &cleanDataList[targetIdx];
1119
1120 // Only clean if the target exists
1121 if (cleanData->exists)
1122 {
1123 // Don't clean file links. It doesn't matter whether the file exists or not since we know it is in the manifest.
1124 if (cleanData->target->file == NULL)
1125 {
1126 // Only log when doing a delta restore because otherwise the targets should be empty. We'll still run the clean
1127 // to fix permissions/ownership on the target paths.
1128 if (delta)
1129 LOG_INFO_FMT("remove invalid files/links/paths from '%s'", strZ(cleanData->targetPath));
1130
1131 // Check target ownership/permissions
1132 const ManifestPath *manifestPath = manifestPathFind(cleanData->manifest, cleanData->targetName);
1133 StorageInfo info = storageInfoP(storageLocal(), cleanData->targetPath, .followLink = true);
1134
1135 restoreCleanOwnership(
1136 cleanData->targetPath, manifestPath->user, manifestPath->group, info.userId, info.groupId, false);
1137 restoreCleanMode(cleanData->targetPath, manifestPath->mode, &info);
1138
1139 // Clean the target
1140 storageInfoListP(
1141 storageLocalWrite(), cleanData->targetPath, restoreCleanInfoListCallback, cleanData, .errorOnMissing = true,
1142 .sortOrder = sortOrderAsc);
1143 }
1144 }
1145 // If the target does not exist we'll attempt to create it
1146 else
1147 {
1148 const ManifestPath *path = NULL;
1149
1150 // There is no path information for a file link so we'll need to use the data directory
1151 if (cleanData->target->file != NULL)
1152 {
1153 path = manifestPathFind(manifest, MANIFEST_TARGET_PGDATA_STR);
1154 }
1155 // Else grab the info for the path that matches the link name
1156 else
1157 path = manifestPathFind(manifest, cleanData->target->name);
1158
1159 storagePathCreateP(storageLocalWrite(), cleanData->targetPath, .mode = path->mode);
1160 restoreCleanOwnership(cleanData->targetPath, path->user, path->group, userId(), groupId(), true);
1161 }
1162 }
1163
1164 // Step 3: Create missing paths and path links
1165 // -------------------------------------------------------------------------------------------------------------------------
1166 for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(manifest); pathIdx++)
1167 {
1168 const ManifestPath *path = manifestPath(manifest, pathIdx);
1169
1170 // Skip the pg_tblspc path because it only maps to the manifest. We should remove this in a future release but not much
1171 // can be done about it for now.
1172 if (strEq(path->name, MANIFEST_TARGET_PGTBLSPC_STR))
1173 continue;
1174
1175 // If this path has been mapped as a link then create a link. The path has already been created as part of target
1176 // creation (or it might have already existed).
1177 const ManifestLink *link = manifestLinkFindDefault(
1178 manifest,
1179 strBeginsWith(path->name, MANIFEST_TARGET_PGTBLSPC_STR) ?
1180 strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(path->name)) : path->name,
1181 NULL);
1182
1183 if (link != NULL)
1184 {
1185 const String *pgPath = storagePathP(storagePg(), manifestPathPg(link->name));
1186 StorageInfo linkInfo = storageInfoP(storagePg(), pgPath, .ignoreMissing = true);
1187
1188 // Create the link if it is missing. If it exists it should already have the correct ownership and destination.
1189 if (!linkInfo.exists)
1190 {
1191 LOG_DETAIL_FMT("create symlink '%s' to '%s'", strZ(pgPath), strZ(link->destination));
1192
1193 THROW_ON_SYS_ERROR_FMT(
1194 symlink(strZ(link->destination), strZ(pgPath)) == -1, FileOpenError,
1195 "unable to create symlink '%s' to '%s'", strZ(pgPath), strZ(link->destination));
1196 restoreCleanOwnership(pgPath, link->user, link->group, userId(), groupId(), true);
1197 }
1198 }
1199 // Create the path normally
1200 else
1201 {
1202 const String *pgPath = storagePathP(storagePg(), manifestPathPg(path->name));
1203 StorageInfo pathInfo = storageInfoP(storagePg(), pgPath, .ignoreMissing = true);
1204
1205 // Create the path if it is missing If it exists it should already have the correct ownership and mode.
1206 if (!pathInfo.exists)
1207 {
1208 LOG_DETAIL_FMT("create path '%s'", strZ(pgPath));
1209
1210 storagePathCreateP(storagePgWrite(), pgPath, .mode = path->mode, .noParentCreate = true, .errorOnExists = true);
1211 restoreCleanOwnership(storagePathP(storagePg(), pgPath), path->user, path->group, userId(), groupId(), true);
1212 }
1213 }
1214 }
1215
1216 // Step 4: Create file links. These don't get created during path creation because they do not have a matching path entry.
1217 // -------------------------------------------------------------------------------------------------------------------------
1218 for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(manifest); linkIdx++)
1219 {
1220 const ManifestLink *link = manifestLink(manifest, linkIdx);
1221
1222 const String *pgPath = storagePathP(storagePg(), manifestPathPg(link->name));
1223 StorageInfo linkInfo = storageInfoP(storagePg(), pgPath, .ignoreMissing = true);
1224
1225 // Create the link if it is missing. If it exists it should already have the correct ownership and destination.
1226 if (!linkInfo.exists)
1227 {
1228 LOG_DETAIL_FMT("create symlink '%s' to '%s'", strZ(pgPath), strZ(link->destination));
1229
1230 THROW_ON_SYS_ERROR_FMT(
1231 symlink(strZ(link->destination), strZ(pgPath)) == -1, FileOpenError,
1232 "unable to create symlink '%s' to '%s'", strZ(pgPath), strZ(link->destination));
1233 restoreCleanOwnership(pgPath, link->user, link->group, userId(), groupId(), true);
1234 }
1235 }
1236 }
1237 MEM_CONTEXT_TEMP_END();
1238
1239 FUNCTION_LOG_RETURN_VOID();
1240 }
1241
1242 /***********************************************************************************************************************************
1243 Generate the expression to zero files that are not needed for selective restore
1244 ***********************************************************************************************************************************/
1245 static String *
restoreSelectiveExpression(Manifest * manifest)1246 restoreSelectiveExpression(Manifest *manifest)
1247 {
1248 FUNCTION_LOG_BEGIN(logLevelDebug);
1249 FUNCTION_LOG_PARAM(MANIFEST, manifest);
1250 FUNCTION_LOG_END();
1251
1252 ASSERT(manifest != NULL);
1253
1254 String *result = NULL;
1255
1256 // Continue if databases to include or exclude have been specified
1257 if (cfgOptionTest(cfgOptDbExclude) || cfgOptionTest(cfgOptDbInclude))
1258 {
1259 MEM_CONTEXT_TEMP_BEGIN()
1260 {
1261 // Generate base expression
1262 RegExp *baseRegExp = regExpNew(STRDEF("^" MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/[0-9]+/" PG_FILE_PGVERSION));
1263
1264 // Generate tablespace expression
1265 RegExp *tablespaceRegExp = NULL;
1266 const String *tablespaceId = pgTablespaceId(
1267 manifestData(manifest)->pgVersion, manifestData(manifest)->pgCatalogVersion);
1268
1269 if (tablespaceId == NULL)
1270 tablespaceRegExp = regExpNew(STRDEF("^" MANIFEST_TARGET_PGTBLSPC "/[0-9]+/[0-9]+/" PG_FILE_PGVERSION));
1271 else
1272 {
1273 tablespaceRegExp = regExpNew(
1274 strNewFmt("^" MANIFEST_TARGET_PGTBLSPC "/[0-9]+/%s/[0-9]+/" PG_FILE_PGVERSION, strZ(tablespaceId)));
1275 }
1276
1277 // Generate a list of databases in base or in a tablespace and get all standard system databases, even in cases where
1278 // users have recreated them
1279 StringList *systemDbIdList = strLstNew();
1280 StringList *dbList = strLstNew();
1281
1282 for (unsigned int systemDbIdx = 0; systemDbIdx < manifestDbTotal(manifest); systemDbIdx++)
1283 {
1284 const ManifestDb *systemDb = manifestDb(manifest, systemDbIdx);
1285
1286 if (strEqZ(systemDb->name, "template0") || strEqZ(systemDb->name, "template1") ||
1287 strEqZ(systemDb->name, "postgres") || systemDb->id < PG_USER_OBJECT_MIN_ID)
1288 {
1289 // Build the system id list and add to the dbList for logging and checking
1290 const String *systemDbId = varStrForce(VARUINT(systemDb->id));
1291 strLstAdd(systemDbIdList, systemDbId);
1292 strLstAdd(dbList, systemDbId);
1293 }
1294 }
1295
1296 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
1297 {
1298 const ManifestFile *file = manifestFile(manifest, fileIdx);
1299
1300 if (regExpMatch(baseRegExp, file->name) || regExpMatch(tablespaceRegExp, file->name))
1301 {
1302 String *dbId = strBase(strPath(file->name));
1303
1304 // In the highly unlikely event that a system database was somehow added after the backup began, it will only be
1305 // found in the file list and not the manifest db section, so add it to the system database list
1306 if (cvtZToUInt64(strZ(dbId)) < PG_USER_OBJECT_MIN_ID)
1307 strLstAddIfMissing(systemDbIdList, dbId);
1308
1309 strLstAddIfMissing(dbList, dbId);
1310 }
1311 }
1312
1313 strLstSort(dbList, sortOrderAsc);
1314
1315 // If no databases were found then this backup is not a valid cluster
1316 if (strLstEmpty(dbList))
1317 THROW(FormatError, "no databases found for selective restore\nHINT: is this a valid cluster?");
1318
1319 // Log databases found
1320 LOG_DETAIL_FMT("databases found for selective restore (%s)", strZ(strLstJoin(dbList, ", ")));
1321
1322 // Generate list with ids of databases to exclude
1323 StringList *excludeDbIdList = strLstNew();
1324 const StringList *excludeList = strLstNewVarLst(cfgOptionLst(cfgOptDbExclude));
1325
1326 for (unsigned int excludeIdx = 0; excludeIdx < strLstSize(excludeList); excludeIdx++)
1327 {
1328 const String *excludeDb = strLstGet(excludeList, excludeIdx);
1329
1330 // If the db to exclude is not in the list as an id then search by name
1331 if (!strLstExists(dbList, excludeDb))
1332 {
1333 const ManifestDb *db = manifestDbFindDefault(manifest, excludeDb, NULL);
1334
1335 if (db == NULL || !strLstExists(dbList, varStrForce(VARUINT(db->id))))
1336 THROW_FMT(DbMissingError, "database to exclude '%s' does not exist", strZ(excludeDb));
1337
1338 // Set the exclude db to the id if the name mapping was successful
1339 excludeDb = varStrForce(VARUINT(db->id));
1340 }
1341
1342 // Add to exclude list
1343 strLstAdd(excludeDbIdList, excludeDb);
1344 }
1345
1346 // Remove included databases from the list
1347 const StringList *includeList = strLstNewVarLst(cfgOptionLst(cfgOptDbInclude));
1348
1349 for (unsigned int includeIdx = 0; includeIdx < strLstSize(includeList); includeIdx++)
1350 {
1351 const String *includeDb = strLstGet(includeList, includeIdx);
1352
1353 // If the db to include is not in the list as an id then search by name
1354 if (!strLstExists(dbList, includeDb))
1355 {
1356 const ManifestDb *db = manifestDbFindDefault(manifest, includeDb, NULL);
1357
1358 if (db == NULL || !strLstExists(dbList, varStrForce(VARUINT(db->id))))
1359 THROW_FMT(DbMissingError, "database to include '%s' does not exist", strZ(includeDb));
1360
1361 // Set the include db to the id if the name mapping was successful
1362 includeDb = varStrForce(VARUINT(db->id));
1363 }
1364
1365 // Error if the db is a system db
1366 if (strLstExists(systemDbIdList, includeDb))
1367 THROW(DbInvalidError, "system databases (template0, postgres, etc.) are included by default");
1368
1369 // Error if the db id is in the exclude list
1370 if (strLstExists(excludeDbIdList, includeDb))
1371 THROW_FMT(DbInvalidError, "database to include '%s' is in the exclude list", strZ(includeDb));
1372
1373 // Remove from list of DBs to zero
1374 strLstRemove(dbList, includeDb);
1375 }
1376
1377 // Only exclude specified db in case no db to include has been provided
1378 if (strLstEmpty(includeList))
1379 {
1380 dbList = strLstDup(excludeDbIdList);
1381 }
1382 // Else, remove the system databases from list of DBs to zero unless they are excluded explicitly
1383 else
1384 {
1385 strLstSort(systemDbIdList, sortOrderAsc);
1386 strLstSort(excludeDbIdList, sortOrderAsc);
1387 systemDbIdList = strLstMergeAnti(systemDbIdList, excludeDbIdList);
1388 dbList = strLstMergeAnti(dbList, systemDbIdList);
1389 }
1390
1391 // Build regular expression to identify files that will be zeroed
1392 String *expression = NULL;
1393
1394 if (!strLstEmpty(dbList))
1395 {
1396 LOG_DETAIL_FMT("databases excluded (zeroed) from selective restore (%s)", strZ(strLstJoin(dbList, ", ")));
1397
1398 // Generate the expression from the list of databases to be zeroed. Only user created databases can be zeroed, never
1399 // system databases.
1400 for (unsigned int dbIdx = 0; dbIdx < strLstSize(dbList); dbIdx++)
1401 {
1402 const String *db = strLstGet(dbList, dbIdx);
1403
1404 // Create expression string or append |
1405 if (expression == NULL)
1406 expression = strNew();
1407 else
1408 strCatZ(expression, "|");
1409
1410 // Filter files in base directory
1411 strCatFmt(expression, "(^" MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/%s/)", strZ(db));
1412
1413 // Filter files in tablespace directories
1414 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
1415 {
1416 const ManifestTarget *target = manifestTarget(manifest, targetIdx);
1417
1418 if (target->tablespaceId != 0)
1419 {
1420 if (tablespaceId == NULL)
1421 strCatFmt(expression, "|(^%s/%s/)", strZ(target->name), strZ(db));
1422 else
1423 strCatFmt(expression, "|(^%s/%s/%s/)", strZ(target->name), strZ(tablespaceId), strZ(db));
1424 }
1425 }
1426 }
1427 }
1428
1429 // If all user databases have been selected then nothing to do
1430 if (expression == NULL)
1431 {
1432 LOG_INFO_FMT("nothing to filter - all user databases have been selected");
1433 }
1434 // Else return the expression
1435 else
1436 {
1437 MEM_CONTEXT_PRIOR_BEGIN()
1438 {
1439 result = strDup(expression);
1440 }
1441 MEM_CONTEXT_PRIOR_END();
1442 }
1443 }
1444 MEM_CONTEXT_TEMP_END();
1445 }
1446
1447 FUNCTION_LOG_RETURN(STRING, result);
1448 }
1449
1450 /***********************************************************************************************************************************
1451 Generate the recovery file
1452 ***********************************************************************************************************************************/
1453 // Helper to generate recovery options
1454 static KeyValue *
restoreRecoveryOption(unsigned int pgVersion)1455 restoreRecoveryOption(unsigned int pgVersion)
1456 {
1457 FUNCTION_LOG_BEGIN(logLevelDebug);
1458 FUNCTION_LOG_PARAM(UINT, pgVersion);
1459 FUNCTION_LOG_END();
1460
1461 KeyValue *result = NULL;
1462
1463 MEM_CONTEXT_TEMP_BEGIN()
1464 {
1465 result = kvNew();
1466
1467 StringList *recoveryOptionKey = strLstNew();
1468
1469 if (cfgOptionTest(cfgOptRecoveryOption))
1470 {
1471 const KeyValue *recoveryOption = cfgOptionKv(cfgOptRecoveryOption);
1472 recoveryOptionKey = strLstSort(strLstNewVarLst(kvKeyList(recoveryOption)), sortOrderAsc);
1473
1474 for (unsigned int keyIdx = 0; keyIdx < strLstSize(recoveryOptionKey); keyIdx++)
1475 {
1476 // Get the key and value
1477 String *key = strLstGet(recoveryOptionKey, keyIdx);
1478 const String *value = varStr(kvGet(recoveryOption, VARSTR(key)));
1479
1480 // Replace - in key with _. Since we use - users naturally will as well.
1481 strReplaceChr(key, '-', '_');
1482
1483 kvPut(result, VARSTR(key), VARSTR(value));
1484 }
1485
1486 strLstSort(recoveryOptionKey, sortOrderAsc);
1487 }
1488
1489 // If archive-mode is not preserve
1490 if (cfgOptionStrId(cfgOptArchiveMode) != CFGOPTVAL_ARCHIVE_MODE_PRESERVE)
1491 {
1492 if (pgVersion < PG_VERSION_12)
1493 {
1494 THROW_FMT(
1495 OptionInvalidError,
1496 "option '" CFGOPT_ARCHIVE_MODE "' is not supported on " PG_NAME " < " PG_VERSION_12_STR "\n"
1497 "HINT: 'archive_mode' should be manually set to 'off' in postgresql.conf.");
1498 }
1499
1500 // The only other valid option is off
1501 ASSERT(cfgOptionStrId(cfgOptArchiveMode) == CFGOPTVAL_ARCHIVE_MODE_OFF);
1502
1503 // If archive-mode=off then set archive_mode=off
1504 kvPut(result, VARSTRDEF(ARCHIVE_MODE), VARSTRDEF(CFGOPTVAL_ARCHIVE_MODE_OFF_Z));
1505 }
1506
1507 // Write restore_command
1508 if (!strLstExists(recoveryOptionKey, RESTORE_COMMAND_STR))
1509 {
1510 // Null out options that it does not make sense to pass from the restore command to archive-get. All of these have
1511 // reasonable defaults so there is no danger of an error -- they just might not be optimal. In any case, it seems
1512 // better than, for example, passing --process-max=32 to archive-get because it was specified for restore.
1513 KeyValue *optionReplace = kvNew();
1514
1515 kvPut(optionReplace, VARSTRDEF(CFGOPT_EXEC_ID), NULL);
1516 kvPut(optionReplace, VARSTRDEF(CFGOPT_JOB_RETRY), NULL);
1517 kvPut(optionReplace, VARSTRDEF(CFGOPT_JOB_RETRY_INTERVAL), NULL);
1518 kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_CONSOLE), NULL);
1519 kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_FILE), NULL);
1520 kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_STDERR), NULL);
1521 kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_SUBPROCESS), NULL);
1522 kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_TIMESTAMP), NULL);
1523 kvPut(optionReplace, VARSTRDEF(CFGOPT_PROCESS_MAX), NULL);
1524
1525 kvPut(
1526 result, VARSTRZ(RESTORE_COMMAND),
1527 VARSTR(
1528 strNewFmt(
1529 "%s %s %%f \"%%p\"", strZ(cfgExe()),
1530 strZ(strLstJoin(cfgExecParam(cfgCmdArchiveGet, cfgCmdRoleMain, optionReplace, true, true), " ")))));
1531 }
1532
1533 // If recovery type is immediate
1534 if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_IMMEDIATE)
1535 {
1536 kvPut(result, VARSTRZ(RECOVERY_TARGET), VARSTRZ(CFGOPTVAL_TYPE_IMMEDIATE_Z));
1537 }
1538 // Else recovery type is standby
1539 else if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_STANDBY)
1540 {
1541 // Write standby_mode for PostgreSQL versions that support it
1542 if (pgVersion < PG_VERSION_RECOVERY_GUC)
1543 kvPut(result, VARSTR(STANDBY_MODE_STR), VARSTRDEF("on"));
1544 }
1545 // Else recovery type is not default so write target options
1546 else if (cfgOptionStrId(cfgOptType) != CFGOPTVAL_TYPE_DEFAULT)
1547 {
1548 // Write the recovery target
1549 kvPut(
1550 result, VARSTR(strNewFmt(RECOVERY_TARGET "_%s", strZ(cfgOptionStr(cfgOptType)))),
1551 VARSTR(cfgOptionStr(cfgOptTarget)));
1552
1553 // Write recovery_target_inclusive
1554 if (cfgOptionTest(cfgOptTargetExclusive) && cfgOptionBool(cfgOptTargetExclusive))
1555 kvPut(result, VARSTRZ(RECOVERY_TARGET_INCLUSIVE), VARSTR(FALSE_STR));
1556 }
1557
1558 // Write pause_at_recovery_target/recovery_target_action
1559 if (cfgOptionTest(cfgOptTargetAction))
1560 {
1561 StringId targetAction = cfgOptionStrId(cfgOptTargetAction);
1562
1563 if (targetAction != CFGOPTVAL_TARGET_ACTION_PAUSE)
1564 {
1565 // Write recovery_target on supported PostgreSQL versions
1566 if (pgVersion >= PG_VERSION_RECOVERY_TARGET_ACTION)
1567 {
1568 kvPut(result, VARSTRZ(RECOVERY_TARGET_ACTION), VARSTR(strIdToStr(targetAction)));
1569 }
1570 // Write pause_at_recovery_target on supported PostgreSQL versions
1571 else if (pgVersion >= PG_VERSION_RECOVERY_TARGET_PAUSE)
1572 {
1573 // Shutdown action is not supported with pause_at_recovery_target setting
1574 if (targetAction == CFGOPTVAL_TARGET_ACTION_SHUTDOWN)
1575 {
1576 THROW_FMT(
1577 OptionInvalidError,
1578 CFGOPT_TARGET_ACTION "=" CFGOPTVAL_TARGET_ACTION_SHUTDOWN_Z " is only available in PostgreSQL >= %s",
1579 strZ(pgVersionToStr(PG_VERSION_RECOVERY_TARGET_ACTION)));
1580 }
1581
1582 kvPut(result, VARSTRZ(PAUSE_AT_RECOVERY_TARGET), VARSTR(FALSE_STR));
1583 }
1584 // Else error on unsupported version
1585 else
1586 {
1587 THROW_FMT(
1588 OptionInvalidError, CFGOPT_TARGET_ACTION " option is only available in PostgreSQL >= %s",
1589 strZ(pgVersionToStr(PG_VERSION_RECOVERY_TARGET_PAUSE)));
1590 }
1591 }
1592 }
1593
1594 // Write recovery_target_timeline
1595 if (cfgOptionTest(cfgOptTargetTimeline))
1596 kvPut(result, VARSTRZ(RECOVERY_TARGET_TIMELINE), VARSTR(cfgOptionStr(cfgOptTargetTimeline)));
1597
1598 // Move to prior context
1599 kvMove(result, memContextPrior());
1600 }
1601 MEM_CONTEXT_TEMP_END();
1602
1603 FUNCTION_LOG_RETURN(KEY_VALUE, result);
1604 }
1605
1606 // Helper to convert recovery options to text format
1607 static String *
restoreRecoveryConf(unsigned int pgVersion,const String * restoreLabel)1608 restoreRecoveryConf(unsigned int pgVersion, const String *restoreLabel)
1609 {
1610 FUNCTION_LOG_BEGIN(logLevelDebug);
1611 FUNCTION_LOG_PARAM(UINT, pgVersion);
1612 FUNCTION_LOG_PARAM(STRING, restoreLabel);
1613 FUNCTION_LOG_END();
1614
1615 String *result = NULL;
1616
1617 MEM_CONTEXT_TEMP_BEGIN()
1618 {
1619 result = strNewFmt("# Recovery settings generated by " PROJECT_NAME " restore on %s\n", strZ(restoreLabel));
1620
1621 // Output all recovery options
1622 KeyValue *optionKv = restoreRecoveryOption(pgVersion);
1623 const VariantList *optionKeyList = kvKeyList(optionKv);
1624
1625 for (unsigned int optionKeyIdx = 0; optionKeyIdx < varLstSize(optionKeyList); optionKeyIdx++)
1626 {
1627 const Variant *optionKey = varLstGet(optionKeyList, optionKeyIdx);
1628 strCatFmt(result, "%s = '%s'\n", strZ(varStr(optionKey)), strZ(varStr(kvGet(optionKv, optionKey))));
1629 }
1630
1631 // Move to prior context
1632 MEM_CONTEXT_PRIOR_BEGIN()
1633 {
1634 result = strDup(result);
1635 }
1636 MEM_CONTEXT_PRIOR_END();
1637 }
1638 MEM_CONTEXT_TEMP_END();
1639
1640 FUNCTION_LOG_RETURN(STRING, result);
1641 }
1642
1643 // Helper to write recovery options into recovery.conf
1644 static void
restoreRecoveryWriteConf(const Manifest * manifest,unsigned int pgVersion,const String * restoreLabel)1645 restoreRecoveryWriteConf(const Manifest *manifest, unsigned int pgVersion, const String *restoreLabel)
1646 {
1647 FUNCTION_LOG_BEGIN(logLevelDebug);
1648 FUNCTION_LOG_PARAM(MANIFEST, manifest);
1649 FUNCTION_LOG_PARAM(UINT, pgVersion);
1650 FUNCTION_LOG_PARAM(STRING, restoreLabel);
1651 FUNCTION_LOG_END();
1652
1653 // Only write recovery.conf if recovery type != none
1654 if (cfgOptionStrId(cfgOptType) != CFGOPTVAL_TYPE_NONE)
1655 {
1656 LOG_INFO_FMT("write %s", strZ(storagePathP(storagePg(), PG_FILE_RECOVERYCONF_STR)));
1657
1658 // Use the data directory to set permissions and ownership for recovery file
1659 const ManifestPath *dataPath = manifestPathFind(manifest, MANIFEST_TARGET_PGDATA_STR);
1660 mode_t recoveryFileMode = dataPath->mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
1661
1662 // Write recovery.conf
1663 storagePutP(
1664 storageNewWriteP(
1665 storagePgWrite(), PG_FILE_RECOVERYCONF_STR, .noCreatePath = true, .modeFile = recoveryFileMode, .noAtomic = true,
1666 .noSyncPath = true, .user = dataPath->user, .group = dataPath->group),
1667 BUFSTR(restoreRecoveryConf(pgVersion, restoreLabel)));
1668 }
1669
1670 FUNCTION_LOG_RETURN_VOID();
1671 }
1672
1673 // Helper to write recovery options into postgresql.auto.conf
1674 static void
restoreRecoveryWriteAutoConf(unsigned int pgVersion,const String * restoreLabel)1675 restoreRecoveryWriteAutoConf(unsigned int pgVersion, const String *restoreLabel)
1676 {
1677 FUNCTION_LOG_BEGIN(logLevelDebug);
1678 FUNCTION_LOG_PARAM(UINT, pgVersion);
1679 FUNCTION_LOG_PARAM(STRING, restoreLabel);
1680 FUNCTION_LOG_END();
1681
1682 MEM_CONTEXT_TEMP_BEGIN()
1683 {
1684 String *content = strNew();
1685
1686 // Load postgresql.auto.conf so we can preserve the existing contents
1687 Buffer *autoConf = storageGetP(storageNewReadP(storagePg(), PG_FILE_POSTGRESQLAUTOCONF_STR, .ignoreMissing = true));
1688
1689 // It is unusual for the file not to exist, but we'll continue processing by creating a blank file
1690 if (autoConf == NULL)
1691 {
1692 LOG_WARN(PG_FILE_POSTGRESQLAUTOCONF " does not exist -- creating to contain recovery settings");
1693 }
1694 // Else the file does exist so comment out old recovery options that could interfere with the current recovery. Don't
1695 // comment out *all* recovery options because some should only be commented out if there is a new option to replace it, e.g.
1696 // primary_conninfo. If the option shouldn't be commented out all the time then it won't ever be commnented out -- this
1697 // may not be ideal but it is what was decided. PostgreSQL will use the last value set so this is safe as long as the option
1698 // does not have dependencies on other options.
1699 else
1700 {
1701 // Generate a regexp that will match on all current recovery_target settings
1702 RegExp *recoveryExp =
1703 regExpNew(
1704 STRDEF(
1705 "^[\t ]*(" RECOVERY_TARGET "|" RECOVERY_TARGET_ACTION "|" RECOVERY_TARGET_INCLUSIVE "|"
1706 RECOVERY_TARGET_LSN "|" RECOVERY_TARGET_NAME "|" RECOVERY_TARGET_TIME "|" RECOVERY_TARGET_TIMELINE "|"
1707 RECOVERY_TARGET_XID ")[\t ]*="));
1708
1709 // Check each line for recovery settings
1710 const StringList *contentList = strLstNewSplit(strNewBuf(autoConf), LF_STR);
1711
1712 for (unsigned int contentIdx = 0; contentIdx < strLstSize(contentList); contentIdx++)
1713 {
1714 if (contentIdx != 0)
1715 strCat(content, LF_STR);
1716
1717 const String *line = strLstGet(contentList, contentIdx);
1718
1719 if (regExpMatch(recoveryExp, line))
1720 strCatFmt(content, "# Removed by " PROJECT_NAME " restore on %s # ", strZ(restoreLabel));
1721
1722 strCat(content, line);
1723 }
1724
1725 // If settings will be appended then format the file so a blank line will be between old and new settings
1726 if (cfgOptionStrId(cfgOptType) != CFGOPTVAL_TYPE_NONE)
1727 {
1728 strTrim(content);
1729 strCatZ(content, "\n\n");
1730 }
1731 }
1732
1733 // If recovery was requested then write the recovery options
1734 if (cfgOptionStrId(cfgOptType) != CFGOPTVAL_TYPE_NONE)
1735 {
1736 // If the user specified standby_mode as a recovery option then error. It's tempting to just set type=standby in this
1737 // case but since config parsing has already happened the target options could be in an invalid state.
1738 if (cfgOptionTest(cfgOptRecoveryOption))
1739 {
1740 const KeyValue *recoveryOption = cfgOptionKv(cfgOptRecoveryOption);
1741 StringList *recoveryOptionKey = strLstNewVarLst(kvKeyList(recoveryOption));
1742
1743 for (unsigned int keyIdx = 0; keyIdx < strLstSize(recoveryOptionKey); keyIdx++)
1744 {
1745 // Get the key and value
1746 String *key = strLstGet(recoveryOptionKey, keyIdx);
1747
1748 // Replace - in key with _. Since we use - users naturally will as well.
1749 strReplaceChr(key, '-', '_');
1750
1751 if (strEq(key, STANDBY_MODE_STR))
1752 {
1753 THROW_FMT(
1754 OptionInvalidError,
1755 "'" STANDBY_MODE "' setting is not valid for " PG_NAME " >= %s\n"
1756 "HINT: use --" CFGOPT_TYPE "=" CFGOPTVAL_TYPE_STANDBY_Z " instead of --" CFGOPT_RECOVERY_OPTION "="
1757 STANDBY_MODE "=on.",
1758 strZ(pgVersionToStr(PG_VERSION_RECOVERY_GUC)));
1759 }
1760 }
1761 }
1762
1763 strCatFmt(content, "%s", strZ(restoreRecoveryConf(pgVersion, restoreLabel)));
1764 }
1765
1766 LOG_INFO_FMT(
1767 "write %s%s", autoConf == NULL ? "" : "updated ", strZ(storagePathP(storagePg(), PG_FILE_POSTGRESQLAUTOCONF_STR)));
1768
1769 // Use the data directory to set permissions and ownership for recovery file
1770 const StorageInfo dataPath = storageInfoP(storagePg(), NULL);
1771 mode_t recoveryFileMode = dataPath.mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
1772
1773 // Write postgresql.auto.conf
1774 storagePutP(
1775 storageNewWriteP(
1776 storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF_STR, .noCreatePath = true, .modeFile = recoveryFileMode,
1777 .noAtomic = true, .noSyncPath = true, .user = dataPath.user, .group = dataPath.group),
1778 BUFSTR(content));
1779
1780 // The standby.signal file is required for standby mode
1781 if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_STANDBY)
1782 {
1783 storagePutP(
1784 storageNewWriteP(
1785 storagePgWrite(), PG_FILE_STANDBYSIGNAL_STR, .noCreatePath = true, .modeFile = recoveryFileMode,
1786 .noAtomic = true, .noSyncPath = true, .user = dataPath.user, .group = dataPath.group),
1787 NULL);
1788 }
1789 // Else the recovery.signal file is required for targeted recovery
1790 else
1791 {
1792 storagePutP(
1793 storageNewWriteP(
1794 storagePgWrite(), PG_FILE_RECOVERYSIGNAL_STR, .noCreatePath = true, .modeFile = recoveryFileMode,
1795 .noAtomic = true, .noSyncPath = true, .user = dataPath.user, .group = dataPath.group),
1796 NULL);
1797 }
1798 }
1799 MEM_CONTEXT_TEMP_END();
1800
1801 FUNCTION_LOG_RETURN_VOID();
1802 }
1803
1804 static void
restoreRecoveryWrite(const Manifest * manifest)1805 restoreRecoveryWrite(const Manifest *manifest)
1806 {
1807 FUNCTION_LOG_BEGIN(logLevelDebug);
1808 FUNCTION_LOG_PARAM(MANIFEST, manifest);
1809 FUNCTION_LOG_END();
1810
1811 // Get PostgreSQL version to write recovery for
1812 unsigned int pgVersion = manifestData(manifest)->pgVersion;
1813
1814 MEM_CONTEXT_TEMP_BEGIN()
1815 {
1816 // If recovery type is preserve then leave recovery file as it is
1817 if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE)
1818 {
1819 // Determine which file recovery setttings will be written to
1820 const String *recoveryFile = pgVersion >= PG_VERSION_RECOVERY_GUC ?
1821 PG_FILE_POSTGRESQLAUTOCONF_STR : PG_FILE_RECOVERYCONF_STR;
1822
1823 if (!storageExistsP(storagePg(), recoveryFile))
1824 {
1825 LOG_WARN_FMT(
1826 "recovery type is " CFGOPTVAL_TYPE_PRESERVE_Z " but recovery file does not exist at '%s'",
1827 strZ(storagePathP(storagePg(), recoveryFile)));
1828 }
1829 }
1830 // Else write recovery file
1831 else
1832 {
1833 // Generate a label used to identify this restore in the recovery file
1834 struct tm timePart;
1835 char restoreTimestamp[20];
1836 time_t timestamp = time(NULL);
1837
1838 strftime(restoreTimestamp, sizeof(restoreTimestamp), "%Y-%m-%d %H:%M:%S", localtime_r(×tamp, &timePart));
1839 const String *restoreLabel = STR(restoreTimestamp);
1840
1841 // Write recovery file based on PostgreSQL version
1842 if (pgVersion >= PG_VERSION_RECOVERY_GUC)
1843 restoreRecoveryWriteAutoConf(pgVersion, restoreLabel);
1844 else
1845 restoreRecoveryWriteConf(manifest, pgVersion, restoreLabel);
1846 }
1847 }
1848 MEM_CONTEXT_TEMP_END();
1849
1850 FUNCTION_LOG_RETURN_VOID();
1851 }
1852
1853 /***********************************************************************************************************************************
1854 Generate a list of queues that determine the order of file processing
1855 ***********************************************************************************************************************************/
1856 // Comparator to order ManifestFile objects by size then name
1857 static int
restoreProcessQueueComparator(const void * item1,const void * item2)1858 restoreProcessQueueComparator(const void *item1, const void *item2)
1859 {
1860 FUNCTION_TEST_BEGIN();
1861 FUNCTION_TEST_PARAM_P(VOID, item1);
1862 FUNCTION_TEST_PARAM_P(VOID, item2);
1863 FUNCTION_TEST_END();
1864
1865 ASSERT(item1 != NULL);
1866 ASSERT(item2 != NULL);
1867
1868 // If the size differs then that's enough to determine order
1869 if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size)
1870 FUNCTION_TEST_RETURN(-1);
1871 else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size)
1872 FUNCTION_TEST_RETURN(1);
1873
1874 // If size is the same then use name to generate a deterministic ordering (names must be unique)
1875 FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name));
1876 }
1877
1878 static uint64_t
restoreProcessQueue(Manifest * manifest,List ** queueList)1879 restoreProcessQueue(Manifest *manifest, List **queueList)
1880 {
1881 FUNCTION_LOG_BEGIN(logLevelDebug);
1882 FUNCTION_LOG_PARAM(MANIFEST, manifest);
1883 FUNCTION_LOG_PARAM_P(LIST, queueList);
1884 FUNCTION_LOG_END();
1885
1886 ASSERT(manifest != NULL);
1887
1888 uint64_t result = 0;
1889
1890 MEM_CONTEXT_TEMP_BEGIN()
1891 {
1892 // Create list of process queue
1893 *queueList = lstNewP(sizeof(List *));
1894
1895 // Generate the list of processing queues (there is always at least one)
1896 StringList *targetList = strLstNew();
1897 strLstAdd(targetList, STRDEF(MANIFEST_TARGET_PGDATA "/"));
1898
1899 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
1900 {
1901 const ManifestTarget *target = manifestTarget(manifest, targetIdx);
1902
1903 if (target->tablespaceId != 0)
1904 strLstAdd(targetList, strNewFmt("%s/", strZ(target->name)));
1905 }
1906
1907 // Generate the processing queues
1908 MEM_CONTEXT_BEGIN(lstMemContext(*queueList))
1909 {
1910 for (unsigned int targetIdx = 0; targetIdx < strLstSize(targetList); targetIdx++)
1911 {
1912 List *queue = lstNewP(sizeof(ManifestFile *), .comparator = restoreProcessQueueComparator);
1913 lstAdd(*queueList, &queue);
1914 }
1915 }
1916 MEM_CONTEXT_END();
1917
1918 // Now put all files into the processing queues
1919 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
1920 {
1921 const ManifestFile *file = manifestFile(manifest, fileIdx);
1922
1923 // Find the target that contains this file
1924 unsigned int targetIdx = 0;
1925
1926 do
1927 {
1928 // A target should always be found
1929 CHECK(targetIdx < strLstSize(targetList));
1930
1931 if (strBeginsWith(file->name, strLstGet(targetList, targetIdx)))
1932 break;
1933
1934 targetIdx++;
1935 }
1936 while (1);
1937
1938 // Add file to queue
1939 lstAdd(*(List **)lstGet(*queueList, targetIdx), &file);
1940
1941 // Add size to total
1942 result += file->size;
1943 }
1944
1945 // Sort the queues
1946 for (unsigned int targetIdx = 0; targetIdx < strLstSize(targetList); targetIdx++)
1947 lstSort(*(List **)lstGet(*queueList, targetIdx), sortOrderDesc);
1948
1949 // Move process queues to prior context
1950 lstMove(*queueList, memContextPrior());
1951 }
1952 MEM_CONTEXT_TEMP_END();
1953
1954 FUNCTION_LOG_RETURN(UINT64, result);
1955 }
1956
1957 /***********************************************************************************************************************************
1958 Log the results of a job and throw errors
1959 ***********************************************************************************************************************************/
1960 // Helper function to determine if a file should be zeroed
1961 static bool
restoreFileZeroed(const String * manifestName,RegExp * zeroExp)1962 restoreFileZeroed(const String *manifestName, RegExp *zeroExp)
1963 {
1964 FUNCTION_TEST_BEGIN();
1965 FUNCTION_TEST_PARAM(STRING, manifestName);
1966 FUNCTION_TEST_PARAM(REGEXP, zeroExp);
1967 FUNCTION_TEST_END();
1968
1969 ASSERT(manifestName != NULL);
1970
1971 FUNCTION_TEST_RETURN(
1972 zeroExp == NULL ? false : regExpMatch(zeroExp, manifestName) && !strEndsWith(manifestName, STRDEF("/" PG_FILE_PGVERSION)));
1973 }
1974
1975 // Helper function to construct the absolute pg path for any file
1976 static String *
restoreFilePgPath(const Manifest * manifest,const String * manifestName)1977 restoreFilePgPath(const Manifest *manifest, const String *manifestName)
1978 {
1979 FUNCTION_TEST_BEGIN();
1980 FUNCTION_TEST_PARAM(MANIFEST, manifest);
1981 FUNCTION_TEST_PARAM(STRING, manifestName);
1982 FUNCTION_TEST_END();
1983
1984 ASSERT(manifest != NULL);
1985 ASSERT(manifestName != NULL);
1986
1987 String *result = strNewFmt("%s/%s", strZ(manifestTargetBase(manifest)->path), strZ(manifestPathPg(manifestName)));
1988
1989 if (strEq(manifestName, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
1990 result = strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strZ(result));
1991
1992 FUNCTION_TEST_RETURN(result);
1993 }
1994
1995 static uint64_t
restoreJobResult(const Manifest * manifest,ProtocolParallelJob * job,RegExp * zeroExp,uint64_t sizeTotal,uint64_t sizeRestored)1996 restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zeroExp, uint64_t sizeTotal, uint64_t sizeRestored)
1997 {
1998 FUNCTION_LOG_BEGIN(logLevelDebug);
1999 FUNCTION_LOG_PARAM(MANIFEST, manifest);
2000 FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, job);
2001 FUNCTION_LOG_PARAM(REGEXP, zeroExp);
2002 FUNCTION_LOG_PARAM(UINT64, sizeTotal);
2003 FUNCTION_LOG_PARAM(UINT64, sizeRestored);
2004 FUNCTION_LOG_END();
2005
2006 ASSERT(manifest != NULL);
2007
2008 // The job was successful
2009 if (protocolParallelJobErrorCode(job) == 0)
2010 {
2011 MEM_CONTEXT_TEMP_BEGIN()
2012 {
2013 const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
2014 bool zeroed = restoreFileZeroed(file->name, zeroExp);
2015 bool copy = pckReadBoolP(protocolParallelJobResult(job));
2016
2017 String *log = strNewZ("restore");
2018
2019 // Note if file was zeroed (i.e. selective restore)
2020 if (zeroed)
2021 strCatZ(log, " zeroed");
2022
2023 // Add filename
2024 strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file->name)));
2025
2026 // If not copied and not zeroed add details to explain why it was not copied
2027 if (!copy && !zeroed)
2028 {
2029 strCatZ(log, " - ");
2030
2031 // On force we match on size and modification time
2032 if (cfgOptionBool(cfgOptForce))
2033 {
2034 strCatFmt(
2035 log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file->size,
2036 (uint64_t)file->timestamp);
2037 }
2038 // Else a checksum delta or file is zero-length
2039 else
2040 {
2041 strCatZ(log, "exists and ");
2042
2043 // No need to copy zero-length files
2044 if (file->size == 0)
2045 {
2046 strCatZ(log, "is zero size");
2047 }
2048 // The file matched the manifest checksum so did not need to be copied
2049 else
2050 strCatZ(log, "matches backup");
2051 }
2052 }
2053
2054 // Add size and percent complete
2055 sizeRestored += file->size;
2056 strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file->size)), sizeRestored * 100 / sizeTotal);
2057
2058 // If not zero-length add the checksum
2059 if (file->size != 0 && !zeroed)
2060 strCatFmt(log, " checksum %s", file->checksumSha1);
2061
2062 LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
2063 }
2064 MEM_CONTEXT_TEMP_END();
2065
2066 // Free the job
2067 protocolParallelJobFree(job);
2068 }
2069 // Else the job errored
2070 else
2071 THROW_CODE(protocolParallelJobErrorCode(job), strZ(protocolParallelJobErrorMessage(job)));
2072
2073 FUNCTION_LOG_RETURN(UINT64, sizeRestored);
2074 }
2075
2076 /***********************************************************************************************************************************
2077 Return new restore jobs as requested
2078 ***********************************************************************************************************************************/
2079 typedef struct RestoreJobData
2080 {
2081 unsigned int repoIdx; // Internal repo idx
2082 Manifest *manifest; // Backup manifest
2083 List *queueList; // List of processing queues
2084 RegExp *zeroExp; // Identify files that should be sparse zeroed
2085 const String *cipherSubPass; // Passphrase used to decrypt files in the backup
2086 } RestoreJobData;
2087
2088 // Helper to caculate the next queue to scan based on the client index
2089 static int
restoreJobQueueNext(unsigned int clientIdx,int queueIdx,unsigned int queueTotal)2090 restoreJobQueueNext(unsigned int clientIdx, int queueIdx, unsigned int queueTotal)
2091 {
2092 FUNCTION_TEST_BEGIN();
2093 FUNCTION_TEST_PARAM(UINT, clientIdx);
2094 FUNCTION_TEST_PARAM(INT, queueIdx);
2095 FUNCTION_TEST_PARAM(UINT, queueTotal);
2096 FUNCTION_TEST_END();
2097
2098 // Move (forward or back) to the next queue
2099 queueIdx += clientIdx % 2 ? -1 : 1;
2100
2101 // Deal with wrapping on either end
2102 if (queueIdx < 0)
2103 FUNCTION_TEST_RETURN((int)queueTotal - 1);
2104 else if (queueIdx == (int)queueTotal)
2105 FUNCTION_TEST_RETURN(0);
2106
2107 FUNCTION_TEST_RETURN(queueIdx);
2108 }
2109
2110 // Callback to fetch restore jobs for the parallel executor
restoreJobCallback(void * data,unsigned int clientIdx)2111 static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientIdx)
2112 {
2113 FUNCTION_TEST_BEGIN();
2114 FUNCTION_TEST_PARAM_P(VOID, data);
2115 FUNCTION_TEST_PARAM(UINT, clientIdx);
2116 FUNCTION_TEST_END();
2117
2118 ASSERT(data != NULL);
2119
2120 ProtocolParallelJob *result = NULL;
2121
2122 MEM_CONTEXT_TEMP_BEGIN()
2123 {
2124 // Get a new job if there are any left
2125 RestoreJobData *jobData = data;
2126
2127 // Determine where to begin scanning the queue (we'll stop when we get back here)
2128 int queueIdx = (int)(clientIdx % lstSize(jobData->queueList));
2129 int queueEnd = queueIdx;
2130
2131 do
2132 {
2133 List *queue = *(List **)lstGet(jobData->queueList, (unsigned int)queueIdx);
2134
2135 if (!lstEmpty(queue))
2136 {
2137 const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0);
2138
2139 // Create restore job
2140 ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
2141 PackWrite *const param = protocolCommandParam(command);
2142
2143 pckWriteStrP(param, file->name);
2144 pckWriteU32P(param, jobData->repoIdx);
2145 pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
2146 pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
2147 pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
2148 pckWriteStrP(param, STR(file->checksumSha1));
2149 pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
2150 pckWriteU64P(param, file->size);
2151 pckWriteTimeP(param, file->timestamp);
2152 pckWriteModeP(param, file->mode);
2153 pckWriteStrP(param, file->user);
2154 pckWriteStrP(param, file->group);
2155 pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
2156 pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
2157 pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
2158 pckWriteStrP(param, jobData->cipherSubPass);
2159
2160 // Remove job from the queue
2161 lstRemoveIdx(queue, 0);
2162
2163 // Assign job to result
2164 MEM_CONTEXT_PRIOR_BEGIN()
2165 {
2166 result = protocolParallelJobNew(VARSTR(file->name), command);
2167 }
2168 MEM_CONTEXT_PRIOR_END();
2169
2170 // Break out of the loop early since we found a job
2171 break;
2172 }
2173
2174 queueIdx = restoreJobQueueNext(clientIdx, queueIdx, lstSize(jobData->queueList));
2175 }
2176 while (queueIdx != queueEnd);
2177 }
2178 MEM_CONTEXT_TEMP_END();
2179
2180 FUNCTION_TEST_RETURN(result);
2181 }
2182
2183 /**********************************************************************************************************************************/
2184 void
cmdRestore(void)2185 cmdRestore(void)
2186 {
2187 FUNCTION_LOG_VOID(logLevelDebug);
2188
2189 MEM_CONTEXT_TEMP_BEGIN()
2190 {
2191 // Get information for the current user
2192 userInit();
2193
2194 // PostgreSQL must be local
2195 pgIsLocalVerify();
2196
2197 // Validate restore path
2198 restorePathValidate();
2199
2200 // Remove stanza archive spool path so existing files do not interfere with the new cluster. For instance, old archive-push
2201 // acknowledgements could cause a new cluster to skip archiving. This should not happen if a new timeline is selected but
2202 // better to be safe. Missing stanza spool paths are ignored.
2203 storagePathRemoveP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_STR, .recurse = true);
2204
2205 // Get the backup set
2206 RestoreBackupData backupData = restoreBackupSet();
2207
2208 // Load manifest
2209 RestoreJobData jobData = {.repoIdx = backupData.repoIdx};
2210
2211 jobData.manifest = manifestLoadFile(
2212 storageRepoIdx(backupData.repoIdx),
2213 strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupData.backupSet)), backupData.repoCipherType,
2214 backupData.backupCipherPass);
2215
2216 // Remotes (if any) are no longer needed since the rest of the repository reads will be done by the local processes
2217 protocolFree();
2218
2219 // Validate manifest. Don't use strict mode because we'd rather ignore problems that won't affect a restore.
2220 manifestValidate(jobData.manifest, false);
2221
2222 // Get the cipher subpass used to decrypt files in the backup
2223 jobData.cipherSubPass = manifestCipherSubPass(jobData.manifest);
2224
2225 // Validate the manifest
2226 restoreManifestValidate(jobData.manifest, backupData.backupSet);
2227
2228 // Log the backup set to restore
2229 LOG_INFO_FMT(
2230 "repo%u: restore backup set %s", cfgOptionGroupIdxToKey(cfgOptGrpRepo, backupData.repoIdx), strZ(backupData.backupSet));
2231
2232 // Map manifest
2233 restoreManifestMap(jobData.manifest);
2234
2235 // Check that links are sane
2236 manifestLinkCheck(jobData.manifest);
2237
2238 // Update ownership
2239 restoreManifestOwner(jobData.manifest);
2240
2241 // Generate the selective restore expression
2242 String *expression = restoreSelectiveExpression(jobData.manifest);
2243 jobData.zeroExp = expression == NULL ? NULL : regExpNew(expression);
2244
2245 // Clean the data directory and build path/link structure
2246 restoreCleanBuild(jobData.manifest);
2247
2248 // Generate processing queues
2249 uint64_t sizeTotal = restoreProcessQueue(jobData.manifest, &jobData.queueList);
2250
2251 // Save manifest to the data directory so we can restart a delta restore even if the PG_VERSION file is missing
2252 manifestSave(jobData.manifest, storageWriteIo(storageNewWriteP(storagePgWrite(), BACKUP_MANIFEST_FILE_STR)));
2253
2254 // Create the parallel executor
2255 ProtocolParallel *parallelExec = protocolParallelNew(
2256 cfgOptionUInt64(cfgOptProtocolTimeout) / 2, restoreJobCallback, &jobData);
2257
2258 for (unsigned int processIdx = 1; processIdx <= cfgOptionUInt(cfgOptProcessMax); processIdx++)
2259 protocolParallelClientAdd(parallelExec, protocolLocalGet(protocolStorageTypeRepo, 0, processIdx));
2260
2261 // Process jobs
2262 uint64_t sizeRestored = 0;
2263
2264 MEM_CONTEXT_TEMP_RESET_BEGIN()
2265 {
2266 do
2267 {
2268 unsigned int completed = protocolParallelProcess(parallelExec);
2269
2270 for (unsigned int jobIdx = 0; jobIdx < completed; jobIdx++)
2271 {
2272 sizeRestored = restoreJobResult(
2273 jobData.manifest, protocolParallelResult(parallelExec), jobData.zeroExp, sizeTotal, sizeRestored);
2274 }
2275
2276 // Reset the memory context occasionally so we don't use too much memory or slow down processing
2277 MEM_CONTEXT_TEMP_RESET(1000);
2278 }
2279 while (!protocolParallelDone(parallelExec));
2280 }
2281 MEM_CONTEXT_TEMP_END();
2282
2283 // Write recovery settings
2284 restoreRecoveryWrite(jobData.manifest);
2285
2286 // Remove backup.manifest
2287 storageRemoveP(storagePgWrite(), BACKUP_MANIFEST_FILE_STR);
2288
2289 // Sync file link paths. These need to be synced separately because they are not linked from the data directory.
2290 StringList *pathSynced = strLstNew();
2291
2292 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(jobData.manifest); targetIdx++)
2293 {
2294 const ManifestTarget *target = manifestTarget(jobData.manifest, targetIdx);
2295
2296 if (target->type == manifestTargetTypeLink && target->file != NULL)
2297 {
2298 const String *pgPath = manifestTargetPath(jobData.manifest, target);
2299
2300 // Don't sync the same path twice. There can be multiple links to files in the same path, but syncing it more than
2301 // once makes the logs noisy and looks like a bug even though it doesn't hurt anything or realistically affect
2302 // performance.
2303 if (strLstExists(pathSynced, pgPath))
2304 continue;
2305 else
2306 strLstAdd(pathSynced, pgPath);
2307
2308 // Sync the path
2309 LOG_DETAIL_FMT("sync path '%s'", strZ(pgPath));
2310 storagePathSyncP(storageLocalWrite(), pgPath);
2311 }
2312 }
2313
2314 // Sync paths in the data directory
2315 for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(jobData.manifest); pathIdx++)
2316 {
2317 const String *manifestName = manifestPath(jobData.manifest, pathIdx)->name;
2318
2319 // Skip the pg_tblspc path because it only maps to the manifest. We should remove this in a future release but not much
2320 // can be done about it for now.
2321 if (strEqZ(manifestName, MANIFEST_TARGET_PGTBLSPC))
2322 continue;
2323
2324 // We'll sync global after pg_control is written
2325 if (strEq(manifestName, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL)))
2326 continue;
2327
2328 const String *pgPath = storagePathP(storagePg(), manifestPathPg(manifestName));
2329
2330 LOG_DETAIL_FMT("sync path '%s'", strZ(pgPath));
2331 storagePathSyncP(storagePgWrite(), pgPath);
2332 }
2333
2334 // Rename pg_control
2335 if (storageExistsP(storagePg(), STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL "." STORAGE_FILE_TEMP_EXT)))
2336 {
2337 LOG_INFO(
2338 "restore " PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL " (performed last to ensure aborted restores cannot be started)");
2339
2340 storageMoveP(
2341 storagePgWrite(),
2342 storageNewReadP(storagePg(), STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL "." STORAGE_FILE_TEMP_EXT)),
2343 storageNewWriteP(storagePgWrite(), STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .noSyncPath = true));
2344 }
2345 else
2346 LOG_WARN("backup does not contain '" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL "' -- cluster will not start");
2347
2348 // Sync global path
2349 LOG_DETAIL_FMT("sync path '%s'", strZ(storagePathP(storagePg(), PG_PATH_GLOBAL_STR)));
2350 storagePathSyncP(storagePgWrite(), PG_PATH_GLOBAL_STR);
2351
2352 // Restore info
2353 LOG_INFO_FMT(
2354 "restore size = %s, file total = %u", strZ(strSizeFormat(sizeRestored)), manifestFileTotal(jobData.manifest));
2355 }
2356 MEM_CONTEXT_TEMP_END();
2357
2358 FUNCTION_LOG_RETURN_VOID();
2359 }
2360