1 /***********************************************************************************************************************************
2 Backup Manifest Handler
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include <ctype.h>
7 #include <string.h>
8 #include <time.h>
9
10 #include "common/crypto/cipherBlock.h"
11 #include "common/debug.h"
12 #include "common/log.h"
13 #include "common/regExp.h"
14 #include "common/type/json.h"
15 #include "common/type/list.h"
16 #include "common/type/mcv.h"
17 #include "info/manifest.h"
18 #include "postgres/interface.h"
19 #include "postgres/version.h"
20 #include "storage/storage.h"
21 #include "version.h"
22
23 /***********************************************************************************************************************************
24 Constants
25 ***********************************************************************************************************************************/
26 STRING_EXTERN(BACKUP_MANIFEST_FILE_STR, BACKUP_MANIFEST_FILE);
27
28 STRING_EXTERN(MANIFEST_TARGET_PGDATA_STR, MANIFEST_TARGET_PGDATA);
29 STRING_EXTERN(MANIFEST_TARGET_PGTBLSPC_STR, MANIFEST_TARGET_PGTBLSPC);
30
31 STRING_STATIC(MANIFEST_TARGET_TYPE_LINK_STR, "link");
32 STRING_STATIC(MANIFEST_TARGET_TYPE_PATH_STR, "path");
33
34 STRING_STATIC(MANIFEST_SECTION_BACKUP_STR, "backup");
35 STRING_STATIC(MANIFEST_SECTION_BACKUP_DB_STR, "backup:db");
36 STRING_STATIC(MANIFEST_SECTION_BACKUP_OPTION_STR, "backup:option");
37 STRING_STATIC(MANIFEST_SECTION_BACKUP_TARGET_STR, "backup:target");
38
39 STRING_STATIC(MANIFEST_SECTION_DB_STR, "db");
40
41 STRING_STATIC(MANIFEST_SECTION_TARGET_FILE_STR, "target:file");
42 STRING_STATIC(MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, "target:file:default");
43
44 STRING_STATIC(MANIFEST_SECTION_TARGET_LINK_STR, "target:link");
45 STRING_STATIC(MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, "target:link:default");
46
47 STRING_STATIC(MANIFEST_SECTION_TARGET_PATH_STR, "target:path");
48 STRING_STATIC(MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, "target:path:default");
49
50 #define MANIFEST_KEY_BACKUP_ARCHIVE_START "backup-archive-start"
51 STRING_STATIC(MANIFEST_KEY_BACKUP_ARCHIVE_START_STR, MANIFEST_KEY_BACKUP_ARCHIVE_START);
52 #define MANIFEST_KEY_BACKUP_ARCHIVE_STOP "backup-archive-stop"
53 STRING_STATIC(MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_STOP);
54 #define MANIFEST_KEY_BACKUP_LABEL "backup-label"
55 STRING_STATIC(MANIFEST_KEY_BACKUP_LABEL_STR, MANIFEST_KEY_BACKUP_LABEL);
56 #define MANIFEST_KEY_BACKUP_LSN_START "backup-lsn-start"
57 STRING_STATIC(MANIFEST_KEY_BACKUP_LSN_START_STR, MANIFEST_KEY_BACKUP_LSN_START);
58 #define MANIFEST_KEY_BACKUP_LSN_STOP "backup-lsn-stop"
59 STRING_STATIC(MANIFEST_KEY_BACKUP_LSN_STOP_STR, MANIFEST_KEY_BACKUP_LSN_STOP);
60 #define MANIFEST_KEY_BACKUP_PRIOR "backup-prior"
61 STRING_STATIC(MANIFEST_KEY_BACKUP_PRIOR_STR, MANIFEST_KEY_BACKUP_PRIOR);
62 #define MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START "backup-timestamp-copy-start"
63 STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START);
64 #define MANIFEST_KEY_BACKUP_TIMESTAMP_START "backup-timestamp-start"
65 STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_START);
66 #define MANIFEST_KEY_BACKUP_TIMESTAMP_STOP "backup-timestamp-stop"
67 STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP);
68 #define MANIFEST_KEY_BACKUP_TYPE "backup-type"
69 STRING_STATIC(MANIFEST_KEY_BACKUP_TYPE_STR, MANIFEST_KEY_BACKUP_TYPE);
70 #define MANIFEST_KEY_CHECKSUM "checksum"
71 VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_VAR, MANIFEST_KEY_CHECKSUM);
72 #define MANIFEST_KEY_CHECKSUM_PAGE "checksum-page"
73 VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_PAGE_VAR, MANIFEST_KEY_CHECKSUM_PAGE);
74 #define MANIFEST_KEY_CHECKSUM_PAGE_ERROR "checksum-page-error"
75 VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, MANIFEST_KEY_CHECKSUM_PAGE_ERROR);
76 #define MANIFEST_KEY_DB_CATALOG_VERSION "db-catalog-version"
77 STRING_STATIC(MANIFEST_KEY_DB_CATALOG_VERSION_STR, MANIFEST_KEY_DB_CATALOG_VERSION);
78 #define MANIFEST_KEY_DB_ID "db-id"
79 STRING_STATIC(MANIFEST_KEY_DB_ID_STR, MANIFEST_KEY_DB_ID);
80 VARIANT_STRDEF_STATIC(MANIFEST_KEY_DB_ID_VAR, MANIFEST_KEY_DB_ID);
81 #define MANIFEST_KEY_DB_LAST_SYSTEM_ID "db-last-system-id"
82 VARIANT_STRDEF_STATIC(MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR, MANIFEST_KEY_DB_LAST_SYSTEM_ID);
83 #define MANIFEST_KEY_DB_SYSTEM_ID "db-system-id"
84 STRING_STATIC(MANIFEST_KEY_DB_SYSTEM_ID_STR, MANIFEST_KEY_DB_SYSTEM_ID);
85 #define MANIFEST_KEY_DB_VERSION "db-version"
86 STRING_STATIC(MANIFEST_KEY_DB_VERSION_STR, MANIFEST_KEY_DB_VERSION);
87 #define MANIFEST_KEY_DESTINATION "destination"
88 VARIANT_STRDEF_STATIC(MANIFEST_KEY_DESTINATION_VAR, MANIFEST_KEY_DESTINATION);
89 #define MANIFEST_KEY_FILE "file"
90 VARIANT_STRDEF_STATIC(MANIFEST_KEY_FILE_VAR, MANIFEST_KEY_FILE);
91 #define MANIFEST_KEY_GROUP "group"
92 STRING_STATIC(MANIFEST_KEY_GROUP_STR, MANIFEST_KEY_GROUP);
93 VARIANT_STRDEF_STATIC(MANIFEST_KEY_GROUP_VAR, MANIFEST_KEY_GROUP);
94 #define MANIFEST_KEY_PRIMARY "ma" "st" "er"
95 STRING_STATIC(MANIFEST_KEY_PRIMARY_STR, MANIFEST_KEY_PRIMARY);
96 VARIANT_STRDEF_STATIC(MANIFEST_KEY_PRIMARY_VAR, MANIFEST_KEY_PRIMARY);
97 #define MANIFEST_KEY_MODE "mode"
98 STRING_STATIC(MANIFEST_KEY_MODE_STR, MANIFEST_KEY_MODE);
99 VARIANT_STRDEF_STATIC(MANIFEST_KEY_MODE_VAR, MANIFEST_KEY_MODE);
100 #define MANIFEST_KEY_PATH "path"
101 VARIANT_STRDEF_STATIC(MANIFEST_KEY_PATH_VAR, MANIFEST_KEY_PATH);
102 #define MANIFEST_KEY_REFERENCE "reference"
103 VARIANT_STRDEF_STATIC(MANIFEST_KEY_REFERENCE_VAR, MANIFEST_KEY_REFERENCE);
104 #define MANIFEST_KEY_SIZE "size"
105 VARIANT_STRDEF_STATIC(MANIFEST_KEY_SIZE_VAR, MANIFEST_KEY_SIZE);
106 #define MANIFEST_KEY_SIZE_REPO "repo-size"
107 VARIANT_STRDEF_STATIC(MANIFEST_KEY_SIZE_REPO_VAR, MANIFEST_KEY_SIZE_REPO);
108 #define MANIFEST_KEY_TABLESPACE_ID "tablespace-id"
109 VARIANT_STRDEF_STATIC(MANIFEST_KEY_TABLESPACE_ID_VAR, MANIFEST_KEY_TABLESPACE_ID);
110 #define MANIFEST_KEY_TABLESPACE_NAME "tablespace-name"
111 VARIANT_STRDEF_STATIC(MANIFEST_KEY_TABLESPACE_NAME_VAR, MANIFEST_KEY_TABLESPACE_NAME);
112 #define MANIFEST_KEY_TIMESTAMP "timestamp"
113 VARIANT_STRDEF_STATIC(MANIFEST_KEY_TIMESTAMP_VAR, MANIFEST_KEY_TIMESTAMP);
114 #define MANIFEST_KEY_TYPE "type"
115 VARIANT_STRDEF_STATIC(MANIFEST_KEY_TYPE_VAR, MANIFEST_KEY_TYPE);
116 #define MANIFEST_KEY_USER "user"
117 STRING_STATIC(MANIFEST_KEY_USER_STR, MANIFEST_KEY_USER);
118 VARIANT_STRDEF_STATIC(MANIFEST_KEY_USER_VAR, MANIFEST_KEY_USER);
119
120 #define MANIFEST_KEY_OPTION_ARCHIVE_CHECK "option-archive-check"
121 STRING_STATIC(MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR, MANIFEST_KEY_OPTION_ARCHIVE_CHECK);
122 #define MANIFEST_KEY_OPTION_ARCHIVE_COPY "option-archive-copy"
123 STRING_STATIC(MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR, MANIFEST_KEY_OPTION_ARCHIVE_COPY);
124 #define MANIFEST_KEY_OPTION_BACKUP_STANDBY "option-backup-standby"
125 STRING_STATIC(MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR, MANIFEST_KEY_OPTION_BACKUP_STANDBY);
126 #define MANIFEST_KEY_OPTION_BUFFER_SIZE "option-buffer-size"
127 STRING_STATIC(MANIFEST_KEY_OPTION_BUFFER_SIZE_STR, MANIFEST_KEY_OPTION_BUFFER_SIZE);
128 #define MANIFEST_KEY_OPTION_CHECKSUM_PAGE "option-checksum-page"
129 STRING_STATIC(MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR, MANIFEST_KEY_OPTION_CHECKSUM_PAGE);
130 #define MANIFEST_KEY_OPTION_COMPRESS "option-compress"
131 STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_STR, MANIFEST_KEY_OPTION_COMPRESS);
132 #define MANIFEST_KEY_OPTION_COMPRESS_TYPE "option-compress-type"
133 STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR, MANIFEST_KEY_OPTION_COMPRESS_TYPE);
134 #define MANIFEST_KEY_OPTION_COMPRESS_LEVEL "option-compress-level"
135 STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL);
136 #define MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK "option-compress-level-network"
137 STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK);
138 #define MANIFEST_KEY_OPTION_DELTA "option-delta"
139 STRING_STATIC(MANIFEST_KEY_OPTION_DELTA_STR, MANIFEST_KEY_OPTION_DELTA);
140 #define MANIFEST_KEY_OPTION_HARDLINK "option-hardlink"
141 STRING_STATIC(MANIFEST_KEY_OPTION_HARDLINK_STR, MANIFEST_KEY_OPTION_HARDLINK);
142 #define MANIFEST_KEY_OPTION_ONLINE "option-online"
143 STRING_STATIC(MANIFEST_KEY_OPTION_ONLINE_STR, MANIFEST_KEY_OPTION_ONLINE);
144 #define MANIFEST_KEY_OPTION_PROCESS_MAX "option-process-max"
145 STRING_STATIC(MANIFEST_KEY_OPTION_PROCESS_MAX_STR, MANIFEST_KEY_OPTION_PROCESS_MAX);
146
147 /***********************************************************************************************************************************
148 Object type
149 ***********************************************************************************************************************************/
150 struct Manifest
151 {
152 ManifestPub pub; // Publicly accessible variables
153 StringList *ownerList; // List of users/groups
154 StringList *referenceList; // List of file references
155 };
156
157 /***********************************************************************************************************************************
158 Internal functions to add types to their lists
159 ***********************************************************************************************************************************/
160 // Helper to add owner to the owner list if it is not there already and return the pointer. This saves a lot of space.
161 static const String *
manifestOwnerCache(Manifest * this,const String * owner)162 manifestOwnerCache(Manifest *this, const String *owner)
163 {
164 FUNCTION_TEST_BEGIN();
165 FUNCTION_TEST_PARAM(MANIFEST, this);
166 FUNCTION_TEST_PARAM(STRING, owner);
167 FUNCTION_TEST_END();
168
169 ASSERT(this != NULL);
170
171 if (owner != NULL)
172 FUNCTION_TEST_RETURN(strLstAddIfMissing(this->ownerList, owner));
173
174 FUNCTION_TEST_RETURN(NULL);
175 }
176
177 static void
manifestDbAdd(Manifest * this,const ManifestDb * db)178 manifestDbAdd(Manifest *this, const ManifestDb *db)
179 {
180 FUNCTION_TEST_BEGIN();
181 FUNCTION_TEST_PARAM(MANIFEST, this);
182 FUNCTION_TEST_PARAM(MANIFEST_DB, db);
183 FUNCTION_TEST_END();
184
185 ASSERT(this != NULL);
186 ASSERT(db != NULL);
187 ASSERT(db->name != NULL);
188
189 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.dbList))
190 {
191 ManifestDb dbAdd =
192 {
193 .id = db->id,
194 .lastSystemId = db->lastSystemId,
195 .name = strDup(db->name),
196 };
197
198 lstAdd(this->pub.dbList, &dbAdd);
199 }
200 MEM_CONTEXT_END();
201
202 FUNCTION_TEST_RETURN_VOID();
203 }
204
205 void
manifestFileAdd(Manifest * this,const ManifestFile * file)206 manifestFileAdd(Manifest *this, const ManifestFile *file)
207 {
208 FUNCTION_TEST_BEGIN();
209 FUNCTION_TEST_PARAM(MANIFEST, this);
210 FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
211 FUNCTION_TEST_END();
212
213 ASSERT(this != NULL);
214 ASSERT(file != NULL);
215 ASSERT(file->name != NULL);
216
217 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
218 {
219 ManifestFile fileAdd =
220 {
221 .checksumPage = file->checksumPage,
222 .checksumPageError = file->checksumPageError,
223 .checksumPageErrorList = varLstDup(file->checksumPageErrorList),
224 .group = manifestOwnerCache(this, file->group),
225 .mode = file->mode,
226 .name = strDup(file->name),
227 .primary = file->primary,
228 .size = file->size,
229 .sizeRepo = file->sizeRepo,
230 .timestamp = file->timestamp,
231 .user = manifestOwnerCache(this, file->user),
232 };
233
234 memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
235
236 if (file->reference != NULL)
237 {
238 // Search for the reference in the list
239 for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++)
240 {
241 const String *found = strLstGet(this->referenceList, referenceIdx);
242
243 if (strEq(file->reference, found))
244 {
245 fileAdd.reference = found;
246 break;
247 }
248 }
249
250 // If not found then add it
251 if (fileAdd.reference == NULL)
252 {
253 strLstAdd(this->referenceList, file->reference);
254 fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1);
255 }
256 }
257
258 lstAdd(this->pub.fileList, &fileAdd);
259 }
260 MEM_CONTEXT_END();
261
262 FUNCTION_TEST_RETURN_VOID();
263 }
264
265 static void
manifestLinkAdd(Manifest * this,const ManifestLink * link)266 manifestLinkAdd(Manifest *this, const ManifestLink *link)
267 {
268 FUNCTION_TEST_BEGIN();
269 FUNCTION_TEST_PARAM(MANIFEST, this);
270 FUNCTION_TEST_PARAM(MANIFEST_LINK, link);
271 FUNCTION_TEST_END();
272
273 ASSERT(this != NULL);
274 ASSERT(link != NULL);
275 ASSERT(link->name != NULL);
276 ASSERT(link->destination != NULL);
277
278 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.linkList))
279 {
280 ManifestLink linkAdd =
281 {
282 .destination = strDup(link->destination),
283 .name = strDup(link->name),
284 .group = manifestOwnerCache(this, link->group),
285 .user = manifestOwnerCache(this, link->user),
286 };
287
288 lstAdd(this->pub.linkList, &linkAdd);
289 }
290 MEM_CONTEXT_END();
291
292 FUNCTION_TEST_RETURN_VOID();
293 }
294
295 static void
manifestPathAdd(Manifest * this,const ManifestPath * path)296 manifestPathAdd(Manifest *this, const ManifestPath *path)
297 {
298 FUNCTION_TEST_BEGIN();
299 FUNCTION_TEST_PARAM(MANIFEST, this);
300 FUNCTION_TEST_PARAM(MANIFEST_PATH, path);
301 FUNCTION_TEST_END();
302
303 ASSERT(this != NULL);
304 ASSERT(path != NULL);
305 ASSERT(path->name != NULL);
306
307 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.pathList))
308 {
309 ManifestPath pathAdd =
310 {
311 .mode = path->mode,
312 .name = strDup(path->name),
313 .group = manifestOwnerCache(this, path->group),
314 .user = manifestOwnerCache(this, path->user),
315 };
316
317 lstAdd(this->pub.pathList, &pathAdd);
318 }
319 MEM_CONTEXT_END();
320
321 FUNCTION_TEST_RETURN_VOID();
322 }
323
324 static void
manifestTargetAdd(Manifest * this,const ManifestTarget * target)325 manifestTargetAdd(Manifest *this, const ManifestTarget *target)
326 {
327 FUNCTION_TEST_BEGIN();
328 FUNCTION_TEST_PARAM(MANIFEST, this);
329 FUNCTION_TEST_PARAM(MANIFEST_TARGET, target);
330 FUNCTION_TEST_END();
331
332 ASSERT(this != NULL);
333 ASSERT(target != NULL);
334 ASSERT(target->path != NULL);
335 ASSERT(target->name != NULL);
336
337 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.targetList))
338 {
339 ManifestTarget targetAdd =
340 {
341 .file = strDup(target->file),
342 .name = strDup(target->name),
343 .path = strDup(target->path),
344 .tablespaceId = target->tablespaceId,
345 .tablespaceName = strDup(target->tablespaceName),
346 .type = target->type,
347 };
348
349 lstAdd(this->pub.targetList, &targetAdd);
350 }
351 MEM_CONTEXT_END();
352
353 FUNCTION_TEST_RETURN_VOID();
354 }
355
356 /***********************************************************************************************************************************
357 Internal constructor
358 ***********************************************************************************************************************************/
359 static Manifest *
manifestNewInternal(void)360 manifestNewInternal(void)
361 {
362 FUNCTION_TEST_VOID();
363
364 Manifest *this = memNew(sizeof(Manifest));
365
366 *this = (Manifest)
367 {
368 .pub =
369 {
370 .memContext = memContextCurrent(),
371 .dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr),
372 .fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr),
373 .linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
374 .pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
375 .targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
376 },
377 .ownerList = strLstNew(),
378 .referenceList = strLstNew(),
379 };
380
381 FUNCTION_TEST_RETURN(this);
382 }
383
384 /***********************************************************************************************************************************
385 Ensure that symlinks do not point to the same file, directory, or subdirectory of another link
386
387 There are two implementations: manifestLinkCheck(), which is externed, and manifestLinkCheckOne(), which is intended to be
388 used internally during processing. manifestLinkCheck() works simply by calling manifestLinkCheckOne() for every link in the target
389 list. manifestLinkCheckOne() is optimized to work quickly on a single link.
390 ***********************************************************************************************************************************/
391 // Data needed in link list
392 typedef struct ManifestLinkCheckItem
393 {
394 const String *path; // Link destination path terminated with /
395 const String *file; // Link file if a file link
396 unsigned int targetIdx; // Index of target used for error messages
397 } ManifestLinkCheckItem;
398
399 // Persistent data needed during processing of manifestLinkCheck/One()
400 typedef struct ManifestLinkCheck
401 {
402 const String *basePath; // Base data path (initialized on first call)
403 List *linkList; // Current list of link destination paths
404 } ManifestLinkCheck;
405
406 // Helper to initialize the link data
407 static ManifestLinkCheck
manifestLinkCheckInit(void)408 manifestLinkCheckInit(void)
409 {
410 FUNCTION_TEST_VOID();
411 FUNCTION_TEST_RETURN((ManifestLinkCheck){.linkList = lstNewP(sizeof(ManifestLinkCheckItem), .comparator = lstComparatorStr)});
412 }
413
414 // Helper to check a single link specified by targetIdx
415 static void
manifestLinkCheckOne(const Manifest * this,ManifestLinkCheck * linkCheck,unsigned int targetIdx)416 manifestLinkCheckOne(const Manifest *this, ManifestLinkCheck *linkCheck, unsigned int targetIdx)
417 {
418 FUNCTION_LOG_BEGIN(logLevelTrace);
419 FUNCTION_LOG_PARAM(MANIFEST, this);
420 FUNCTION_LOG_PARAM_P(VOID, linkCheck);
421 FUNCTION_LOG_PARAM(UINT, targetIdx);
422 FUNCTION_LOG_END();
423
424 ASSERT(this != NULL);
425 ASSERT(linkCheck != NULL);
426 ASSERT(linkCheck->linkList != NULL);
427 ASSERT(targetIdx < manifestTargetTotal(this));
428
429 MEM_CONTEXT_TEMP_BEGIN()
430 {
431 const ManifestTarget *target1 = manifestTarget(this, targetIdx);
432
433 // Only check link targets
434 if (target1->type == manifestTargetTypeLink)
435 {
436 // Create link destination path for comparison with other paths. It must end in / so subpaths can be detected without
437 // matching valid partial path names at the end of the path.
438 const String *path = NULL;
439
440 MEM_CONTEXT_BEGIN(lstMemContext(linkCheck->linkList))
441 {
442 path = strNewFmt("%s/", strZ(manifestTargetPath(this, target1)));
443
444 // Get base bath
445 if (linkCheck->basePath == NULL)
446 {
447 linkCheck->basePath = strNewFmt(
448 "%s/", strZ(manifestTargetPath(this, manifestTargetFind(this, MANIFEST_TARGET_PGDATA_STR))));
449 }
450 }
451 MEM_CONTEXT_END();
452
453 // Check that link destination is not in base data path
454 if (strBeginsWith(path, linkCheck->basePath))
455 {
456 THROW_FMT(
457 LinkDestinationError,
458 "link '%s' destination '%s' is in PGDATA",
459 strZ(manifestPathPg(target1->name)), strZ(manifestTargetPath(this, target1)));
460 }
461
462 // Check if the link destination path already exists
463 const ManifestLinkCheckItem *const link = lstFind(linkCheck->linkList, &path);
464
465 if (link != NULL)
466 {
467 // If both links are files make sure they don't link to the same file
468 if (target1->file != NULL && link->file != NULL)
469 {
470 if (strEq(target1->file, link->file))
471 {
472 const ManifestTarget *const target2 = manifestTarget(this, link->targetIdx);
473
474 THROW_FMT(
475 LinkDestinationError,
476 "link '%s' (%s/%s) destination is the same file as link '%s' (%s/%s)",
477 strZ(manifestPathPg(target1->name)), strZ(manifestTargetPath(this, target1)), strZ(target1->file),
478 strZ(manifestPathPg(target2->name)), strZ(manifestTargetPath(this, target2)), strZ(target2->file));
479 }
480 }
481 // Else error because one of the links is a path and cannot link to the same path as another file/path link
482 else
483 {
484 const ManifestTarget *const target2 = manifestTarget(this, link->targetIdx);
485
486 THROW_FMT(
487 LinkDestinationError,
488 "link '%s' (%s) destination is the same directory as link '%s' (%s)",
489 strZ(manifestPathPg(target1->name)), strZ(manifestTargetPath(this, target1)),
490 strZ(manifestPathPg(target2->name)), strZ(manifestTargetPath(this, target2)));
491 }
492 }
493 // Else add to the link list and check against other links
494 else
495 {
496 // Add the link destination path and sort
497 lstAdd(linkCheck->linkList, &(ManifestLinkCheckItem){.path = path, .file = target1->file, .targetIdx = targetIdx});
498 lstSort(linkCheck->linkList, sortOrderAsc);
499
500 // Find the path in the sorted list
501 unsigned int linkIdx = lstFindIdx(linkCheck->linkList, &path);
502 ASSERT(linkIdx != LIST_NOT_FOUND);
503
504 // Check path links against other links (file links have already been checked)
505 if (target1->file == NULL)
506 {
507 // Check the link destination path to be sure it is not a subpath of a prior link destination path
508 for (unsigned int priorLinkIdx = linkIdx - 1; (int)priorLinkIdx >= 0; priorLinkIdx--)
509 {
510 const ManifestLinkCheckItem *const priorLink = lstGet(linkCheck->linkList, priorLinkIdx);
511
512 // Skip file links since they are allowed to be in the same path with each other and in the parent path of a
513 // linked destination path.
514 if (priorLink->file != NULL)
515 continue;
516
517 if (strBeginsWith(path, priorLink->path))
518 {
519 const ManifestTarget *const target2 = manifestTarget(this, priorLink->targetIdx);
520
521 THROW_FMT(
522 LinkDestinationError,
523 "link '%s' (%s) destination is a subdirectory of link '%s' (%s)",
524 strZ(manifestPathPg(target1->name)), strZ(manifestTargetPath(this, target1)),
525 strZ(manifestPathPg(target2->name)), strZ(manifestTargetPath(this, target2)));
526 }
527
528 // Stop once the first prior path link has been checked since it must be a parent (if there is one)
529 break;
530 }
531
532 // Check the link destination path to be sure it is not a parent path of a subsequent link destination path
533 for (unsigned int nextLinkIdx = linkIdx + 1; nextLinkIdx < lstSize(linkCheck->linkList); nextLinkIdx++)
534 {
535 const ManifestLinkCheckItem *const nextLink = lstGet(linkCheck->linkList, nextLinkIdx);
536
537 // Skip file links since they are allowed to be in the same path with each other and in the parent path of a
538 // linked destination path.
539 if (nextLink->file != NULL)
540 continue;
541
542 if (strBeginsWith(nextLink->path, path))
543 {
544 const ManifestTarget *const target2 = manifestTarget(this, nextLink->targetIdx);
545
546 THROW_FMT(
547 LinkDestinationError,
548 "link '%s' (%s) destination is a subdirectory of link '%s' (%s)",
549 strZ(manifestPathPg(target2->name)), strZ(manifestTargetPath(this, target2)),
550 strZ(manifestPathPg(target1->name)), strZ(manifestTargetPath(this, target1)));
551 }
552
553 // Stop once the first next path link has been checked since it must be a subpath (if there is one)
554 break;
555 }
556 }
557 }
558 }
559 }
560 MEM_CONTEXT_TEMP_END();
561
562 FUNCTION_LOG_RETURN_VOID();
563 }
564
565 void
manifestLinkCheck(const Manifest * this)566 manifestLinkCheck(const Manifest *this)
567 {
568 FUNCTION_LOG_BEGIN(logLevelDebug);
569 FUNCTION_LOG_PARAM(MANIFEST, this);
570 FUNCTION_LOG_END();
571
572 ASSERT(this != NULL);
573
574 MEM_CONTEXT_TEMP_BEGIN()
575 {
576 // Check all links
577 ManifestLinkCheck linkCheck = manifestLinkCheckInit();
578
579 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(this); targetIdx++)
580 manifestLinkCheckOne(this, &linkCheck, targetIdx);
581 }
582 MEM_CONTEXT_TEMP_END();
583
584 FUNCTION_LOG_RETURN_VOID();
585 }
586
587 /**********************************************************************************************************************************/
588 typedef struct ManifestBuildData
589 {
590 Manifest *manifest;
591 const Storage *storagePg;
592 const String *tablespaceId; // Tablespace id if PostgreSQL version has one
593 bool online; // Is this an online backup?
594 bool checksumPage; // Are page checksums being checked?
595 const String *manifestWalName; // Wal manifest name for this version of PostgreSQL
596 RegExp *dbPathExp; // Identify paths containing relations
597 RegExp *tempRelationExp; // Identify temp relations
598 RegExp *standbyExp; // Identify files that must be copied from the primary
599 const VariantList *tablespaceList; // List of tablespaces in the database
600 ManifestLinkCheck linkCheck; // List of links found during build (used for prefix check)
601 StringList *excludeContent; // Exclude contents of directories
602 StringList *excludeSingle; // Exclude a single file/link/path
603
604 // These change with each level of recursion
605 const String *manifestParentName; // Manifest name of this file/link/path's parent
606 const String *pgPath; // Current path in the PostgreSQL data directory
607 bool dbPath; // Does this path contain relations?
608 } ManifestBuildData;
609
610 // Callback to process files/links/paths and add them to the manifest
611 static void
manifestBuildCallback(void * data,const StorageInfo * info)612 manifestBuildCallback(void *data, const StorageInfo *info)
613 {
614 FUNCTION_TEST_BEGIN();
615 FUNCTION_TEST_PARAM_P(VOID, data);
616 FUNCTION_TEST_PARAM(STORAGE_INFO, *storageInfo);
617 FUNCTION_TEST_END();
618
619 ASSERT(data != NULL);
620 ASSERT(info != NULL);
621
622 // Skip all . paths because they have already been recorded on the previous level of recursion
623 if (strEq(info->name, DOT_STR))
624 {
625 FUNCTION_TEST_RETURN_VOID();
626 return;
627 }
628
629 // Skip any path/file/link that begins with pgsql_tmp. The files are removed when the server is restarted and the directories
630 // are recreated.
631 if (strBeginsWithZ(info->name, PG_PREFIX_PGSQLTMP))
632 {
633 FUNCTION_TEST_RETURN_VOID();
634 return;
635 }
636
637 // Get build data
638 ManifestBuildData buildData = *(ManifestBuildData *)data;
639 unsigned int pgVersion = buildData.manifest->pub.data.pgVersion;
640
641 // Contruct the name used to identify this file/link/path in the manifest
642 const String *manifestName = strNewFmt("%s/%s", strZ(buildData.manifestParentName), strZ(info->name));
643
644 // Skip excluded files/links/paths
645 if (buildData.excludeSingle != NULL && strLstExists(buildData.excludeSingle, manifestName))
646 {
647 LOG_INFO_FMT(
648 "exclude '%s/%s' from backup using '%s' exclusion", strZ(buildData.pgPath), strZ(info->name),
649 strZ(strSub(manifestName, sizeof(MANIFEST_TARGET_PGDATA))));
650
651 FUNCTION_TEST_RETURN_VOID();
652 return;
653 }
654
655 // Process storage types
656 switch (info->type)
657 {
658 // Add paths
659 // -------------------------------------------------------------------------------------------------------------------------
660 case storageTypePath:
661 {
662 // There should not be any paths in pg_tblspc
663 if (strEqZ(buildData.manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
664 {
665 THROW_FMT(
666 LinkExpectedError, "'%s' is not a symlink - " MANIFEST_TARGET_PGTBLSPC " should contain only symlinks",
667 strZ(manifestName));
668 }
669
670 // Add path to manifest
671 ManifestPath path =
672 {
673 .name = manifestName,
674 .mode = info->mode,
675 .user = info->user,
676 .group = info->group,
677 };
678
679 manifestPathAdd(buildData.manifest, &path);
680
681 // Skip excluded path content
682 if (buildData.excludeContent != NULL && strLstExists(buildData.excludeContent, manifestName))
683 {
684 LOG_INFO_FMT(
685 "exclude contents of '%s/%s' from backup using '%s/' exclusion", strZ(buildData.pgPath), strZ(info->name),
686 strZ(strSub(manifestName, sizeof(MANIFEST_TARGET_PGDATA))));
687
688 FUNCTION_TEST_RETURN_VOID();
689 return;
690 }
691
692 // Skip the contents of these paths if they exist in the base path since they won't be reused after recovery
693 if (strEq(buildData.manifestParentName, MANIFEST_TARGET_PGDATA_STR))
694 {
695 // Skip pg_dynshmem/* since these files cannot be reused on recovery
696 if (strEqZ(info->name, PG_PATH_PGDYNSHMEM) && pgVersion >= PG_VERSION_94)
697 {
698 FUNCTION_TEST_RETURN_VOID();
699 return;
700 }
701
702 // Skip pg_notify/* since these files cannot be reused on recovery
703 if (strEqZ(info->name, PG_PATH_PGNOTIFY) && pgVersion >= PG_VERSION_90)
704 {
705 FUNCTION_TEST_RETURN_VOID();
706 return;
707 }
708
709 // Skip pg_replslot/* since these files are generally not useful after a restore
710 if (strEqZ(info->name, PG_PATH_PGREPLSLOT) && pgVersion >= PG_VERSION_94)
711 {
712 FUNCTION_TEST_RETURN_VOID();
713 return;
714 }
715
716 // Skip pg_serial/* since these files are reset
717 if (strEqZ(info->name, PG_PATH_PGSERIAL) && pgVersion >= PG_VERSION_91)
718 {
719 FUNCTION_TEST_RETURN_VOID();
720 return;
721 }
722
723 // Skip pg_snapshots/* since these files cannot be reused on recovery
724 if (strEqZ(info->name, PG_PATH_PGSNAPSHOTS) && pgVersion >= PG_VERSION_92)
725 {
726 FUNCTION_TEST_RETURN_VOID();
727 return;
728 }
729
730 // Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always
731 // created there
732 if (strEqZ(info->name, PG_PATH_PGSTATTMP) && pgVersion >= PG_VERSION_84)
733 {
734 FUNCTION_TEST_RETURN_VOID();
735 return;
736 }
737
738 // Skip pg_subtrans/* since these files are reset
739 if (strEqZ(info->name, PG_PATH_PGSUBTRANS))
740 {
741 FUNCTION_TEST_RETURN_VOID();
742 return;
743 }
744 }
745
746 // Skip the contents of archive_status when online
747 if (buildData.online && strEq(buildData.manifestParentName, buildData.manifestWalName) &&
748 strEqZ(info->name, PG_PATH_ARCHIVE_STATUS))
749 {
750 FUNCTION_TEST_RETURN_VOID();
751 return;
752 }
753
754 // Recurse into the path
755 ManifestBuildData buildDataSub = buildData;
756 buildDataSub.manifestParentName = manifestName;
757 buildDataSub.pgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(info->name));
758
759 if (buildData.dbPathExp != NULL)
760 buildDataSub.dbPath = regExpMatch(buildData.dbPathExp, manifestName);
761
762 storageInfoListP(
763 buildDataSub.storagePg, buildDataSub.pgPath, manifestBuildCallback, &buildDataSub, .sortOrder = sortOrderAsc);
764
765 break;
766 }
767
768 // Add files
769 // -------------------------------------------------------------------------------------------------------------------------
770 case storageTypeFile:
771 {
772 // There should not be any files in pg_tblspc
773 if (strEqZ(buildData.manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
774 {
775 THROW_FMT(
776 LinkExpectedError, "'%s' is not a symlink - " MANIFEST_TARGET_PGTBLSPC " should contain only symlinks",
777 strZ(manifestName));
778 }
779
780 // Skip pg_internal.init since it is recreated on startup. It's also possible, (though unlikely) that a temp file with
781 // the creating process id as the extension can exist so skip that as well. This seems to be a bug in PostgreSQL since
782 // the temp file should be removed on startup. Use regExpMatchOne() here instead of preparing a regexp in advance since
783 // the likelihood of needing the regexp should be very small.
784 if ((pgVersion <= PG_VERSION_84 || buildData.dbPath) && strBeginsWithZ(info->name, PG_FILE_PGINTERNALINIT) &&
785 (strSize(info->name) == sizeof(PG_FILE_PGINTERNALINIT) - 1 ||
786 regExpMatchOne(STRDEF("\\.[0-9]+"), strSub(info->name, sizeof(PG_FILE_PGINTERNALINIT) - 1))))
787 {
788 FUNCTION_TEST_RETURN_VOID();
789 return;
790 }
791
792 // Skip files in the root data path
793 if (strEq(buildData.manifestParentName, MANIFEST_TARGET_PGDATA_STR))
794 {
795 // Skip recovery files
796 if (((strEqZ(info->name, PG_FILE_RECOVERYSIGNAL) || strEqZ(info->name, PG_FILE_STANDBYSIGNAL)) &&
797 pgVersion >= PG_VERSION_12) ||
798 ((strEqZ(info->name, PG_FILE_RECOVERYCONF) || strEqZ(info->name, PG_FILE_RECOVERYDONE)) &&
799 pgVersion < PG_VERSION_12) ||
800 // Skip temp file for safely writing postgresql.auto.conf
801 (strEqZ(info->name, PG_FILE_POSTGRESQLAUTOCONFTMP) && pgVersion >= PG_VERSION_94) ||
802 // Skip backup_label in versions where non-exclusive backup is supported
803 (strEqZ(info->name, PG_FILE_BACKUPLABEL) && pgVersion >= PG_VERSION_96) ||
804 // Skip old backup labels
805 strEqZ(info->name, PG_FILE_BACKUPLABELOLD) ||
806 // Skip backup_manifest/tmp in versions where it is created
807 ((strEqZ(info->name, PG_FILE_BACKUPMANIFEST) || strEqZ(info->name, PG_FILE_BACKUPMANIFEST_TMP)) &&
808 pgVersion >= PG_VERSION_13) ||
809 // Skip running process options
810 strEqZ(info->name, PG_FILE_POSTMASTEROPTS) ||
811 // Skip process id file to avoid confusing postgres after restore
812 strEqZ(info->name, PG_FILE_POSTMASTERPID))
813 {
814 FUNCTION_TEST_RETURN_VOID();
815 return;
816 }
817 }
818
819 // Skip the contents of the wal path when online. WAL will be restored from the archive or stored in the wal directory
820 // at the end of the backup if the archive-copy option is set.
821 if (buildData.online && strEq(buildData.manifestParentName, buildData.manifestWalName))
822 {
823 FUNCTION_TEST_RETURN_VOID();
824 return;
825 }
826
827 // Skip temp relations in db paths
828 if (buildData.dbPath && regExpMatch(buildData.tempRelationExp, info->name))
829 {
830 FUNCTION_TEST_RETURN_VOID();
831 return;
832 }
833
834 // Add file to manifest
835 ManifestFile file =
836 {
837 .name = manifestName,
838 .mode = info->mode,
839 .user = info->user,
840 .group = info->group,
841 .size = info->size,
842 .sizeRepo = info->size,
843 .timestamp = info->timeModified,
844 };
845
846 // Set a flag to indicate if this file must be copied from the primary
847 file.primary =
848 strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
849 !regExpMatch(buildData.standbyExp, manifestName);
850
851 // Determine if this file should be page checksummed
852 if (buildData.dbPath && buildData.checksumPage)
853 {
854 file.checksumPage =
855 !strEndsWithZ(manifestName, "/" PG_FILE_PGFILENODEMAP) && !strEndsWithZ(manifestName, "/" PG_FILE_PGVERSION) &&
856 !strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
857 }
858
859 manifestFileAdd(buildData.manifest, &file);
860 break;
861 }
862
863 // Add links
864 // -------------------------------------------------------------------------------------------------------------------------
865 case storageTypeLink:
866 {
867 // If the destination is another link then error. In the future we'll allow this by following the link chain to the
868 // eventual destination but for now we are trying to maintain compatibility during the migration. To do this check we
869 // need to read outside of the data directory but it is a read-only operation so is considered safe.
870 const String *linkDestinationAbsolute = strPathAbsolute(info->linkDestination, buildData.pgPath);
871
872 StorageInfo linkedCheck = storageInfoP(
873 buildData.storagePg, linkDestinationAbsolute, .ignoreMissing = true, .noPathEnforce = true);
874
875 if (linkedCheck.exists && linkedCheck.type == storageTypeLink)
876 {
877 THROW_FMT(
878 LinkDestinationError, "link '%s/%s' cannot reference another link '%s'", strZ(buildData.pgPath),
879 strZ(info->name), strZ(linkDestinationAbsolute));
880 }
881
882 // Initialize link and target
883 ManifestLink link =
884 {
885 .name = manifestName,
886 .user = info->user,
887 .group = info->group,
888 .destination = info->linkDestination,
889 };
890
891 ManifestTarget target =
892 {
893 .name = manifestName,
894 .type = manifestTargetTypeLink,
895 };
896
897 // Make a copy of the link name because it will need to be modified when there are tablespace ids
898 const String *linkName = info->name;
899
900 // Is this a tablespace?
901 if (strEq(buildData.manifestParentName, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC)))
902 {
903 // Strip pg_data off the manifest name so it begins with pg_tblspc instead. This reflects how the files are stored
904 // in the backup directory.
905 manifestName = strSub(manifestName, sizeof(MANIFEST_TARGET_PGDATA));
906
907 // Identify this target as a tablespace
908 target.name = manifestName;
909 target.tablespaceId = cvtZToUInt(strZ(info->name));
910
911 // Look for this tablespace in the provided list (list may be null for off-line backup)
912 if (buildData.tablespaceList != NULL)
913 {
914 // Search list
915 for (unsigned int tablespaceIdx = 0; tablespaceIdx < varLstSize(buildData.tablespaceList); tablespaceIdx++)
916 {
917 const VariantList *tablespace = varVarLst(varLstGet(buildData.tablespaceList, tablespaceIdx));
918
919 if (target.tablespaceId == varUIntForce(varLstGet(tablespace, 0)))
920 target.tablespaceName = varStr(varLstGet(tablespace, 1));
921 }
922
923 // Error if the tablespace could not be found. ??? This seems excessive, perhaps just warn here?
924 if (target.tablespaceName == NULL)
925 {
926 THROW_FMT(
927 AssertError,
928 "tablespace with oid %u not found in tablespace map\n"
929 "HINT: was a tablespace created or dropped during the backup?",
930 target.tablespaceId);
931 }
932 }
933
934 // If no tablespace name was found then create one
935 if (target.tablespaceName == NULL)
936 target.tablespaceName = strNewFmt("ts%s", strZ(info->name));
937
938 // Add a dummy pg_tblspc path entry if it does not already exist. This entry will be ignored by restore but it is
939 // part of the original manifest format so we need to have it.
940 lstSort(buildData.manifest->pub.pathList, sortOrderAsc);
941 const ManifestPath *pathBase = manifestPathFind(buildData.manifest, MANIFEST_TARGET_PGDATA_STR);
942
943 if (manifestPathFindDefault(buildData.manifest, MANIFEST_TARGET_PGTBLSPC_STR, NULL) == NULL)
944 {
945 ManifestPath path =
946 {
947 .name = MANIFEST_TARGET_PGTBLSPC_STR,
948 .mode = pathBase->mode,
949 .user = pathBase->user,
950 .group = pathBase->group,
951 };
952
953 manifestPathAdd(buildData.manifest, &path);
954 }
955
956 // If the tablespace id is present then the tablespace link destination path is not the path where data will be
957 // stored so we can just store it as a dummy path.
958 if (buildData.tablespaceId != NULL)
959 {
960 const ManifestPath *pathTblSpc = manifestPathFind(
961 buildData.manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC));
962
963 ManifestPath path =
964 {
965 .name = manifestName,
966 .mode = pathTblSpc->mode,
967 .user = pathTblSpc->user,
968 .group = pathTblSpc->group,
969 };
970
971 manifestPathAdd(buildData.manifest, &path);
972
973 // Update build structure to reflect the path added above and the tablespace id
974 buildData.manifestParentName = manifestName;
975 manifestName = strNewFmt("%s/%s", strZ(manifestName), strZ(buildData.tablespaceId));
976 buildData.pgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(info->name));
977 linkName = buildData.tablespaceId;
978 }
979 // If no tablespace id then parent manifest name is the tablespace directory
980 else
981 buildData.manifestParentName = MANIFEST_TARGET_PGTBLSPC_STR;
982 }
983
984 // Add info about the linked file/path
985 const String *linkPgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(linkName));
986 StorageInfo linkedInfo = storageInfoP(
987 buildData.storagePg, linkPgPath, .followLink = true, .ignoreMissing = true);
988 linkedInfo.name = linkName;
989
990 // If the link destination exists then build the target
991 if (linkedInfo.exists)
992 {
993 // If a path link then recurse
994 if (linkedInfo.type == storageTypePath)
995 {
996 target.path = info->linkDestination;
997 }
998 // Else it must be a file or special (since we have already checked if it is a link)
999 else
1000 {
1001 // Tablespace links should never be to a file
1002 CHECK(target.tablespaceId == 0);
1003
1004 // Identify target as a file
1005 target.path = strPath(info->linkDestination);
1006 target.file = strBase(info->linkDestination);
1007 }
1008 }
1009 // Else dummy up the target with a destination so manifestLinkCheck() can be run. This is so errors about links with
1010 // destinations in PGDATA will take precedence over missing a destination. We will probably simplify this once the
1011 // migration is done and it doesn't matter which error takes precedence.
1012 else
1013 target.path = info->linkDestination;
1014
1015 // Add target and link
1016 manifestTargetAdd(buildData.manifest, &target);
1017 manifestLinkAdd(buildData.manifest, &link);
1018
1019 // Make sure the link is valid
1020 manifestLinkCheckOne(buildData.manifest, &buildData.linkCheck, manifestTargetTotal(buildData.manifest) - 1);
1021
1022 // If the link check was successful but the destination does not exist then check it again to generate an error
1023 if (!linkedInfo.exists)
1024 storageInfoP(buildData.storagePg, linkPgPath, .followLink = true);
1025
1026 // Recurse into the link destination
1027 manifestBuildCallback(&buildData, &linkedInfo);
1028
1029 break;
1030 }
1031
1032 // Skip special files
1033 // -------------------------------------------------------------------------------------------------------------------------
1034 case storageTypeSpecial:
1035 LOG_WARN_FMT("exclude special file '%s/%s' from backup", strZ(buildData.pgPath), strZ(info->name));
1036 break;
1037 }
1038
1039 FUNCTION_TEST_RETURN_VOID();
1040 }
1041
1042 // Regular expression constants
1043 #define RELATION_EXP "[0-9]+(_(fsm|vm)){0,1}(\\.[0-9]+){0,1}"
1044 #define DB_PATH_EXP \
1045 "(" MANIFEST_TARGET_PGDATA "/(" PG_PATH_GLOBAL "|" PG_PATH_BASE "/[0-9]+)|" MANIFEST_TARGET_PGTBLSPC "/[0-9]+/%s/[0-9]+)"
1046
1047 Manifest *
manifestNewBuild(const Storage * storagePg,unsigned int pgVersion,unsigned int pgCatalogVersion,bool online,bool checksumPage,const StringList * excludeList,const VariantList * tablespaceList)1048 manifestNewBuild(
1049 const Storage *storagePg, unsigned int pgVersion, unsigned int pgCatalogVersion, bool online, bool checksumPage,
1050 const StringList *excludeList, const VariantList *tablespaceList)
1051 {
1052 FUNCTION_LOG_BEGIN(logLevelDebug);
1053 FUNCTION_LOG_PARAM(STORAGE, storagePg);
1054 FUNCTION_LOG_PARAM(UINT, pgVersion);
1055 FUNCTION_LOG_PARAM(UINT, pgCatalogVersion);
1056 FUNCTION_LOG_PARAM(BOOL, online);
1057 FUNCTION_LOG_PARAM(BOOL, checksumPage);
1058 FUNCTION_LOG_PARAM(STRING_LIST, excludeList);
1059 FUNCTION_LOG_PARAM(VARIANT_LIST, tablespaceList);
1060 FUNCTION_LOG_END();
1061
1062 ASSERT(storagePg != NULL);
1063 ASSERT(pgVersion != 0);
1064 ASSERT(!checksumPage || pgVersion >= PG_VERSION_93);
1065
1066 Manifest *this = NULL;
1067
1068 MEM_CONTEXT_NEW_BEGIN("Manifest")
1069 {
1070 this = manifestNewInternal();
1071 this->pub.info = infoNew(NULL);
1072 this->pub.data.backrestVersion = strNewZ(PROJECT_VERSION);
1073 this->pub.data.pgVersion = pgVersion;
1074 this->pub.data.pgCatalogVersion = pgCatalogVersion;
1075 this->pub.data.backupType = backupTypeFull;
1076 this->pub.data.backupOptionOnline = online;
1077 this->pub.data.backupOptionChecksumPage = varNewBool(checksumPage);
1078
1079 MEM_CONTEXT_TEMP_BEGIN()
1080 {
1081 // Data needed to build the manifest
1082 ManifestBuildData buildData =
1083 {
1084 .manifest = this,
1085 .storagePg = storagePg,
1086 .tablespaceId = pgTablespaceId(pgVersion, pgCatalogVersion),
1087 .online = online,
1088 .checksumPage = checksumPage,
1089 .tablespaceList = tablespaceList,
1090 .linkCheck = manifestLinkCheckInit(),
1091 .manifestParentName = MANIFEST_TARGET_PGDATA_STR,
1092 .manifestWalName = strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(pgWalPath(pgVersion))),
1093 .pgPath = storagePathP(storagePg, NULL),
1094 };
1095
1096 // We won't identify db paths for PostgreSQL < 9.0. This means that temp relations will not be excluded but it doesn't
1097 // seem worth supporting this feature on such old versions of PostgreSQL.
1098 // ---------------------------------------------------------------------------------------------------------------------
1099 if (pgVersion >= PG_VERSION_90)
1100 {
1101 ASSERT(buildData.tablespaceId != NULL);
1102
1103 // Expression to identify database paths
1104 buildData.dbPathExp = regExpNew(strNewFmt("^" DB_PATH_EXP "$", strZ(buildData.tablespaceId)));
1105
1106 // Expression to find temp relations
1107 buildData.tempRelationExp = regExpNew(STRDEF("^t[0-9]+_" RELATION_EXP "$"));
1108 }
1109
1110 // Build expression to identify files that can be copied from the standby when standby backup is supported
1111 // ---------------------------------------------------------------------------------------------------------------------
1112 buildData.standbyExp = regExpNew(
1113 strNewFmt(
1114 "^((" MANIFEST_TARGET_PGDATA "/(" PG_PATH_BASE "|" PG_PATH_GLOBAL "|%s|" PG_PATH_PGMULTIXACT "))|"
1115 MANIFEST_TARGET_PGTBLSPC ")/",
1116 strZ(pgXactPath(pgVersion))));
1117
1118 // Build list of exclusions
1119 // ---------------------------------------------------------------------------------------------------------------------
1120 if (excludeList != NULL)
1121 {
1122 for (unsigned int excludeIdx = 0; excludeIdx < strLstSize(excludeList); excludeIdx++)
1123 {
1124 const String *exclude = strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(strLstGet(excludeList, excludeIdx)));
1125
1126 // If the exclusions refers to the contents of a path
1127 if (strEndsWithZ(exclude, "/"))
1128 {
1129 if (buildData.excludeContent == NULL)
1130 buildData.excludeContent = strLstNew();
1131
1132 strLstAdd(buildData.excludeContent, strSubN(exclude, 0, strSize(exclude) - 1));
1133 }
1134 // Otherwise exclude a single file/link/path
1135 else
1136 {
1137 if (buildData.excludeSingle == NULL)
1138 buildData.excludeSingle = strLstNew();
1139
1140 strLstAdd(buildData.excludeSingle, exclude);
1141 }
1142 }
1143 }
1144
1145 // Build manifest
1146 // ---------------------------------------------------------------------------------------------------------------------
1147 StorageInfo info = storageInfoP(storagePg, buildData.pgPath, .followLink = true);
1148
1149 ManifestPath path =
1150 {
1151 .name = MANIFEST_TARGET_PGDATA_STR,
1152 .mode = info.mode,
1153 .user = info.user,
1154 .group = info.group,
1155 };
1156
1157 manifestPathAdd(this, &path);
1158
1159 ManifestTarget target =
1160 {
1161 .name = MANIFEST_TARGET_PGDATA_STR,
1162 .path = buildData.pgPath,
1163 .type = manifestTargetTypePath,
1164 };
1165
1166 manifestTargetAdd(this, &target);
1167
1168 // Gather info for the rest of the files/links/paths
1169 storageInfoListP(
1170 storagePg, buildData.pgPath, manifestBuildCallback, &buildData, .errorOnMissing = true, .sortOrder = sortOrderAsc);
1171
1172 // These may not be in order even if the incoming data was sorted
1173 lstSort(this->pub.fileList, sortOrderAsc);
1174 lstSort(this->pub.linkList, sortOrderAsc);
1175 lstSort(this->pub.pathList, sortOrderAsc);
1176 lstSort(this->pub.targetList, sortOrderAsc);
1177
1178 // Remove unlogged relations from the manifest. This can't be done during the initial build because of the requirement
1179 // to check for _init files which will sort after the vast majority of the relation files. We could check storage for
1180 // each _init file but that would be expensive.
1181 // -------------------------------------------------------------------------------------------------------------------------
1182 if (pgVersion >= PG_VERSION_91)
1183 {
1184 RegExp *relationExp = regExpNew(strNewFmt("^" DB_PATH_EXP "/" RELATION_EXP "$", strZ(buildData.tablespaceId)));
1185 unsigned int fileIdx = 0;
1186 char lastRelationFileId[21] = ""; // Large enough for a 64-bit unsigned integer
1187 bool lastRelationFileIdUnlogged = false;
1188
1189 #ifdef DEBUG_MEM
1190 // Record the temp context size before the loop begins
1191 size_t sizeBegin = memContextSize(memContextCurrent());
1192 #endif
1193
1194 while (fileIdx < manifestFileTotal(this))
1195 {
1196 // If this file looks like a relation. Note that this never matches on _init forks.
1197 const ManifestFile *file = manifestFile(this, fileIdx);
1198
1199 if (regExpMatch(relationExp, file->name))
1200 {
1201 // Get the filename (without path)
1202 const char *fileName = strBaseZ(file->name);
1203 size_t fileNameSize = strlen(fileName);
1204
1205 // Strip off the numeric part of the relation
1206 char relationFileId[sizeof(lastRelationFileId)];
1207 unsigned int nameIdx = 0;
1208
1209 for (; nameIdx < fileNameSize; nameIdx++)
1210 {
1211 if (!isdigit(fileName[nameIdx]))
1212 break;
1213
1214 relationFileId[nameIdx] = fileName[nameIdx];
1215 }
1216
1217 relationFileId[nameIdx] = '\0';
1218
1219 // The filename must have characters
1220 ASSERT(relationFileId[0] != '\0');
1221
1222 // Store the last relation so it does not need to be found everytime
1223 if (strcmp(lastRelationFileId, relationFileId) != 0)
1224 {
1225 // Determine if the relation is unlogged
1226 String *relationInit = strNewFmt(
1227 "%.*s%s_init", (int)(strSize(file->name) - fileNameSize), strZ(file->name), relationFileId);
1228 lastRelationFileIdUnlogged = manifestFileFindDefault(this, relationInit, NULL) != NULL;
1229 strFree(relationInit);
1230
1231 // Save the file id so we don't need to do the lookup next time if it doesn't change
1232 strcpy(lastRelationFileId, relationFileId);
1233 }
1234
1235 // If relation is unlogged then remove it
1236 if (lastRelationFileIdUnlogged)
1237 {
1238 manifestFileRemove(this, file->name);
1239 continue;
1240 }
1241 }
1242
1243 fileIdx++;
1244 }
1245
1246 #ifdef DEBUG_MEM
1247 // Make sure that the temp context did not grow too much during the loop
1248 ASSERT(memContextSize(memContextCurrent()) - sizeBegin < 256);
1249 #endif
1250 }
1251 }
1252 MEM_CONTEXT_TEMP_END();
1253 }
1254 MEM_CONTEXT_NEW_END();
1255
1256 FUNCTION_LOG_RETURN(MANIFEST, this);
1257 }
1258
1259 /**********************************************************************************************************************************/
1260 void
manifestBuildValidate(Manifest * this,bool delta,time_t copyStart,CompressType compressType)1261 manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType compressType)
1262 {
1263 FUNCTION_LOG_BEGIN(logLevelDebug);
1264 FUNCTION_LOG_PARAM(MANIFEST, this);
1265 FUNCTION_LOG_PARAM(BOOL, delta);
1266 FUNCTION_LOG_PARAM(TIME, copyStart);
1267 FUNCTION_LOG_PARAM(ENUM, compressType);
1268 FUNCTION_LOG_END();
1269
1270 ASSERT(this != NULL);
1271 ASSERT(copyStart > 0);
1272
1273 MEM_CONTEXT_BEGIN(this->pub.memContext)
1274 {
1275 // Store the delta option. If true we can skip checks that automatically enable delta.
1276 this->pub.data.backupOptionDelta = varNewBool(delta);
1277
1278 // If online then add one second to the copy start time to allow for database updates during the last second that the
1279 // manifest was being built. It's up to the caller to actually wait the remainder of the second, but for comparison
1280 // purposes we want the time when the waiting started.
1281 this->pub.data.backupTimestampCopyStart = copyStart + (this->pub.data.backupOptionOnline ? 1 : 0);
1282
1283 // This value is not needed in this function, but it is needed for resumed manifests and this is last place to set it before
1284 // processing begins
1285 this->pub.data.backupOptionCompressType = compressType;
1286 }
1287 MEM_CONTEXT_END();
1288
1289 // Check the manifest for timestamp anomalies that require a delta backup (if delta is not already specified)
1290 if (!varBool(this->pub.data.backupOptionDelta))
1291 {
1292 MEM_CONTEXT_TEMP_BEGIN()
1293 {
1294 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
1295 {
1296 const ManifestFile *file = manifestFile(this, fileIdx);
1297
1298 // Check for timestamp in the future
1299 if (file->timestamp > copyStart)
1300 {
1301 LOG_WARN_FMT(
1302 "file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file->name)));
1303
1304 this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
1305 break;
1306 }
1307 }
1308 }
1309 MEM_CONTEXT_TEMP_END();
1310 }
1311
1312 FUNCTION_LOG_RETURN_VOID();
1313 }
1314
1315 /**********************************************************************************************************************************/
1316 void
manifestBuildIncr(Manifest * this,const Manifest * manifestPrior,BackupType type,const String * archiveStart)1317 manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type, const String *archiveStart)
1318 {
1319 FUNCTION_LOG_BEGIN(logLevelDebug);
1320 FUNCTION_LOG_PARAM(MANIFEST, this);
1321 FUNCTION_LOG_PARAM(MANIFEST, manifestPrior);
1322 FUNCTION_LOG_PARAM(STRING_ID, type);
1323 FUNCTION_LOG_PARAM(STRING, archiveStart);
1324 FUNCTION_LOG_END();
1325
1326 ASSERT(this != NULL);
1327 ASSERT(manifestPrior != NULL);
1328 ASSERT(type == backupTypeDiff || type == backupTypeIncr);
1329 ASSERT(type != backupTypeDiff || manifestPrior->pub.data.backupType == backupTypeFull);
1330 ASSERT(archiveStart == NULL || strSize(archiveStart) == 24);
1331
1332 MEM_CONTEXT_BEGIN(this->pub.memContext)
1333 {
1334 // Set prior backup label
1335 this->pub.data.backupLabelPrior = strDup(manifestPrior->pub.data.backupLabel);
1336
1337 // Set diff/incr backup type
1338 this->pub.data.backupType = type;
1339 }
1340 MEM_CONTEXT_END();
1341
1342 MEM_CONTEXT_TEMP_BEGIN()
1343 {
1344 // Enable delta if timelines differ
1345 if (archiveStart != NULL && manifestData(manifestPrior)->archiveStop != NULL &&
1346 !strEq(strSubN(archiveStart, 0, 8), strSubN(manifestData(manifestPrior)->archiveStop, 0, 8)))
1347 {
1348 LOG_WARN_FMT(
1349 "a timeline switch has occurred since the %s backup, enabling delta checksum\n"
1350 "HINT: this is normal after restoring from backup or promoting a standby.",
1351 strZ(manifestData(manifestPrior)->backupLabel));
1352
1353 this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
1354 }
1355 // Else enable delta if online differs
1356 else if (manifestData(manifestPrior)->backupOptionOnline != this->pub.data.backupOptionOnline)
1357 {
1358 LOG_WARN_FMT(
1359 "the online option has changed since the %s backup, enabling delta checksum",
1360 strZ(manifestData(manifestPrior)->backupLabel));
1361
1362 this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
1363 }
1364
1365 // Check for anomalies between manifests if delta is not already enabled. This can't be combined with the main comparison
1366 // loop below because delta changes the behavior of that loop.
1367 if (!varBool(this->pub.data.backupOptionDelta))
1368 {
1369 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
1370 {
1371 const ManifestFile *file = manifestFile(this, fileIdx);
1372 const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
1373
1374 // If file was found in prior manifest then perform checks
1375 if (filePrior != NULL)
1376 {
1377 // Check for timestamp earlier than the prior backup
1378 if (file->timestamp < filePrior->timestamp)
1379 {
1380 LOG_WARN_FMT(
1381 "file '%s' has timestamp earlier than prior backup, enabling delta checksum",
1382 strZ(manifestPathPg(file->name)));
1383
1384 this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
1385 break;
1386 }
1387
1388 // Check for size change with no timestamp change
1389 if (file->size != filePrior->size && file->timestamp == filePrior->timestamp)
1390 {
1391 LOG_WARN_FMT(
1392 "file '%s' has same timestamp as prior but different size, enabling delta checksum",
1393 strZ(manifestPathPg(file->name)));
1394
1395 this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
1396 break;
1397 }
1398 }
1399 }
1400 }
1401
1402 // Find files to reference in the prior manifest:
1403 // 1) that don't need to be copied because delta is disabled and the size and timestamp match or size matches and is zero
1404 // 2) where delta is enabled and size matches so checksum will be verified during backup and the file copied on mismatch
1405 bool delta = varBool(this->pub.data.backupOptionDelta);
1406
1407 for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++)
1408 {
1409 const ManifestFile *file = manifestFile(this, fileIdx);
1410 const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
1411
1412 // Check if prior file can be used
1413 if (filePrior != NULL && file->size == filePrior->size &&
1414 (delta || file->size == 0 || file->timestamp == filePrior->timestamp))
1415 {
1416 manifestFileUpdate(
1417 this, file->name, file->size, filePrior->sizeRepo, filePrior->checksumSha1,
1418 VARSTR(filePrior->reference != NULL ? filePrior->reference : manifestPrior->pub.data.backupLabel),
1419 filePrior->checksumPage, filePrior->checksumPageError, filePrior->checksumPageErrorList);
1420 }
1421 }
1422 }
1423 MEM_CONTEXT_TEMP_END();
1424
1425 FUNCTION_LOG_RETURN_VOID();
1426 }
1427
1428 /**********************************************************************************************************************************/
1429 void
manifestBuildComplete(Manifest * this,time_t timestampStart,const String * lsnStart,const String * archiveStart,time_t timestampStop,const String * lsnStop,const String * archiveStop,unsigned int pgId,uint64_t pgSystemId,const VariantList * dbList,bool optionArchiveCheck,bool optionArchiveCopy,size_t optionBufferSize,unsigned int optionCompressLevel,unsigned int optionCompressLevelNetwork,bool optionHardLink,unsigned int optionProcessMax,bool optionStandby)1430 manifestBuildComplete(
1431 Manifest *this, time_t timestampStart, const String *lsnStart, const String *archiveStart, time_t timestampStop,
1432 const String *lsnStop, const String *archiveStop, unsigned int pgId, uint64_t pgSystemId, const VariantList *dbList,
1433 bool optionArchiveCheck, bool optionArchiveCopy, size_t optionBufferSize, unsigned int optionCompressLevel,
1434 unsigned int optionCompressLevelNetwork, bool optionHardLink, unsigned int optionProcessMax,
1435 bool optionStandby)
1436 {
1437 FUNCTION_LOG_BEGIN(logLevelDebug);
1438 FUNCTION_LOG_PARAM(MANIFEST, this);
1439 FUNCTION_LOG_PARAM(TIME, timestampStart);
1440 FUNCTION_LOG_PARAM(STRING, lsnStart);
1441 FUNCTION_LOG_PARAM(STRING, archiveStart);
1442 FUNCTION_LOG_PARAM(TIME, timestampStop);
1443 FUNCTION_LOG_PARAM(STRING, lsnStop);
1444 FUNCTION_LOG_PARAM(STRING, archiveStop);
1445 FUNCTION_LOG_PARAM(UINT, pgId);
1446 FUNCTION_LOG_PARAM(UINT64, pgSystemId);
1447 FUNCTION_LOG_PARAM(VARIANT_LIST, dbList);
1448 FUNCTION_LOG_PARAM(BOOL, optionArchiveCheck);
1449 FUNCTION_LOG_PARAM(BOOL, optionArchiveCopy);
1450 FUNCTION_LOG_PARAM(SIZE, optionBufferSize);
1451 FUNCTION_LOG_PARAM(UINT, optionCompressLevel);
1452 FUNCTION_LOG_PARAM(UINT, optionCompressLevelNetwork);
1453 FUNCTION_LOG_PARAM(BOOL, optionHardLink);
1454 FUNCTION_LOG_PARAM(UINT, optionProcessMax);
1455 FUNCTION_LOG_PARAM(BOOL, optionStandby);
1456 FUNCTION_LOG_END();
1457
1458 MEM_CONTEXT_BEGIN(this->pub.memContext)
1459 {
1460 // Save info
1461 this->pub.data.backupTimestampStart = timestampStart;
1462 this->pub.data.lsnStart = strDup(lsnStart);
1463 this->pub.data.archiveStart = strDup(archiveStart);
1464 this->pub.data.backupTimestampStop = timestampStop;
1465 this->pub.data.lsnStop = strDup(lsnStop);
1466 this->pub.data.archiveStop = strDup(archiveStop);
1467 this->pub.data.pgId = pgId;
1468 this->pub.data.pgSystemId = pgSystemId;
1469
1470 // Save db list
1471 if (dbList != NULL)
1472 {
1473 for (unsigned int dbIdx = 0; dbIdx < varLstSize(dbList); dbIdx++)
1474 {
1475 const VariantList *dbRow = varVarLst(varLstGet(dbList, dbIdx));
1476
1477 ManifestDb db =
1478 {
1479 .id = varUIntForce(varLstGet(dbRow, 0)),
1480 .name = varStr(varLstGet(dbRow, 1)),
1481 .lastSystemId = varUIntForce(varLstGet(dbRow, 2)),
1482 };
1483
1484 manifestDbAdd(this, &db);
1485 }
1486
1487 lstSort(this->pub.dbList, sortOrderAsc);
1488 }
1489
1490 // Save options
1491 this->pub.data.backupOptionArchiveCheck = optionArchiveCheck;
1492 this->pub.data.backupOptionArchiveCopy = optionArchiveCopy;
1493 this->pub.data.backupOptionBufferSize = varNewUInt64(optionBufferSize);
1494 this->pub.data.backupOptionCompressLevel = varNewUInt(optionCompressLevel);
1495 this->pub.data.backupOptionCompressLevelNetwork = varNewUInt(optionCompressLevelNetwork);
1496 this->pub.data.backupOptionHardLink = optionHardLink;
1497 this->pub.data.backupOptionProcessMax = varNewUInt(optionProcessMax);
1498 this->pub.data.backupOptionStandby = varNewBool(optionStandby);
1499 }
1500 MEM_CONTEXT_END();
1501
1502 FUNCTION_LOG_RETURN_VOID();
1503 }
1504
1505 /**********************************************************************************************************************************/
1506 // Keep track of which values were found during load and which need to be loaded from defaults. There is no point in having
1507 // multiple structs since most of the fields are the same and the size shouldn't be more than 4/8 bytes.
1508 typedef struct ManifestLoadFound
1509 {
1510 bool group:1;
1511 bool mode:1;
1512 bool primary:1;
1513 bool user:1;
1514 } ManifestLoadFound;
1515
1516 typedef struct ManifestLoadData
1517 {
1518 MemContext *memContext; // Mem context for data needed only during load
1519 Manifest *manifest; // Manifest info
1520
1521 List *fileFoundList; // Values found in files
1522 const Variant *fileGroupDefault; // File default group
1523 mode_t fileModeDefault; // File default mode
1524 bool filePrimaryDefault; // File default primary
1525 const Variant *fileUserDefault; // File default user
1526
1527 List *linkFoundList; // Values found in links
1528 const Variant *linkGroupDefault; // Link default group
1529 const Variant *linkUserDefault; // Link default user
1530
1531 List *pathFoundList; // Values found in paths
1532 const Variant *pathGroupDefault; // Path default group
1533 mode_t pathModeDefault; // Path default mode
1534 const Variant *pathUserDefault; // Path default user
1535 } ManifestLoadData;
1536
1537 // Helper to transform a variant that could be boolean or string into a string. If the boolean is false return NULL else return
1538 // the string. The boolean cannot be true.
1539 static const String *
manifestOwnerGet(const Variant * owner)1540 manifestOwnerGet(const Variant *owner)
1541 {
1542 FUNCTION_TEST_BEGIN();
1543 FUNCTION_TEST_PARAM(VARIANT, owner);
1544 FUNCTION_TEST_END();
1545
1546 ASSERT(owner != NULL);
1547
1548 // If bool then it should be false. This indicates that the owner could not be mapped to a name during the backup.
1549 if (varType(owner) == varTypeBool)
1550 {
1551 CHECK(!varBool(owner));
1552 FUNCTION_TEST_RETURN(NULL);
1553 }
1554
1555 FUNCTION_TEST_RETURN(varStr(owner));
1556 }
1557
1558 // Helper to check the variant type of owner and duplicate (call in the containing context)
1559 static const Variant *
manifestOwnerDefaultGet(const Variant * ownerDefault)1560 manifestOwnerDefaultGet(const Variant *ownerDefault)
1561 {
1562 FUNCTION_TEST_BEGIN();
1563 FUNCTION_TEST_PARAM(STRING, ownerDefault);
1564 FUNCTION_TEST_END();
1565
1566 ASSERT(ownerDefault != NULL);
1567
1568 // Bool = false means the owner was not mapped to a name
1569 if (varType(ownerDefault) == varTypeBool)
1570 {
1571 // Value must be false
1572 CHECK(!varBool(ownerDefault));
1573 FUNCTION_TEST_RETURN(BOOL_FALSE_VAR);
1574 }
1575
1576 // Return a duplicate of the owner passed in
1577 FUNCTION_TEST_RETURN(varDup(ownerDefault));
1578 }
1579
1580 static void
manifestLoadCallback(void * callbackData,const String * section,const String * key,const Variant * value)1581 manifestLoadCallback(void *callbackData, const String *section, const String *key, const Variant *value)
1582 {
1583 FUNCTION_TEST_BEGIN();
1584 FUNCTION_TEST_PARAM_P(VOID, callbackData);
1585 FUNCTION_TEST_PARAM(STRING, section);
1586 FUNCTION_TEST_PARAM(STRING, key);
1587 FUNCTION_TEST_PARAM(VARIANT, value);
1588 FUNCTION_TEST_END();
1589
1590 ASSERT(callbackData != NULL);
1591 ASSERT(section != NULL);
1592 ASSERT(key != NULL);
1593
1594 ManifestLoadData *loadData = (ManifestLoadData *)callbackData;
1595 Manifest *manifest = loadData->manifest;
1596
1597 // -----------------------------------------------------------------------------------------------------------------------------
1598 if (strEq(section, MANIFEST_SECTION_TARGET_FILE_STR))
1599 {
1600 KeyValue *fileKv = varKv(value);
1601
1602 MEM_CONTEXT_BEGIN(lstMemContext(manifest->pub.fileList))
1603 {
1604 ManifestLoadFound valueFound = {0};
1605
1606 ManifestFile file =
1607 {
1608 .name = key,
1609 .reference = varStr(kvGetDefault(fileKv, MANIFEST_KEY_REFERENCE_VAR, NULL)),
1610 };
1611
1612 // Timestamp is required so error if it is not present
1613 const Variant *timestamp = kvGet(fileKv, MANIFEST_KEY_TIMESTAMP_VAR);
1614
1615 if (timestamp == NULL)
1616 THROW_FMT(FormatError, "missing timestamp for file '%s'", strZ(key));
1617
1618 file.timestamp = (time_t)varUInt64(timestamp);
1619
1620 // Size is required so error if it is not present. Older versions removed the size before the backup to ensure that the
1621 // manifest was updated during the backup, so size can be missing in partial manifests. This error will prevent older
1622 // partials from being resumed.
1623 const Variant *size = kvGet(fileKv, MANIFEST_KEY_SIZE_VAR);
1624
1625 if (size == NULL)
1626 THROW_FMT(FormatError, "missing size for file '%s'", strZ(key));
1627
1628 file.size = varUInt64(size);
1629
1630 // If "repo-size" is not present in the manifest file, then it is the same as size (i.e. uncompressed) - to save space,
1631 // the repo-size is only stored in the manifest file if it is different than size.
1632 file.sizeRepo = varUInt64(kvGetDefault(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.size)));
1633
1634 // If file size is zero then assign the static zero hash
1635 if (file.size == 0)
1636 {
1637 memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
1638 }
1639 // Else if the key exists then load it. The key might not exist if this is a partial save that was done during the
1640 // backup to preserve checksums for already backed up files.
1641 else if (kvKeyExists(fileKv, MANIFEST_KEY_CHECKSUM_VAR))
1642 memcpy(file.checksumSha1, strZ(varStr(kvGet(fileKv, MANIFEST_KEY_CHECKSUM_VAR))), HASH_TYPE_SHA1_SIZE_HEX + 1);
1643
1644 const Variant *checksumPage = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, NULL);
1645
1646 if (checksumPage != NULL)
1647 {
1648 file.checksumPage = true;
1649 file.checksumPageError = !varBool(checksumPage);
1650
1651 const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL);
1652
1653 if (checksumPageErrorList != NULL)
1654 file.checksumPageErrorList = varVarLst(checksumPageErrorList);
1655 }
1656
1657 if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR))
1658 {
1659 valueFound.group = true;
1660 file.group = manifestOwnerGet(kvGet(fileKv, MANIFEST_KEY_GROUP_VAR));
1661 }
1662
1663 if (kvKeyExists(fileKv, MANIFEST_KEY_MODE_VAR))
1664 {
1665 valueFound.mode = true;
1666 file.mode = cvtZToMode(strZ(varStr(kvGet(fileKv, MANIFEST_KEY_MODE_VAR))));
1667 }
1668
1669 if (kvKeyExists(fileKv, MANIFEST_KEY_PRIMARY_VAR))
1670 {
1671 valueFound.primary = true;
1672 file.primary = varBool(kvGet(fileKv, MANIFEST_KEY_PRIMARY_VAR));
1673 }
1674
1675 if (kvKeyExists(fileKv, MANIFEST_KEY_USER_VAR))
1676 {
1677 valueFound.user = true;
1678 file.user = manifestOwnerGet(kvGet(fileKv, MANIFEST_KEY_USER_VAR));
1679 }
1680
1681 lstAdd(loadData->fileFoundList, &valueFound);
1682 manifestFileAdd(manifest, &file);
1683 }
1684 MEM_CONTEXT_END();
1685 }
1686
1687 // -----------------------------------------------------------------------------------------------------------------------------
1688 else if (strEq(section, MANIFEST_SECTION_TARGET_PATH_STR))
1689 {
1690 KeyValue *pathKv = varKv(value);
1691
1692 MEM_CONTEXT_BEGIN(lstMemContext(manifest->pub.pathList))
1693 {
1694 ManifestLoadFound valueFound = {0};
1695
1696 ManifestPath path =
1697 {
1698 .name = key,
1699 };
1700
1701 if (kvKeyExists(pathKv, MANIFEST_KEY_GROUP_VAR))
1702 {
1703 valueFound.group = true;
1704 path.group = manifestOwnerGet(kvGet(pathKv, MANIFEST_KEY_GROUP_VAR));
1705 }
1706
1707 if (kvKeyExists(pathKv, MANIFEST_KEY_MODE_VAR))
1708 {
1709 valueFound.mode = true;
1710 path.mode = cvtZToMode(strZ(varStr(kvGet(pathKv, MANIFEST_KEY_MODE_VAR))));
1711 }
1712
1713 if (kvKeyExists(pathKv, MANIFEST_KEY_USER_VAR))
1714 {
1715 valueFound.user = true;
1716 path.user = manifestOwnerGet(kvGet(pathKv, MANIFEST_KEY_USER_VAR));
1717 }
1718
1719 lstAdd(loadData->pathFoundList, &valueFound);
1720 manifestPathAdd(manifest, &path);
1721 }
1722 MEM_CONTEXT_END();
1723 }
1724
1725 // -----------------------------------------------------------------------------------------------------------------------------
1726 else if (strEq(section, MANIFEST_SECTION_TARGET_LINK_STR))
1727 {
1728 KeyValue *linkKv = varKv(value);
1729
1730 MEM_CONTEXT_BEGIN(lstMemContext(manifest->pub.linkList))
1731 {
1732 ManifestLoadFound valueFound = {0};
1733
1734 ManifestLink link =
1735 {
1736 .name = key,
1737 .destination = varStr(kvGet(linkKv, MANIFEST_KEY_DESTINATION_VAR)),
1738 };
1739
1740 if (kvKeyExists(linkKv, MANIFEST_KEY_GROUP_VAR))
1741 {
1742 valueFound.group = true;
1743 link.group = manifestOwnerGet(kvGet(linkKv, MANIFEST_KEY_GROUP_VAR));
1744 }
1745
1746 if (kvKeyExists(linkKv, MANIFEST_KEY_USER_VAR))
1747 {
1748 valueFound.user = true;
1749 link.user = manifestOwnerGet(kvGet(linkKv, MANIFEST_KEY_USER_VAR));
1750 }
1751
1752 lstAdd(loadData->linkFoundList, &valueFound);
1753 manifestLinkAdd(manifest, &link);
1754 }
1755 MEM_CONTEXT_END();
1756 }
1757
1758 // -----------------------------------------------------------------------------------------------------------------------------
1759 else if (strEq(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR))
1760 {
1761 MEM_CONTEXT_BEGIN(loadData->memContext)
1762 {
1763 if (strEq(key, MANIFEST_KEY_GROUP_STR))
1764 loadData->fileGroupDefault = manifestOwnerDefaultGet(value);
1765 else if (strEq(key, MANIFEST_KEY_MODE_STR))
1766 loadData->fileModeDefault = cvtZToMode(strZ(varStr(value)));
1767 else if (strEq(key, MANIFEST_KEY_PRIMARY_STR))
1768 loadData->filePrimaryDefault = varBool(value);
1769 else if (strEq(key, MANIFEST_KEY_USER_STR))
1770 loadData->fileUserDefault = manifestOwnerDefaultGet(value);
1771 }
1772 MEM_CONTEXT_END();
1773 }
1774
1775 // -----------------------------------------------------------------------------------------------------------------------------
1776 else if (strEq(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR))
1777 {
1778 MEM_CONTEXT_BEGIN(loadData->memContext)
1779 {
1780 if (strEq(key, MANIFEST_KEY_GROUP_STR))
1781 loadData->pathGroupDefault = manifestOwnerDefaultGet(value);
1782 else if (strEq(key, MANIFEST_KEY_MODE_STR))
1783 loadData->pathModeDefault = cvtZToMode(strZ(varStr(value)));
1784 else if (strEq(key, MANIFEST_KEY_USER_STR))
1785 loadData->pathUserDefault = manifestOwnerDefaultGet(value);
1786 }
1787 MEM_CONTEXT_END();
1788 }
1789
1790 // -----------------------------------------------------------------------------------------------------------------------------
1791 else if (strEq(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR))
1792 {
1793 MEM_CONTEXT_BEGIN(loadData->memContext)
1794 {
1795 if (strEq(key, MANIFEST_KEY_GROUP_STR))
1796 loadData->linkGroupDefault = manifestOwnerDefaultGet(value);
1797 else if (strEq(key, MANIFEST_KEY_USER_STR))
1798 loadData->linkUserDefault = manifestOwnerDefaultGet(value);
1799 }
1800 MEM_CONTEXT_END();
1801 }
1802
1803 // -----------------------------------------------------------------------------------------------------------------------------
1804 else if (strEq(section, MANIFEST_SECTION_BACKUP_TARGET_STR))
1805 {
1806 KeyValue *targetKv = varKv(value);
1807 const String *targetType = varStr(kvGet(targetKv, MANIFEST_KEY_TYPE_VAR));
1808
1809 ASSERT(strEq(targetType, MANIFEST_TARGET_TYPE_LINK_STR) || strEq(targetType, MANIFEST_TARGET_TYPE_PATH_STR));
1810
1811 ManifestTarget target =
1812 {
1813 .name = key,
1814 .file = varStr(kvGetDefault(targetKv, MANIFEST_KEY_FILE_VAR, NULL)),
1815 .path = varStr(kvGet(targetKv, MANIFEST_KEY_PATH_VAR)),
1816 .tablespaceId = cvtZToUInt(strZ(varStr(kvGetDefault(targetKv, MANIFEST_KEY_TABLESPACE_ID_VAR, VARSTRDEF("0"))))),
1817 .tablespaceName = varStr(kvGetDefault(targetKv, MANIFEST_KEY_TABLESPACE_NAME_VAR, NULL)),
1818 .type = strEq(targetType, MANIFEST_TARGET_TYPE_PATH_STR) ? manifestTargetTypePath : manifestTargetTypeLink,
1819 };
1820
1821 manifestTargetAdd(manifest, &target);
1822 }
1823
1824 // -----------------------------------------------------------------------------------------------------------------------------
1825 else if (strEq(section, MANIFEST_SECTION_DB_STR))
1826 {
1827 KeyValue *dbKv = varKv(value);
1828
1829 MEM_CONTEXT_BEGIN(lstMemContext(manifest->pub.dbList))
1830 {
1831 ManifestDb db =
1832 {
1833 .name = strDup(key),
1834 .id = varUIntForce(kvGet(dbKv, MANIFEST_KEY_DB_ID_VAR)),
1835 .lastSystemId = varUIntForce(kvGet(dbKv, MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR)),
1836 };
1837
1838 manifestDbAdd(manifest, &db);
1839 }
1840 MEM_CONTEXT_END();
1841 }
1842
1843 // -----------------------------------------------------------------------------------------------------------------------------
1844 else if (strEq(section, MANIFEST_SECTION_BACKUP_STR))
1845 {
1846 MEM_CONTEXT_BEGIN(manifest->pub.memContext)
1847 {
1848 if (strEq(key, MANIFEST_KEY_BACKUP_ARCHIVE_START_STR))
1849 manifest->pub.data.archiveStart = strDup(varStr(value));
1850 else if (strEq(key, MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR))
1851 manifest->pub.data.archiveStop = strDup(varStr(value));
1852 else if (strEq(key, MANIFEST_KEY_BACKUP_LABEL_STR))
1853 manifest->pub.data.backupLabel = strDup(varStr(value));
1854 else if (strEq(key, MANIFEST_KEY_BACKUP_LSN_START_STR))
1855 manifest->pub.data.lsnStart = strDup(varStr(value));
1856 else if (strEq(key, MANIFEST_KEY_BACKUP_LSN_STOP_STR))
1857 manifest->pub.data.lsnStop = strDup(varStr(value));
1858 else if (strEq(key, MANIFEST_KEY_BACKUP_PRIOR_STR))
1859 manifest->pub.data.backupLabelPrior = strDup(varStr(value));
1860 else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR))
1861 manifest->pub.data.backupTimestampCopyStart = (time_t)varUInt64(value);
1862 else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR))
1863 manifest->pub.data.backupTimestampStart = (time_t)varUInt64(value);
1864 else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR))
1865 manifest->pub.data.backupTimestampStop = (time_t)varUInt64(value);
1866 else if (strEq(key, MANIFEST_KEY_BACKUP_TYPE_STR))
1867 {
1868 manifest->pub.data.backupType = (BackupType)strIdFromStr(stringIdBit5, varStr(value));
1869 ASSERT(
1870 manifest->pub.data.backupType == backupTypeFull || manifest->pub.data.backupType == backupTypeDiff ||
1871 manifest->pub.data.backupType == backupTypeIncr);
1872 }
1873 }
1874 MEM_CONTEXT_END();
1875 }
1876
1877 // -----------------------------------------------------------------------------------------------------------------------------
1878 else if (strEq(section, MANIFEST_SECTION_BACKUP_DB_STR))
1879 {
1880 if (strEq(key, MANIFEST_KEY_DB_ID_STR))
1881 manifest->pub.data.pgId = varUIntForce(value);
1882 else if (strEq(key, MANIFEST_KEY_DB_SYSTEM_ID_STR))
1883 manifest->pub.data.pgSystemId = varUInt64(value);
1884 else if (strEq(key, MANIFEST_KEY_DB_CATALOG_VERSION_STR))
1885 manifest->pub.data.pgCatalogVersion = varUIntForce(value);
1886 else if (strEq(key, MANIFEST_KEY_DB_VERSION_STR))
1887 manifest->pub.data.pgVersion = pgVersionFromStr(varStr(value));
1888 }
1889
1890 // -----------------------------------------------------------------------------------------------------------------------------
1891 else if (strEq(section, MANIFEST_SECTION_BACKUP_OPTION_STR))
1892 {
1893 MEM_CONTEXT_BEGIN(manifest->pub.memContext)
1894 {
1895 // Required options
1896 if (strEq(key, MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR))
1897 manifest->pub.data.backupOptionArchiveCheck = varBool(value);
1898 else if (strEq(key, MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR))
1899 manifest->pub.data.backupOptionArchiveCopy = varBool(value);
1900 // Historically this option meant to add gz compression
1901 else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_STR))
1902 manifest->pub.data.backupOptionCompressType = varBool(value) ? compressTypeGz : compressTypeNone;
1903 // This new option allows any type of compression to be specified. It must be parsed after the option above so the
1904 // value does not get overwritten. Since options are stored in alpha order this should always be true.
1905 else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR))
1906 manifest->pub.data.backupOptionCompressType = compressTypeEnum(varStr(value));
1907 else if (strEq(key, MANIFEST_KEY_OPTION_HARDLINK_STR))
1908 manifest->pub.data.backupOptionHardLink = varBool(value);
1909 else if (strEq(key, MANIFEST_KEY_OPTION_ONLINE_STR))
1910 manifest->pub.data.backupOptionOnline = varBool(value);
1911
1912 // Options that were added after v1.00 and may not be present in every manifest
1913 else if (strEq(key, MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR))
1914 manifest->pub.data.backupOptionStandby = varNewBool(varBool(value));
1915 else if (strEq(key, MANIFEST_KEY_OPTION_BUFFER_SIZE_STR))
1916 manifest->pub.data.backupOptionBufferSize = varNewUInt(varUIntForce(value));
1917 else if (strEq(key, MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR))
1918 manifest->pub.data.backupOptionChecksumPage = varDup(value);
1919 else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR))
1920 manifest->pub.data.backupOptionCompressLevel = varNewUInt(varUIntForce(value));
1921 else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR))
1922 manifest->pub.data.backupOptionCompressLevelNetwork = varNewUInt(varUIntForce(value));
1923 else if (strEq(key, MANIFEST_KEY_OPTION_DELTA_STR))
1924 manifest->pub.data.backupOptionDelta = varDup(value);
1925 else if (strEq(key, MANIFEST_KEY_OPTION_PROCESS_MAX_STR))
1926 manifest->pub.data.backupOptionProcessMax = varNewUInt(varUIntForce(value));
1927 }
1928 MEM_CONTEXT_END();
1929 }
1930
1931 FUNCTION_TEST_RETURN_VOID();
1932 }
1933
1934 Manifest *
manifestNewLoad(IoRead * read)1935 manifestNewLoad(IoRead *read)
1936 {
1937 FUNCTION_LOG_BEGIN(logLevelDebug);
1938 FUNCTION_LOG_PARAM(IO_READ, read);
1939 FUNCTION_LOG_END();
1940
1941 ASSERT(read != NULL);
1942
1943 Manifest *this = NULL;
1944
1945 MEM_CONTEXT_NEW_BEGIN("Manifest")
1946 {
1947 this = manifestNewInternal();
1948
1949 // Load the manifest
1950 ManifestLoadData loadData =
1951 {
1952 .memContext = memContextNew("load"),
1953 .manifest = this,
1954 };
1955
1956 MEM_CONTEXT_BEGIN(loadData.memContext)
1957 {
1958 loadData.fileFoundList = lstNewP(sizeof(ManifestLoadFound));
1959 loadData.linkFoundList = lstNewP(sizeof(ManifestLoadFound));
1960 loadData.pathFoundList = lstNewP(sizeof(ManifestLoadFound));
1961 }
1962 MEM_CONTEXT_END();
1963
1964 this->pub.info = infoNewLoad(read, manifestLoadCallback, &loadData);
1965 this->pub.data.backrestVersion = infoBackrestVersion(this->pub.info);
1966
1967 // Process file defaults
1968 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
1969 {
1970 ManifestFile *file = lstGet(this->pub.fileList, fileIdx);
1971 ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx);
1972
1973 if (!found->group)
1974 file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
1975
1976 if (!found->mode)
1977 file->mode = loadData.fileModeDefault;
1978
1979 if (!found->primary)
1980 file->primary = loadData.filePrimaryDefault;
1981
1982 if (!found->user)
1983 file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
1984 }
1985
1986 // Process link defaults
1987 for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(this); linkIdx++)
1988 {
1989 ManifestLink *link = lstGet(this->pub.linkList, linkIdx);
1990 ManifestLoadFound *found = lstGet(loadData.linkFoundList, linkIdx);
1991
1992 if (!found->group)
1993 link->group = manifestOwnerCache(this, manifestOwnerGet(loadData.linkGroupDefault));
1994
1995 if (!found->user)
1996 link->user = manifestOwnerCache(this, manifestOwnerGet(loadData.linkUserDefault));
1997 }
1998
1999 // Process path defaults
2000 for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(this); pathIdx++)
2001 {
2002 ManifestPath *path = lstGet(this->pub.pathList, pathIdx);
2003 ManifestLoadFound *found = lstGet(loadData.pathFoundList, pathIdx);
2004
2005 if (!found->group)
2006 path->group = manifestOwnerCache(this, manifestOwnerGet(loadData.pathGroupDefault));
2007
2008 if (!found->mode)
2009 path->mode = loadData.pathModeDefault;
2010
2011 if (!found->user)
2012 path->user = manifestOwnerCache(this, manifestOwnerGet(loadData.pathUserDefault));
2013 }
2014
2015 // Sort the lists. They should already be sorted in the file but it is possible that this system has a different collation
2016 // that renders that sort useless.
2017 //
2018 // This must happen *after* the default processing because found lists are in natural file order and it is not worth writing
2019 // comparator routines for them.
2020 lstSort(this->pub.dbList, sortOrderAsc);
2021 lstSort(this->pub.fileList, sortOrderAsc);
2022 lstSort(this->pub.linkList, sortOrderAsc);
2023 lstSort(this->pub.pathList, sortOrderAsc);
2024 lstSort(this->pub.targetList, sortOrderAsc);
2025
2026 // Make sure the base path exists
2027 manifestTargetBase(this);
2028
2029 // Discard the context holding temporary load data
2030 memContextDiscard();
2031 }
2032 MEM_CONTEXT_NEW_END();
2033
2034 FUNCTION_LOG_RETURN(MANIFEST, this);
2035 }
2036
2037 /**********************************************************************************************************************************/
2038 typedef struct ManifestSaveData
2039 {
2040 Manifest *manifest; // Manifest object to be saved
2041
2042 const Variant *fileGroupDefault; // File default group
2043 mode_t fileModeDefault; // File default mode
2044 bool filePrimaryDefault; // File default primary
2045 const Variant *fileUserDefault; // File default user
2046
2047 const Variant *linkGroupDefault; // Link default group
2048 const Variant *linkUserDefault; // Link default user
2049
2050 const Variant *pathGroupDefault; // Path default group
2051 mode_t pathModeDefault; // Path default mode
2052 const Variant *pathUserDefault; // Path default user
2053 } ManifestSaveData;
2054
2055 // Helper to convert the owner MCV to a default. If the input is NULL boolean false should be returned, else the owner string.
2056 static const Variant *
manifestOwnerVar(const String * ownerDefault)2057 manifestOwnerVar(const String *ownerDefault)
2058 {
2059 FUNCTION_TEST_BEGIN();
2060 FUNCTION_TEST_PARAM(STRING, ownerDefault);
2061 FUNCTION_TEST_END();
2062
2063 FUNCTION_TEST_RETURN(ownerDefault == NULL ? BOOL_FALSE_VAR : varNewStr(ownerDefault));
2064 }
2065
2066 static void
manifestSaveCallback(void * callbackData,const String * sectionNext,InfoSave * infoSaveData)2067 manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *infoSaveData)
2068 {
2069 FUNCTION_TEST_BEGIN();
2070 FUNCTION_TEST_PARAM_P(VOID, callbackData);
2071 FUNCTION_TEST_PARAM(STRING, sectionNext);
2072 FUNCTION_TEST_PARAM(INFO_SAVE, infoSaveData);
2073 FUNCTION_TEST_END();
2074
2075 ASSERT(callbackData != NULL);
2076 ASSERT(infoSaveData != NULL);
2077
2078 ManifestSaveData *saveData = (ManifestSaveData *)callbackData;
2079 Manifest *manifest = saveData->manifest;
2080
2081 // -----------------------------------------------------------------------------------------------------------------------------
2082 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_STR, sectionNext))
2083 {
2084 if (manifest->pub.data.archiveStart != NULL)
2085 {
2086 infoSaveValue(
2087 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_START_STR,
2088 jsonFromStr(manifest->pub.data.archiveStart));
2089 }
2090
2091 if (manifest->pub.data.archiveStop != NULL)
2092 {
2093 infoSaveValue(
2094 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR,
2095 jsonFromStr(manifest->pub.data.archiveStop));
2096 }
2097
2098 infoSaveValue(
2099 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LABEL_STR,
2100 jsonFromStr(manifest->pub.data.backupLabel));
2101
2102 if (manifest->pub.data.lsnStart != NULL)
2103 {
2104 infoSaveValue(
2105 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LSN_START_STR,
2106 jsonFromStr(manifest->pub.data.lsnStart));
2107 }
2108
2109 if (manifest->pub.data.lsnStop != NULL)
2110 {
2111 infoSaveValue(
2112 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LSN_STOP_STR,
2113 jsonFromStr(manifest->pub.data.lsnStop));
2114 }
2115
2116 if (manifest->pub.data.backupLabelPrior != NULL)
2117 {
2118 infoSaveValue(
2119 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_PRIOR_STR,
2120 jsonFromStr(manifest->pub.data.backupLabelPrior));
2121 }
2122
2123 infoSaveValue(
2124 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR,
2125 jsonFromInt64(manifest->pub.data.backupTimestampCopyStart));
2126 infoSaveValue(
2127 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR,
2128 jsonFromInt64(manifest->pub.data.backupTimestampStart));
2129 infoSaveValue(
2130 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR,
2131 jsonFromInt64(manifest->pub.data.backupTimestampStop));
2132 infoSaveValue(
2133 infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TYPE_STR,
2134 jsonFromStr(strIdToStr(manifest->pub.data.backupType)));
2135 }
2136
2137 // -----------------------------------------------------------------------------------------------------------------------------
2138 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, sectionNext))
2139 {
2140 infoSaveValue(
2141 infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_CATALOG_VERSION_STR,
2142 jsonFromUInt(manifest->pub.data.pgCatalogVersion));
2143 infoSaveValue(
2144 infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, STRDEF("db-control-version"),
2145 jsonFromUInt(pgControlVersion(manifest->pub.data.pgVersion)));
2146 infoSaveValue(
2147 infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_ID_STR, jsonFromUInt(manifest->pub.data.pgId));
2148 infoSaveValue(
2149 infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_SYSTEM_ID_STR,
2150 jsonFromUInt64(manifest->pub.data.pgSystemId));
2151 infoSaveValue(
2152 infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_VERSION_STR,
2153 jsonFromStr(pgVersionToStr(manifest->pub.data.pgVersion)));
2154 }
2155
2156 // -----------------------------------------------------------------------------------------------------------------------------
2157 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, sectionNext))
2158 {
2159 infoSaveValue(
2160 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR,
2161 jsonFromBool(manifest->pub.data.backupOptionArchiveCheck));
2162 infoSaveValue(
2163 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR,
2164 jsonFromBool(manifest->pub.data.backupOptionArchiveCopy));
2165
2166 if (manifest->pub.data.backupOptionStandby != NULL)
2167 {
2168 infoSaveValue(
2169 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR,
2170 jsonFromVar(manifest->pub.data.backupOptionStandby));
2171 }
2172
2173 if (manifest->pub.data.backupOptionBufferSize != NULL)
2174 {
2175 infoSaveValue(
2176 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_BUFFER_SIZE_STR,
2177 jsonFromVar(manifest->pub.data.backupOptionBufferSize));
2178 }
2179
2180 if (manifest->pub.data.backupOptionChecksumPage != NULL)
2181 {
2182 infoSaveValue(
2183 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR,
2184 jsonFromVar(manifest->pub.data.backupOptionChecksumPage));
2185 }
2186
2187 // Set the option when compression is turned on. In older versions this also implied gz compression but in newer versions
2188 // the type option must also be set if compression is not gz.
2189 infoSaveValue(
2190 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_STR,
2191 jsonFromBool(manifest->pub.data.backupOptionCompressType != compressTypeNone));
2192
2193 if (manifest->pub.data.backupOptionCompressLevel != NULL)
2194 {
2195 infoSaveValue(
2196 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR,
2197 jsonFromVar(manifest->pub.data.backupOptionCompressLevel));
2198 }
2199
2200 if (manifest->pub.data.backupOptionCompressLevelNetwork != NULL)
2201 {
2202 infoSaveValue(
2203 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR,
2204 jsonFromVar(manifest->pub.data.backupOptionCompressLevelNetwork));
2205 }
2206
2207 // Set the compression type. Older versions will ignore this and assume gz compression if the compress option is set.
2208 infoSaveValue(
2209 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR,
2210 jsonFromStr(compressTypeStr(manifest->pub.data.backupOptionCompressType)));
2211
2212 if (manifest->pub.data.backupOptionDelta != NULL)
2213 {
2214 infoSaveValue(
2215 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_DELTA_STR,
2216 jsonFromVar(manifest->pub.data.backupOptionDelta));
2217 }
2218
2219 infoSaveValue(
2220 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_HARDLINK_STR,
2221 jsonFromBool(manifest->pub.data.backupOptionHardLink));
2222 infoSaveValue(
2223 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ONLINE_STR,
2224 jsonFromBool(manifest->pub.data.backupOptionOnline));
2225
2226 if (manifest->pub.data.backupOptionProcessMax != NULL)
2227 {
2228 infoSaveValue(
2229 infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_PROCESS_MAX_STR,
2230 jsonFromVar(manifest->pub.data.backupOptionProcessMax));
2231 }
2232 }
2233
2234 // -----------------------------------------------------------------------------------------------------------------------------
2235 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_TARGET_STR, sectionNext))
2236 {
2237 MEM_CONTEXT_TEMP_RESET_BEGIN()
2238 {
2239 for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++)
2240 {
2241 const ManifestTarget *target = manifestTarget(manifest, targetIdx);
2242 KeyValue *targetKv = kvNew();
2243
2244 if (target->file != NULL)
2245 kvPut(targetKv, MANIFEST_KEY_FILE_VAR, VARSTR(target->file));
2246
2247 kvPut(targetKv, MANIFEST_KEY_PATH_VAR, VARSTR(target->path));
2248
2249 if (target->tablespaceId != 0)
2250 kvPut(targetKv, MANIFEST_KEY_TABLESPACE_ID_VAR, VARSTR(strNewFmt("%u", target->tablespaceId)));
2251
2252 if (target->tablespaceName != NULL)
2253 kvPut(targetKv, MANIFEST_KEY_TABLESPACE_NAME_VAR, VARSTR(target->tablespaceName));
2254
2255 kvPut(
2256 targetKv, MANIFEST_KEY_TYPE_VAR,
2257 VARSTR(
2258 target->type == manifestTargetTypePath ?
2259 MANIFEST_TARGET_TYPE_PATH_STR : MANIFEST_TARGET_TYPE_LINK_STR));
2260
2261 infoSaveValue(infoSaveData, MANIFEST_SECTION_BACKUP_TARGET_STR, target->name, jsonFromKv(targetKv));
2262
2263 MEM_CONTEXT_TEMP_RESET(1000);
2264 }
2265 }
2266 MEM_CONTEXT_TEMP_END();
2267 }
2268
2269 // -----------------------------------------------------------------------------------------------------------------------------
2270 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_DB_STR, sectionNext))
2271 {
2272 MEM_CONTEXT_TEMP_RESET_BEGIN()
2273 {
2274 for (unsigned int dbIdx = 0; dbIdx < manifestDbTotal(manifest); dbIdx++)
2275 {
2276 const ManifestDb *db = manifestDb(manifest, dbIdx);
2277 KeyValue *dbKv = kvNew();
2278
2279 kvPut(dbKv, MANIFEST_KEY_DB_ID_VAR, VARUINT(db->id));
2280 kvPut(dbKv, MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR, VARUINT(db->lastSystemId));
2281
2282 infoSaveValue(infoSaveData, MANIFEST_SECTION_DB_STR, db->name, jsonFromKv(dbKv));
2283
2284 MEM_CONTEXT_TEMP_RESET(1000);
2285 }
2286 }
2287 MEM_CONTEXT_TEMP_END();
2288 }
2289
2290 // -----------------------------------------------------------------------------------------------------------------------------
2291 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, sectionNext))
2292 {
2293 MEM_CONTEXT_TEMP_RESET_BEGIN()
2294 {
2295 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
2296 {
2297 const ManifestFile *file = manifestFile(manifest, fileIdx);
2298 KeyValue *fileKv = kvNew();
2299
2300 // Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
2301 // save performed during a backup.
2302 if (file->size != 0 && file->checksumSha1[0] != 0)
2303 kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1));
2304
2305 if (file->checksumPage)
2306 {
2307 kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file->checksumPageError));
2308
2309 if (file->checksumPageErrorList != NULL)
2310 kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList));
2311 }
2312
2313 if (!varEq(manifestOwnerVar(file->group), saveData->fileGroupDefault))
2314 kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group));
2315
2316 if (file->primary != saveData->filePrimaryDefault)
2317 kvPut(fileKv, MANIFEST_KEY_PRIMARY_VAR, VARBOOL(file->primary));
2318
2319 if (file->mode != saveData->fileModeDefault)
2320 kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode)));
2321
2322 if (file->reference != NULL)
2323 kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference));
2324
2325 if (file->sizeRepo != file->size)
2326 kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, varNewUInt64(file->sizeRepo));
2327
2328 kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, varNewUInt64(file->size));
2329
2330 kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, varNewUInt64((uint64_t)file->timestamp));
2331
2332 if (!varEq(manifestOwnerVar(file->user), saveData->fileUserDefault))
2333 kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user));
2334
2335 infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv));
2336
2337 MEM_CONTEXT_TEMP_RESET(1000);
2338 }
2339 }
2340 MEM_CONTEXT_TEMP_END();
2341 }
2342
2343 // -----------------------------------------------------------------------------------------------------------------------------
2344 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, sectionNext))
2345 {
2346 infoSaveValue(
2347 infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_GROUP_STR,
2348 jsonFromVar(saveData->fileGroupDefault));
2349 infoSaveValue(
2350 infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_PRIMARY_STR,
2351 jsonFromBool(saveData->filePrimaryDefault));
2352 infoSaveValue(
2353 infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_MODE_STR,
2354 jsonFromStr(strNewFmt("%04o", saveData->fileModeDefault)));
2355 infoSaveValue(
2356 infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_USER_STR,
2357 jsonFromVar(saveData->fileUserDefault));
2358 }
2359
2360 // -----------------------------------------------------------------------------------------------------------------------------
2361 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_LINK_STR, sectionNext))
2362 {
2363 MEM_CONTEXT_TEMP_RESET_BEGIN()
2364 {
2365 for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(manifest); linkIdx++)
2366 {
2367 const ManifestLink *link = manifestLink(manifest, linkIdx);
2368 KeyValue *linkKv = kvNew();
2369
2370 if (!varEq(manifestOwnerVar(link->user), saveData->linkUserDefault))
2371 kvPut(linkKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(link->user));
2372
2373 if (!varEq(manifestOwnerVar(link->group), saveData->linkGroupDefault))
2374 kvPut(linkKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(link->group));
2375
2376 kvPut(linkKv, MANIFEST_KEY_DESTINATION_VAR, VARSTR(link->destination));
2377
2378 infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_LINK_STR, link->name, jsonFromKv(linkKv));
2379
2380 MEM_CONTEXT_TEMP_RESET(1000);
2381 }
2382 }
2383 MEM_CONTEXT_TEMP_END();
2384 }
2385
2386 // -----------------------------------------------------------------------------------------------------------------------------
2387 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, sectionNext))
2388 {
2389 if (manifestLinkTotal(manifest) > 0)
2390 {
2391 infoSaveValue(
2392 infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, MANIFEST_KEY_GROUP_STR,
2393 jsonFromVar(saveData->linkGroupDefault));
2394 infoSaveValue(
2395 infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, MANIFEST_KEY_USER_STR,
2396 jsonFromVar(saveData->linkUserDefault));
2397 }
2398 }
2399
2400 // -----------------------------------------------------------------------------------------------------------------------------
2401 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_PATH_STR, sectionNext))
2402 {
2403 MEM_CONTEXT_TEMP_RESET_BEGIN()
2404 {
2405 for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(manifest); pathIdx++)
2406 {
2407 const ManifestPath *path = manifestPath(manifest, pathIdx);
2408 KeyValue *pathKv = kvNew();
2409
2410 if (!varEq(manifestOwnerVar(path->group), saveData->pathGroupDefault))
2411 kvPut(pathKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(path->group));
2412
2413 if (path->mode != saveData->pathModeDefault)
2414 kvPut(pathKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", path->mode)));
2415
2416 if (!varEq(manifestOwnerVar(path->user), saveData->pathUserDefault))
2417 kvPut(pathKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(path->user));
2418
2419 infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_PATH_STR, path->name, jsonFromKv(pathKv));
2420
2421 MEM_CONTEXT_TEMP_RESET(1000);
2422 }
2423 }
2424 MEM_CONTEXT_TEMP_END();
2425 }
2426
2427 // -----------------------------------------------------------------------------------------------------------------------------
2428 if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, sectionNext))
2429 {
2430 infoSaveValue(
2431 infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_GROUP_STR,
2432 jsonFromVar(saveData->pathGroupDefault));
2433 infoSaveValue(
2434 infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_MODE_STR,
2435 jsonFromStr(strNewFmt("%04o", saveData->pathModeDefault)));
2436 infoSaveValue(
2437 infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_USER_STR,
2438 jsonFromVar(saveData->pathUserDefault));
2439 }
2440
2441 FUNCTION_TEST_RETURN_VOID();
2442 }
2443
2444 void
manifestSave(Manifest * this,IoWrite * write)2445 manifestSave(Manifest *this, IoWrite *write)
2446 {
2447 FUNCTION_LOG_BEGIN(logLevelDebug);
2448 FUNCTION_LOG_PARAM(MANIFEST, this);
2449 FUNCTION_LOG_PARAM(IO_WRITE, write);
2450 FUNCTION_LOG_END();
2451
2452 ASSERT(this != NULL);
2453 ASSERT(write != NULL);
2454
2455 MEM_CONTEXT_TEMP_BEGIN()
2456 {
2457 // Files can be added from outside the manifest so make sure they are sorted
2458 lstSort(this->pub.fileList, sortOrderAsc);
2459
2460 ManifestSaveData saveData =
2461 {
2462 .manifest = this,
2463 };
2464
2465 // Get default file values
2466 MostCommonValue *fileGroupMcv = mcvNew();
2467 MostCommonValue *fileModeMcv = mcvNew();
2468 MostCommonValue *filePrimaryMcv = mcvNew();
2469 MostCommonValue *fileUserMcv = mcvNew();
2470
2471 ASSERT(manifestFileTotal(this) > 0);
2472
2473 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
2474 {
2475 const ManifestFile *file = manifestFile(this, fileIdx);
2476
2477 mcvUpdate(fileGroupMcv, VARSTR(file->group));
2478 mcvUpdate(fileModeMcv, VARUINT(file->mode));
2479 mcvUpdate(filePrimaryMcv, VARBOOL(file->primary));
2480 mcvUpdate(fileUserMcv, VARSTR(file->user));
2481 }
2482
2483 saveData.fileGroupDefault = manifestOwnerVar(varStr(mcvResult(fileGroupMcv)));
2484 saveData.fileModeDefault = (mode_t)varUInt(mcvResult(fileModeMcv));
2485 saveData.filePrimaryDefault = varBool(mcvResult(filePrimaryMcv));
2486 saveData.fileUserDefault = manifestOwnerVar(varStr(mcvResult(fileUserMcv)));
2487
2488 // Get default link values
2489 if (manifestLinkTotal(this) > 0)
2490 {
2491 MostCommonValue *linkGroupMcv = mcvNew();
2492 MostCommonValue *linkUserMcv = mcvNew();
2493
2494 for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(this); linkIdx++)
2495 {
2496 const ManifestLink *link = manifestLink(this, linkIdx);
2497
2498 mcvUpdate(linkGroupMcv, VARSTR(link->group));
2499 mcvUpdate(linkUserMcv, VARSTR(link->user));
2500 }
2501
2502 saveData.linkGroupDefault = manifestOwnerVar(varStr(mcvResult(linkGroupMcv)));
2503 saveData.linkUserDefault = manifestOwnerVar(varStr(mcvResult(linkUserMcv)));
2504 }
2505
2506 // Get default path values
2507 MostCommonValue *pathGroupMcv = mcvNew();
2508 MostCommonValue *pathModeMcv = mcvNew();
2509 MostCommonValue *pathUserMcv = mcvNew();
2510
2511 ASSERT(manifestPathTotal(this) > 0);
2512
2513 for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(this); pathIdx++)
2514 {
2515 const ManifestPath *path = manifestPath(this, pathIdx);
2516
2517 mcvUpdate(pathGroupMcv, VARSTR(path->group));
2518 mcvUpdate(pathModeMcv, VARUINT(path->mode));
2519 mcvUpdate(pathUserMcv, VARSTR(path->user));
2520 }
2521
2522 saveData.pathGroupDefault = manifestOwnerVar(varStr(mcvResult(pathGroupMcv)));
2523 saveData.pathModeDefault = (mode_t)varUInt(mcvResult(pathModeMcv));
2524 saveData.pathUserDefault = manifestOwnerVar(varStr(mcvResult(pathUserMcv)));
2525
2526 // Save manifest
2527 infoSave(this->pub.info, write, manifestSaveCallback, &saveData);
2528 }
2529 MEM_CONTEXT_TEMP_END();
2530
2531 FUNCTION_LOG_RETURN_VOID();
2532 }
2533
2534 /**********************************************************************************************************************************/
2535 void
manifestValidate(Manifest * this,bool strict)2536 manifestValidate(Manifest *this, bool strict)
2537 {
2538 FUNCTION_LOG_BEGIN(logLevelDebug);
2539 FUNCTION_LOG_PARAM(MANIFEST, this);
2540 FUNCTION_LOG_PARAM(BOOL, strict);
2541 FUNCTION_LOG_END();
2542
2543 ASSERT(this != NULL);
2544
2545 MEM_CONTEXT_TEMP_BEGIN()
2546 {
2547 String *error = strNew();
2548
2549 // Validate files
2550 for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
2551 {
2552 const ManifestFile *file = manifestFile(this, fileIdx);
2553
2554 // All files must have a checksum
2555 if (file->checksumSha1[0] == '\0')
2556 strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file->name));
2557
2558 // These are strict checks to be performed only after a backup and before the final manifest save
2559 if (strict)
2560 {
2561 // Zero-length files must have a specific checksum
2562 if (file->size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file->checksumSha1))
2563 strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file->checksumSha1, strZ(file->name));
2564
2565 // Non-zero size files must have non-zero repo size
2566 if (file->sizeRepo == 0 && file->size != 0)
2567 strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file->name));
2568 }
2569 }
2570
2571 // Throw exception when there are errors
2572 if (strSize(error) > 0)
2573 THROW_FMT(FormatError, "manifest validation failed:%s", strZ(error));
2574 }
2575 MEM_CONTEXT_TEMP_END();
2576
2577 FUNCTION_LOG_RETURN_VOID();
2578 }
2579
2580 /***********************************************************************************************************************************
2581 Db functions and getters/setters
2582 ***********************************************************************************************************************************/
2583 const ManifestDb *
manifestDbFind(const Manifest * this,const String * name)2584 manifestDbFind(const Manifest *this, const String *name)
2585 {
2586 FUNCTION_TEST_BEGIN();
2587 FUNCTION_TEST_PARAM(MANIFEST, this);
2588 FUNCTION_TEST_PARAM(STRING, name);
2589 FUNCTION_TEST_END();
2590
2591 ASSERT(this != NULL);
2592 ASSERT(name != NULL);
2593
2594 const ManifestDb *result = lstFind(this->pub.dbList, &name);
2595
2596 if (result == NULL)
2597 THROW_FMT(AssertError, "unable to find '%s' in manifest db list", strZ(name));
2598
2599 FUNCTION_TEST_RETURN(result);
2600 }
2601
2602 /***********************************************************************************************************************************
2603 File functions and getters/setters
2604 ***********************************************************************************************************************************/
2605 const ManifestFile *
manifestFileFind(const Manifest * this,const String * name)2606 manifestFileFind(const Manifest *this, const String *name)
2607 {
2608 FUNCTION_TEST_BEGIN();
2609 FUNCTION_TEST_PARAM(MANIFEST, this);
2610 FUNCTION_TEST_PARAM(STRING, name);
2611 FUNCTION_TEST_END();
2612
2613 ASSERT(this != NULL);
2614 ASSERT(name != NULL);
2615
2616 const ManifestFile *result = lstFind(this->pub.fileList, &name);
2617
2618 if (result == NULL)
2619 THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name));
2620
2621 FUNCTION_TEST_RETURN(result);
2622 }
2623
2624 void
manifestFileRemove(const Manifest * this,const String * name)2625 manifestFileRemove(const Manifest *this, const String *name)
2626 {
2627 FUNCTION_TEST_BEGIN();
2628 FUNCTION_TEST_PARAM(MANIFEST, this);
2629 FUNCTION_TEST_PARAM(STRING, name);
2630 FUNCTION_TEST_END();
2631
2632 ASSERT(this != NULL);
2633 ASSERT(name != NULL);
2634
2635 if (!lstRemove(this->pub.fileList, &name))
2636 THROW_FMT(AssertError, "unable to remove '%s' from manifest file list", strZ(name));
2637
2638 FUNCTION_TEST_RETURN_VOID();
2639 }
2640
2641 void
manifestFileUpdate(Manifest * this,const String * name,uint64_t size,uint64_t sizeRepo,const char * checksumSha1,const Variant * reference,bool checksumPage,bool checksumPageError,const VariantList * checksumPageErrorList)2642 manifestFileUpdate(
2643 Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
2644 bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList)
2645 {
2646 FUNCTION_TEST_BEGIN();
2647 FUNCTION_TEST_PARAM(MANIFEST, this);
2648 FUNCTION_TEST_PARAM(STRING, name);
2649 FUNCTION_TEST_PARAM(UINT64, size);
2650 FUNCTION_TEST_PARAM(UINT64, sizeRepo);
2651 FUNCTION_TEST_PARAM(STRINGZ, checksumSha1);
2652 FUNCTION_TEST_PARAM(VARIANT, reference);
2653 FUNCTION_TEST_PARAM(BOOL, checksumPage);
2654 FUNCTION_TEST_PARAM(BOOL, checksumPageError);
2655 FUNCTION_TEST_PARAM(VARIANT_LIST, checksumPageErrorList);
2656 FUNCTION_TEST_END();
2657
2658 ASSERT(this != NULL);
2659 ASSERT(name != NULL);
2660 ASSERT(
2661 (!checksumPage && !checksumPageError && checksumPageErrorList == NULL) ||
2662 (checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError));
2663
2664 ManifestFile *file = (ManifestFile *)manifestFileFind(this, name);
2665
2666 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
2667 {
2668 // Update reference if set
2669 if (reference != NULL)
2670 {
2671 if (varStr(reference) == NULL)
2672 file->reference = NULL;
2673 else
2674 file->reference = strLstAddIfMissing(this->referenceList, varStr(reference));
2675 }
2676
2677 // Update checksum if set
2678 if (checksumSha1 != NULL)
2679 memcpy(file->checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
2680
2681 // Update repo size
2682 file->size = size;
2683 file->sizeRepo = sizeRepo;
2684
2685 // Update checksum page info
2686 file->checksumPage = checksumPage;
2687 file->checksumPageError = checksumPageError;
2688 file->checksumPageErrorList = varLstDup(checksumPageErrorList);
2689 }
2690 MEM_CONTEXT_END();
2691
2692 FUNCTION_TEST_RETURN_VOID();
2693 }
2694
2695 /***********************************************************************************************************************************
2696 Link functions and getters/setters
2697 ***********************************************************************************************************************************/
2698 const ManifestLink *
manifestLinkFind(const Manifest * this,const String * name)2699 manifestLinkFind(const Manifest *this, const String *name)
2700 {
2701 FUNCTION_TEST_BEGIN();
2702 FUNCTION_TEST_PARAM(MANIFEST, this);
2703 FUNCTION_TEST_PARAM(STRING, name);
2704 FUNCTION_TEST_END();
2705
2706 ASSERT(this != NULL);
2707 ASSERT(name != NULL);
2708
2709 const ManifestLink *result = lstFind(this->pub.linkList, &name);
2710
2711 if (result == NULL)
2712 THROW_FMT(AssertError, "unable to find '%s' in manifest link list", strZ(name));
2713
2714 FUNCTION_TEST_RETURN(result);
2715 }
2716
2717 void
manifestLinkRemove(const Manifest * this,const String * name)2718 manifestLinkRemove(const Manifest *this, const String *name)
2719 {
2720 FUNCTION_TEST_BEGIN();
2721 FUNCTION_TEST_PARAM(MANIFEST, this);
2722 FUNCTION_TEST_PARAM(STRING, name);
2723 FUNCTION_TEST_END();
2724
2725 ASSERT(this != NULL);
2726 ASSERT(name != NULL);
2727
2728 if (!lstRemove(this->pub.linkList, &name))
2729 THROW_FMT(AssertError, "unable to remove '%s' from manifest link list", strZ(name));
2730
2731 FUNCTION_TEST_RETURN_VOID();
2732 }
2733
2734 void
manifestLinkUpdate(const Manifest * this,const String * name,const String * destination)2735 manifestLinkUpdate(const Manifest *this, const String *name, const String *destination)
2736 {
2737 FUNCTION_TEST_BEGIN();
2738 FUNCTION_TEST_PARAM(MANIFEST, this);
2739 FUNCTION_TEST_PARAM(STRING, name);
2740 FUNCTION_TEST_PARAM(STRING, destination);
2741 FUNCTION_TEST_END();
2742
2743 ASSERT(this != NULL);
2744 ASSERT(name != NULL);
2745 ASSERT(destination != NULL);
2746
2747 ManifestLink *link = (ManifestLink *)manifestLinkFind(this, name);
2748
2749 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.linkList))
2750 {
2751 if (!strEq(link->destination, destination))
2752 link->destination = strDup(destination);
2753 }
2754 MEM_CONTEXT_END();
2755
2756 FUNCTION_TEST_RETURN_VOID();
2757 }
2758
2759 /***********************************************************************************************************************************
2760 Path functions and getters/setters
2761 ***********************************************************************************************************************************/
2762 const ManifestPath *
manifestPathFind(const Manifest * this,const String * name)2763 manifestPathFind(const Manifest *this, const String *name)
2764 {
2765 FUNCTION_TEST_BEGIN();
2766 FUNCTION_TEST_PARAM(MANIFEST, this);
2767 FUNCTION_TEST_PARAM(STRING, name);
2768 FUNCTION_TEST_END();
2769
2770 ASSERT(this != NULL);
2771 ASSERT(name != NULL);
2772
2773 const ManifestPath *result = lstFind(this->pub.pathList, &name);
2774
2775 if (result == NULL)
2776 THROW_FMT(AssertError, "unable to find '%s' in manifest path list", strZ(name));
2777
2778 FUNCTION_TEST_RETURN(result);
2779 }
2780
2781 String *
manifestPathPg(const String * manifestPath)2782 manifestPathPg(const String *manifestPath)
2783 {
2784 FUNCTION_TEST_BEGIN();
2785 FUNCTION_TEST_PARAM(STRING, manifestPath);
2786 FUNCTION_TEST_END();
2787
2788 ASSERT(manifestPath != NULL);
2789
2790 // If something in pg_data/
2791 if (strBeginsWith(manifestPath, STRDEF(MANIFEST_TARGET_PGDATA "/")))
2792 {
2793 FUNCTION_TEST_RETURN(strNewZ(strZ(manifestPath) + sizeof(MANIFEST_TARGET_PGDATA)));
2794 }
2795 // Else not pg_data (this is faster since the length of everything else will be different than pg_data)
2796 else if (!strEq(manifestPath, MANIFEST_TARGET_PGDATA_STR))
2797 {
2798 // A tablespace target is the only valid option if not pg_data or pg_data/
2799 ASSERT(
2800 strEq(manifestPath, MANIFEST_TARGET_PGTBLSPC_STR) || strBeginsWith(manifestPath, STRDEF(MANIFEST_TARGET_PGTBLSPC "/")));
2801
2802 FUNCTION_TEST_RETURN(strDup(manifestPath));
2803 }
2804
2805 FUNCTION_TEST_RETURN(NULL);
2806 }
2807
2808 /***********************************************************************************************************************************
2809 Target functions and getters/setters
2810 ***********************************************************************************************************************************/
2811 const ManifestTarget *
manifestTargetFind(const Manifest * this,const String * name)2812 manifestTargetFind(const Manifest *this, const String *name)
2813 {
2814 FUNCTION_TEST_BEGIN();
2815 FUNCTION_TEST_PARAM(MANIFEST, this);
2816 FUNCTION_TEST_PARAM(STRING, name);
2817 FUNCTION_TEST_END();
2818
2819 ASSERT(this != NULL);
2820 ASSERT(name != NULL);
2821
2822 const ManifestTarget *result = lstFind(this->pub.targetList, &name);
2823
2824 if (result == NULL)
2825 THROW_FMT(AssertError, "unable to find '%s' in manifest target list", strZ(name));
2826
2827 FUNCTION_TEST_RETURN(result);
2828 }
2829
2830 String *
manifestTargetPath(const Manifest * this,const ManifestTarget * target)2831 manifestTargetPath(const Manifest *this, const ManifestTarget *target)
2832 {
2833 FUNCTION_TEST_BEGIN();
2834 FUNCTION_TEST_PARAM(MANIFEST, this);
2835 FUNCTION_TEST_PARAM(MANIFEST_TARGET, target);
2836 FUNCTION_TEST_END();
2837
2838 ASSERT(this != NULL);
2839 ASSERT(target != NULL);
2840
2841 // If the target path is already absolute then just return it
2842 if (strBeginsWith(target->path, FSLASH_STR))
2843 FUNCTION_TEST_RETURN(strDup(target->path));
2844
2845 // Construct it from the base pg path and a relative path
2846 String *result = NULL;
2847
2848 MEM_CONTEXT_TEMP_BEGIN()
2849 {
2850 String *pgPath = strPath(manifestPathPg(target->name));
2851
2852 if (strSize(pgPath) != 0)
2853 strCatZ(pgPath, "/");
2854
2855 strCat(pgPath, target->path);
2856
2857 MEM_CONTEXT_PRIOR_BEGIN()
2858 {
2859 result = strPathAbsolute(pgPath, manifestTargetBase(this)->path);
2860 }
2861 MEM_CONTEXT_PRIOR_END();
2862 }
2863 MEM_CONTEXT_TEMP_END();
2864
2865 FUNCTION_TEST_RETURN(result);
2866 }
2867
2868 void
manifestTargetRemove(const Manifest * this,const String * name)2869 manifestTargetRemove(const Manifest *this, const String *name)
2870 {
2871 FUNCTION_TEST_BEGIN();
2872 FUNCTION_TEST_PARAM(MANIFEST, this);
2873 FUNCTION_TEST_PARAM(STRING, name);
2874 FUNCTION_TEST_END();
2875
2876 ASSERT(this != NULL);
2877 ASSERT(name != NULL);
2878
2879 if (!lstRemove(this->pub.targetList, &name))
2880 THROW_FMT(AssertError, "unable to remove '%s' from manifest target list", strZ(name));
2881
2882 FUNCTION_TEST_RETURN_VOID();
2883 }
2884
2885 void
manifestTargetUpdate(const Manifest * this,const String * name,const String * path,const String * file)2886 manifestTargetUpdate(const Manifest *this, const String *name, const String *path, const String *file)
2887 {
2888 FUNCTION_TEST_BEGIN();
2889 FUNCTION_TEST_PARAM(MANIFEST, this);
2890 FUNCTION_TEST_PARAM(STRING, name);
2891 FUNCTION_TEST_PARAM(STRING, path);
2892 FUNCTION_TEST_PARAM(STRING, file);
2893 FUNCTION_TEST_END();
2894
2895 ASSERT(this != NULL);
2896 ASSERT(name != NULL);
2897 ASSERT(path != NULL);
2898
2899 ManifestTarget *target = (ManifestTarget *)manifestTargetFind(this, name);
2900
2901 ASSERT((target->file == NULL && file == NULL) || (target->file != NULL && file != NULL));
2902
2903 MEM_CONTEXT_BEGIN(lstMemContext(this->pub.targetList))
2904 {
2905 if (!strEq(target->path, path))
2906 target->path = strDup(path);
2907
2908 if (!strEq(target->file, file))
2909 target->file = strDup(file);
2910 }
2911 MEM_CONTEXT_END();
2912
2913 FUNCTION_TEST_RETURN_VOID();
2914 }
2915
2916 /***********************************************************************************************************************************
2917 Getters/Setters
2918 ***********************************************************************************************************************************/
2919 void
manifestBackupLabelSet(Manifest * this,const String * backupLabel)2920 manifestBackupLabelSet(Manifest *this, const String *backupLabel)
2921 {
2922 FUNCTION_TEST_BEGIN();
2923 FUNCTION_TEST_PARAM(MANIFEST, this);
2924 FUNCTION_TEST_PARAM(STRING, backupLabel);
2925 FUNCTION_TEST_END();
2926
2927 ASSERT(this != NULL);
2928
2929 MEM_CONTEXT_BEGIN(this->pub.memContext)
2930 {
2931 this->pub.data.backupLabel = strDup(backupLabel);
2932 }
2933 MEM_CONTEXT_END();
2934
2935 FUNCTION_TEST_RETURN_VOID();
2936 }
2937
2938 /**********************************************************************************************************************************/
2939 typedef struct ManifestLoadFileData
2940 {
2941 MemContext *memContext; // Mem context
2942 const Storage *storage; // Storage to load from
2943 const String *fileName; // Base filename
2944 CipherType cipherType; // Cipher type
2945 const String *cipherPass; // Cipher passphrase
2946 Manifest *manifest; // Loaded manifest object
2947 } ManifestLoadFileData;
2948
2949 static bool
manifestLoadFileCallback(void * data,unsigned int try)2950 manifestLoadFileCallback(void *data, unsigned int try)
2951 {
2952 FUNCTION_LOG_BEGIN(logLevelTrace);
2953 FUNCTION_LOG_PARAM_P(VOID, data);
2954 FUNCTION_LOG_PARAM(UINT, try);
2955 FUNCTION_LOG_END();
2956
2957 ASSERT(data != NULL);
2958
2959 ManifestLoadFileData *loadData = (ManifestLoadFileData *)data;
2960 bool result = false;
2961
2962 if (try < 2)
2963 {
2964 // Construct filename based on try
2965 const String *fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName));
2966
2967 // Attempt to load the file
2968 IoRead *read = storageReadIo(storageNewReadP(loadData->storage, fileName));
2969 cipherBlockFilterGroupAdd(ioReadFilterGroup(read), loadData->cipherType, cipherModeDecrypt, loadData->cipherPass);
2970
2971 MEM_CONTEXT_BEGIN(loadData->memContext)
2972 {
2973 loadData->manifest = manifestNewLoad(read);
2974 result = true;
2975 }
2976 MEM_CONTEXT_END();
2977 }
2978
2979 FUNCTION_LOG_RETURN(BOOL, result);
2980 }
2981
2982 Manifest *
manifestLoadFile(const Storage * storage,const String * fileName,CipherType cipherType,const String * cipherPass)2983 manifestLoadFile(const Storage *storage, const String *fileName, CipherType cipherType, const String *cipherPass)
2984 {
2985 FUNCTION_LOG_BEGIN(logLevelDebug);
2986 FUNCTION_LOG_PARAM(STORAGE, storage);
2987 FUNCTION_LOG_PARAM(STRING, fileName);
2988 FUNCTION_LOG_PARAM(STRING_ID, cipherType);
2989 FUNCTION_TEST_PARAM(STRING, cipherPass);
2990 FUNCTION_LOG_END();
2991
2992 ASSERT(storage != NULL);
2993 ASSERT(fileName != NULL);
2994 ASSERT((cipherType == cipherTypeNone && cipherPass == NULL) || (cipherType != cipherTypeNone && cipherPass != NULL));
2995
2996 ManifestLoadFileData data =
2997 {
2998 .memContext = memContextCurrent(),
2999 .storage = storage,
3000 .fileName = fileName,
3001 .cipherType = cipherType,
3002 .cipherPass = cipherPass,
3003 };
3004
3005 MEM_CONTEXT_TEMP_BEGIN()
3006 {
3007 const char *fileNamePath = strZ(storagePathP(storage, fileName));
3008
3009 infoLoad(
3010 strNewFmt("unable to load backup manifest file '%s' or '%s" INFO_COPY_EXT "'", fileNamePath, fileNamePath),
3011 manifestLoadFileCallback, &data);
3012 }
3013 MEM_CONTEXT_TEMP_END();
3014
3015 FUNCTION_LOG_RETURN(MANIFEST, data.manifest);
3016 }
3017