1 /***********************************************************************************************************************************
2 Test Backup Command
3 ***********************************************************************************************************************************/
4 #include "command/stanza/create.h"
5 #include "command/stanza/upgrade.h"
6 #include "common/crypto/hash.h"
7 #include "common/io/bufferWrite.h"
8 #include "postgres/interface/static.vendor.h"
9 #include "storage/helper.h"
10 #include "storage/posix/storage.h"
11
12 #include "common/harnessConfig.h"
13 #include "common/harnessPostgres.h"
14 #include "common/harnessPq.h"
15 #include "common/harnessProtocol.h"
16 #include "common/harnessStorage.h"
17
18 /***********************************************************************************************************************************
19 Get a list of all files in the backup and a redacted version of the manifest that can be tested against a static string
20 ***********************************************************************************************************************************/
21 typedef struct TestBackupValidateCallbackData
22 {
23 const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
24 const String *path; // Subpath when storage is specified
25 const Manifest *manifest; // Manifest to check for files/links/paths
26 const ManifestData *manifestData; // Manifest data
27 String *content; // String where content should be added
28 } TestBackupValidateCallbackData;
29
30 void
testBackupValidateCallback(void * callbackData,const StorageInfo * info)31 testBackupValidateCallback(void *callbackData, const StorageInfo *info)
32 {
33 TestBackupValidateCallbackData *data = callbackData;
34
35 // Don't include . when it is a path (we'll still include it when it is a link so we can see the destination)
36 if (info->type == storageTypePath && strEq(info->name, DOT_STR))
37 return;
38
39 // Don't include backup.manifest or copy. We'll test that they are present elsewhere
40 if (info->type == storageTypeFile &&
41 (strEqZ(info->name, BACKUP_MANIFEST_FILE) || strEqZ(info->name, BACKUP_MANIFEST_FILE INFO_COPY_EXT)))
42 return;
43
44 // Get manifest name
45 const String *manifestName = info->name;
46
47 strCatFmt(data->content, "%s {", strZ(info->name));
48
49 switch (info->type)
50 {
51 case storageTypeFile:
52 {
53 strCatZ(data->content, "file");
54
55 // Calculate checksum/size and decompress if needed
56 // ---------------------------------------------------------------------------------------------------------------------
57 StorageRead *read = storageNewReadP(
58 data->storage, data->path != NULL ? strNewFmt("%s/%s", strZ(data->path), strZ(info->name)) : info->name);
59
60 if (data->manifestData->backupOptionCompressType != compressTypeNone)
61 {
62 ioFilterGroupAdd(
63 ioReadFilterGroup(storageReadIo(read)), decompressFilter(data->manifestData->backupOptionCompressType));
64 manifestName = strSubN(
65 info->name, 0, strSize(info->name) - strSize(compressExtStr(data->manifestData->backupOptionCompressType)));
66 }
67
68 ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
69
70 uint64_t size = bufUsed(storageGetP(read));
71 const String *checksum = varStr(
72 ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR));
73
74 strCatFmt(data->content, ", s=%" PRIu64, size);
75
76 // Check against the manifest
77 // ---------------------------------------------------------------------------------------------------------------------
78 const ManifestFile *file = manifestFileFind(data->manifest, manifestName);
79
80 // Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
81 // compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
82 // compression.
83 if (size != file->size)
84 THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName));
85
86 if (info->size != file->sizeRepo)
87 THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName));
88
89 if (data->manifestData->backupOptionCompressType != compressTypeNone)
90 ((ManifestFile *)file)->sizeRepo = file->size;
91
92 // Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
93 // the checksum from the test output.
94 if (!strEqZ(checksum, file->checksumSha1))
95 THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName));
96
97 if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
98 strBeginsWith(
99 manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
100 {
101 ((ManifestFile *)file)->checksumSha1[0] = '\0';
102 }
103
104 // Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
105 // mode and current user/group.
106 if (info->mode != 0640)
107 THROW_FMT(AssertError, "'%s' mode is not 0640", strZ(manifestName));
108
109 if (!strEq(info->user, TEST_USER_STR))
110 THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(manifestName));
111
112 if (!strEq(info->group, TEST_GROUP_STR))
113 THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName));
114
115 break;
116 }
117
118 case storageTypeLink:
119 strCatFmt(data->content, "link, d=%s", strZ(info->linkDestination));
120 break;
121
122 case storageTypePath:
123 {
124 strCatZ(data->content, "path");
125
126 // Check against the manifest
127 // ---------------------------------------------------------------------------------------------------------------------
128 manifestPathFind(data->manifest, info->name);
129
130 // Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
131 // mode and current user/group.
132 if (info->mode != 0750)
133 THROW_FMT(AssertError, "'%s' mode is not 00750", strZ(info->name));
134
135 if (!strEq(info->user, TEST_USER_STR))
136 THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(info->name));
137
138 if (!strEq(info->group, TEST_GROUP_STR))
139 THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(info->name));
140
141 break;
142 }
143
144 case storageTypeSpecial:
145 THROW_FMT(AssertError, "unexpected special file '%s'", strZ(info->name));
146 }
147
148 strCatZ(data->content, "}\n");
149 }
150
151 static String *
testBackupValidate(const Storage * storage,const String * path)152 testBackupValidate(const Storage *storage, const String *path)
153 {
154 FUNCTION_HARNESS_BEGIN();
155 FUNCTION_HARNESS_PARAM(STORAGE, storage);
156 FUNCTION_HARNESS_PARAM(STRING, path);
157 FUNCTION_HARNESS_END();
158
159 String *result = strNew();
160
161 MEM_CONTEXT_TEMP_BEGIN()
162 {
163 // Build a list of files in the backup path and verify against the manifest
164 // -------------------------------------------------------------------------------------------------------------------------
165 Manifest *manifest = manifestLoadFile(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strZ(path)), cipherTypeNone, NULL);
166
167 TestBackupValidateCallbackData callbackData =
168 {
169 .storage = storage,
170 .path = path,
171 .content = result,
172 .manifest = manifest,
173 .manifestData = manifestData(manifest),
174 };
175
176 storageInfoListP(storage, path, testBackupValidateCallback, &callbackData, .recurse = true, .sortOrder = sortOrderAsc);
177
178 // Make sure both backup.manifest files exist since we skipped them in the callback above
179 if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strZ(path))))
180 THROW(AssertError, BACKUP_MANIFEST_FILE " is missing");
181
182 if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(path))))
183 THROW(AssertError, BACKUP_MANIFEST_FILE INFO_COPY_EXT " is missing");
184
185 // Output the manifest to a string and exclude sections that don't need validation. Note that each of these sections should
186 // be considered from automatic validation but adding them to the output will make the tests too noisy. One good technique
187 // would be to remove it from the output only after validation so new values will cause changes in the output.
188 // -------------------------------------------------------------------------------------------------------------------------
189 Buffer *manifestSaveBuffer = bufNew(0);
190 manifestSave(manifest, ioBufferWriteNew(manifestSaveBuffer));
191
192 String *manifestEdit = strNew();
193 StringList *manifestLine = strLstNewSplitZ(strTrim(strNewBuf(manifestSaveBuffer)), "\n");
194 bool bSkipSection = false;
195
196 for (unsigned int lineIdx = 0; lineIdx < strLstSize(manifestLine); lineIdx++)
197 {
198 const String *line = strTrim(strLstGet(manifestLine, lineIdx));
199
200 if (strChr(line, '[') == 0)
201 {
202 const String *section = strSubN(line, 1, strSize(line) - 2);
203
204 if (strEq(section, INFO_SECTION_BACKREST_STR) ||
205 strEq(section, MANIFEST_SECTION_BACKUP_STR) ||
206 strEq(section, MANIFEST_SECTION_BACKUP_DB_STR) ||
207 strEq(section, MANIFEST_SECTION_BACKUP_OPTION_STR) ||
208 strEq(section, MANIFEST_SECTION_DB_STR) ||
209 strEq(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR) ||
210 strEq(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR) ||
211 strEq(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR))
212 {
213 bSkipSection = true;
214 }
215 else
216 bSkipSection = false;
217 }
218
219 if (!bSkipSection)
220 strCatFmt(manifestEdit, "%s\n", strZ(line));
221 }
222
223 strCatFmt(result, "--------\n%s\n", strZ(strTrim(manifestEdit)));
224 }
225 MEM_CONTEXT_TEMP_END();
226
227 FUNCTION_HARNESS_RETURN(STRING, result);
228 }
229
230 /***********************************************************************************************************************************
231 Generate pq scripts for versions of PostgreSQL
232 ***********************************************************************************************************************************/
233 typedef struct TestBackupPqScriptParam
234 {
235 VAR_PARAM_HEADER;
236 bool startFast;
237 bool backupStandby;
238 bool errorAfterStart;
239 bool noWal; // Don't write test WAL segments
240 CompressType walCompressType; // Compress type for the archive files
241 unsigned int walTotal; // Total WAL to write
242 unsigned int timeline; // Timeline to use for WAL files
243 } TestBackupPqScriptParam;
244
245 #define testBackupPqScriptP(pgVersion, backupStartTime, ...) \
246 testBackupPqScript(pgVersion, backupStartTime, (TestBackupPqScriptParam){VAR_PARAM_INIT, __VA_ARGS__})
247
248 static void
testBackupPqScript(unsigned int pgVersion,time_t backupTimeStart,TestBackupPqScriptParam param)249 testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqScriptParam param)
250 {
251 const char *pg1Path = TEST_PATH "/pg1";
252 const char *pg2Path = TEST_PATH "/pg2";
253
254 // If no timeline specified then use timeline 1
255 param.timeline = param.timeline == 0 ? 1 : param.timeline;
256
257 // Read pg_control to get info about the cluster
258 PgControl pgControl = pgControlFromFile(storagePg());
259
260 // Set archive timeout really small to save time on errors
261 cfgOptionSet(cfgOptArchiveTimeout, cfgSourceParam, varNewInt64(100));
262
263 uint64_t lsnStart = ((uint64_t)backupTimeStart & 0xFFFFFF00) << 28;
264 uint64_t lsnStop =
265 lsnStart + ((param.walTotal == 0 ? 0 : param.walTotal - 1) * pgControl.walSegmentSize) + (pgControl.walSegmentSize / 2);
266
267 const char *lsnStartStr = strZ(pgLsnToStr(lsnStart));
268 const char *walSegmentStart = strZ(pgLsnToWalSegment(param.timeline, lsnStart, pgControl.walSegmentSize));
269 const char *lsnStopStr = strZ(pgLsnToStr(lsnStop));
270 const char *walSegmentStop = strZ(pgLsnToWalSegment(param.timeline, lsnStop, pgControl.walSegmentSize));
271
272 // Write WAL segments to the archive
273 // -----------------------------------------------------------------------------------------------------------------------------
274 if (!param.noWal)
275 {
276 InfoArchive *infoArchive = infoArchiveLoadFile(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cipherTypeNone, NULL);
277 const String *archiveId = infoArchiveId(infoArchive);
278 StringList *walSegmentList = pgLsnRangeToWalSegmentList(
279 pgControl.version, param.timeline, lsnStart, lsnStop, pgControl.walSegmentSize);
280
281 Buffer *walBuffer = bufNew((size_t)pgControl.walSegmentSize);
282 bufUsedSet(walBuffer, bufSize(walBuffer));
283 memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
284 hrnPgWalToBuffer((PgWal){.version = pgControl.version, .systemId = pgControl.systemId}, walBuffer);
285 const String *walChecksum = bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer));
286
287 for (unsigned int walSegmentIdx = 0; walSegmentIdx < strLstSize(walSegmentList); walSegmentIdx++)
288 {
289 StorageWrite *write = storageNewWriteP(
290 storageRepoWrite(),
291 strNewFmt(
292 STORAGE_REPO_ARCHIVE "/%s/%s-%s%s", strZ(archiveId), strZ(strLstGet(walSegmentList, walSegmentIdx)),
293 strZ(walChecksum), strZ(compressExtStr(param.walCompressType))));
294
295 if (param.walCompressType != compressTypeNone)
296 ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilter(param.walCompressType, 1));
297
298 storagePutP(write, walBuffer);
299 }
300 }
301
302 // -----------------------------------------------------------------------------------------------------------------------------
303 if (pgVersion == PG_VERSION_95)
304 {
305 ASSERT(!param.backupStandby);
306 ASSERT(!param.errorAfterStart);
307
308 harnessPqScriptSet((HarnessPq [])
309 {
310 // Connect to primary
311 HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_95, pg1Path, false, NULL, NULL),
312
313 // Get start time
314 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000),
315
316 // Start backup
317 HRNPQ_MACRO_ADVISORY_LOCK(1, true),
318 HRNPQ_MACRO_IS_IN_BACKUP(1, false),
319 HRNPQ_MACRO_START_BACKUP_84_95(1, param.startFast, lsnStartStr, walSegmentStart),
320 HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"),
321 HRNPQ_MACRO_TABLESPACE_LIST_0(1),
322
323 // Get copy start time
324 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999),
325 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000),
326
327 // Stop backup
328 HRNPQ_MACRO_STOP_BACKUP_LE_95(1, lsnStopStr, walSegmentStop),
329
330 // Get stop time
331 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000),
332
333 HRNPQ_MACRO_DONE()
334 });
335 }
336 // -----------------------------------------------------------------------------------------------------------------------------
337 else if (pgVersion == PG_VERSION_96)
338 {
339 ASSERT(param.backupStandby);
340 ASSERT(!param.errorAfterStart);
341
342 harnessPqScriptSet((HarnessPq [])
343 {
344 // Connect to primary
345 HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_96, pg1Path, false, NULL, NULL),
346
347 // Connect to standby
348 HRNPQ_MACRO_OPEN_GE_96(2, "dbname='postgres' port=5433", PG_VERSION_96, pg2Path, true, NULL, NULL),
349
350 // Get start time
351 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000),
352
353 // Start backup
354 HRNPQ_MACRO_ADVISORY_LOCK(1, true),
355 HRNPQ_MACRO_START_BACKUP_96(1, true, lsnStartStr, walSegmentStart),
356 HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"),
357 HRNPQ_MACRO_TABLESPACE_LIST_0(1),
358
359 // Wait for standby to sync
360 HRNPQ_MACRO_REPLAY_WAIT_96(2, lsnStartStr),
361
362 // Get copy start time
363 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999),
364 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000),
365
366 // Stop backup
367 HRNPQ_MACRO_STOP_BACKUP_96(1, lsnStopStr, walSegmentStop, false),
368
369 // Get stop time
370 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000),
371
372 HRNPQ_MACRO_DONE()
373 });
374 }
375 // -----------------------------------------------------------------------------------------------------------------------------
376 else if (pgVersion == PG_VERSION_11)
377 {
378 ASSERT(!param.backupStandby);
379
380 if (param.errorAfterStart)
381 {
382 harnessPqScriptSet((HarnessPq [])
383 {
384 // Connect to primary
385 HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_11, pg1Path, false, NULL, NULL),
386
387 // Get start time
388 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000),
389
390 // Start backup
391 HRNPQ_MACRO_ADVISORY_LOCK(1, true),
392 HRNPQ_MACRO_START_BACKUP_GE_10(1, param.startFast, lsnStartStr, walSegmentStart),
393 HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"),
394 HRNPQ_MACRO_TABLESPACE_LIST_1(1, 32768, "tblspc32768"),
395
396 // Get copy start time
397 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999),
398 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000),
399
400 HRNPQ_MACRO_DONE()
401 });
402 }
403 else
404 {
405 harnessPqScriptSet((HarnessPq [])
406 {
407 // Connect to primary
408 HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_11, pg1Path, false, NULL, NULL),
409
410 // Get start time
411 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000),
412
413 // Start backup
414 HRNPQ_MACRO_ADVISORY_LOCK(1, true),
415 HRNPQ_MACRO_START_BACKUP_GE_10(1, param.startFast, lsnStartStr, walSegmentStart),
416 HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"),
417 HRNPQ_MACRO_TABLESPACE_LIST_1(1, 32768, "tblspc32768"),
418
419 // Get copy start time
420 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999),
421 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000),
422
423 // Stop backup
424 HRNPQ_MACRO_STOP_BACKUP_GE_10(1, lsnStopStr, walSegmentStop, false),
425
426 // Get stop time
427 HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000),
428
429 HRNPQ_MACRO_DONE()
430 });
431 }
432 }
433 else
434 THROW_FMT(AssertError, "unsupported test version %u", pgVersion); // {uncoverable - no invalid versions in tests}
435 };
436
437 /***********************************************************************************************************************************
438 Test Run
439 ***********************************************************************************************************************************/
440 void
testRun(void)441 testRun(void)
442 {
443 FUNCTION_HARNESS_VOID();
444
445 // Install local command handler shim
446 static const ProtocolServerHandler testLocalHandlerList[] = {PROTOCOL_SERVER_HANDLER_BACKUP_LIST};
447 hrnProtocolLocalShimInstall(testLocalHandlerList, PROTOCOL_SERVER_HANDLER_LIST_SIZE(testLocalHandlerList));
448
449 // The tests expect the timezone to be UTC
450 setenv("TZ", "UTC", true);
451
452 Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
453
454 const String *pgFile = STRDEF("testfile");
455 const String *missingFile = STRDEF("missing");
456 const String *backupLabel = STRDEF("20190718-155825F");
457 const String *backupPathFile = strNewFmt(STORAGE_REPO_BACKUP "/%s/%s", strZ(backupLabel), strZ(pgFile));
458 BackupFileResult result = {0};
459
460 // *****************************************************************************************************************************
461 if (testBegin("segmentNumber()"))
462 {
463 TEST_RESULT_UINT(segmentNumber(pgFile), 0, "No segment number");
464 TEST_RESULT_UINT(segmentNumber(strNewFmt("%s.123", strZ(pgFile))), 123, "Segment number");
465 }
466
467 // *****************************************************************************************************************************
468 if (testBegin("backupFile()"))
469 {
470 // Load Parameters
471 StringList *argList = strLstNew();
472 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
473 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
474 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
475 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
476 HRN_CFG_LOAD(cfgCmdBackup, argList);
477
478 // Create the pg path
479 HRN_STORAGE_PATH_CREATE(storagePgWrite(), NULL, .mode = 0700);
480
481 // -------------------------------------------------------------------------------------------------------------------------
482 TEST_TITLE("pg file missing - ignoreMissing=true");
483
484 TEST_ASSIGN(
485 result,
486 backupFile(
487 missingFile, true, 0, true, NULL, false, 0, missingFile, false, compressTypeNone, 1, backupLabel, false,
488 cipherTypeNone, NULL),
489 "pg file missing, ignoreMissing=true, no delta");
490 TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy/repo size 0");
491 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
492
493 // -------------------------------------------------------------------------------------------------------------------------
494 TEST_TITLE("pg file missing - ignoreMissing=false");
495
496 TEST_ERROR(
497 backupFile(
498 missingFile, false, 0, true, NULL, false, 0, missingFile, false, compressTypeNone, 1, backupLabel, false,
499 cipherTypeNone, NULL),
500 FileMissingError, "unable to open missing file '" TEST_PATH "/pg/missing' for read");
501
502 // Create a pg file to backup
503 HRN_STORAGE_PUT_Z(storagePgWrite(), strZ(pgFile), "atestfile");
504
505 // -------------------------------------------------------------------------------------------------------------------------
506 TEST_TITLE("copy file to repo success - no prior checksum, no compression, no pageChecksum, no delta, no hasReference");
507
508 // With the expected backupCopyResultCopy, unset the storageFeatureCompress bit for the storageRepo for code coverage
509 uint64_t feature = storageRepo()->pub.interface.feature;
510 ((Storage *)storageRepo())->pub.interface.feature = feature & ((1 << storageFeatureCompress) ^ 0xFFFFFFFFFFFFFFFF);
511
512 TEST_ASSIGN(
513 result,
514 backupFile(
515 pgFile, false, 9999999, true, NULL, false, 0, pgFile, false, compressTypeNone, 1, backupLabel, false,
516 cipherTypeNone, NULL),
517 "pg file exists and shrunk, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
518
519 ((Storage *)storageRepo())->pub.interface.feature = feature;
520
521 TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
522 TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
523 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
524 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
525 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
526 TEST_STORAGE_EXISTS(storageRepo(), strZ(backupPathFile));
527
528 // Remove repo file
529 HRN_STORAGE_REMOVE(storageRepoWrite(), strZ(backupPathFile));
530
531 // -------------------------------------------------------------------------------------------------------------------------
532 TEST_TITLE("test pagechecksum while db file grows");
533
534 // Increase the file size but most of the following tests will still treat the file as size 9. This tests the common case
535 // where a file grows while a backup is running.
536 HRN_STORAGE_PUT_Z(storagePgWrite(), strZ(pgFile), "atestfile###");
537
538 TEST_ASSIGN(
539 result,
540 backupFile(
541 pgFile, false, 9, true, NULL, true, 0xFFFFFFFFFFFFFFFF, pgFile, false, compressTypeNone, 1, backupLabel, false,
542 cipherTypeNone, NULL),
543 "file checksummed with pageChecksum enabled");
544 TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
545 TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
546 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
547 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
548 TEST_RESULT_PTR_NE(result.pageChecksumResult, NULL, "pageChecksumResult is set");
549 TEST_RESULT_BOOL(varBool(kvGet(result.pageChecksumResult, VARSTRDEF("valid"))), false, "pageChecksumResult valid=false");
550 TEST_STORAGE_EXISTS(storageRepoWrite(), strZ(backupPathFile), .remove = true, .comment = "check exists in repo, remove");
551
552 // -------------------------------------------------------------------------------------------------------------------------
553 TEST_TITLE("pgFileSize, ignoreMissing=false, backupLabel, pgFileChecksumPage, pgFileChecksumPageLsnLimit");
554
555 VariantList *paramList = varLstNew();
556 varLstAdd(paramList, varNewStr(pgFile)); // pgFile
557 varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing
558 varLstAdd(paramList, varNewUInt64(8)); // pgFileSize
559 varLstAdd(paramList, varNewBool(false)); // pgFileCopyExactSize
560 varLstAdd(paramList, NULL); // pgFileChecksum
561 varLstAdd(paramList, varNewBool(true)); // pgFileChecksumPage
562 varLstAdd(paramList, varNewUInt64(0xFFFFFFFFFFFFFFFF)); // pgFileChecksumPageLsnLimit
563 varLstAdd(paramList, varNewStr(pgFile)); // repoFile
564 varLstAdd(paramList, varNewBool(false)); // repoFileHasReference
565 varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress
566 varLstAdd(paramList, varNewInt(1)); // repoFileCompressLevel
567 varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
568 varLstAdd(paramList, varNewBool(false)); // delta
569 varLstAdd(paramList, varNewUInt64(cipherTypeNone)); // cipherType
570 varLstAdd(paramList, NULL); // cipherSubPass
571
572 TEST_ASSIGN(
573 result,
574 backupFile(
575 pgFile, false, 8, false, NULL, true, 0xFFFFFFFFFFFFFFFF, pgFile, false, compressTypeNone, 1, backupLabel, false,
576 cipherTypeNone, NULL),
577 "backup file");
578 TEST_RESULT_UINT(result.copySize, 12, "copy size");
579 TEST_RESULT_UINT(result.repoSize, 12, "repo size");
580 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
581 TEST_RESULT_STR_Z(result.copyChecksum, "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "checksum");
582 TEST_RESULT_STR_Z(jsonFromKv(result.pageChecksumResult), "{\"align\":false,\"valid\":false}", "page checksum");
583 TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###");
584
585 // -------------------------------------------------------------------------------------------------------------------------
586 TEST_TITLE("file exists in repo and db, checksum match - NOOP");
587
588 // File exists in repo and db, pg checksum match, delta set, ignoreMissing false, hasReference - NOOP
589 TEST_ASSIGN(
590 result,
591 backupFile(
592 pgFile, false, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true,
593 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
594 "file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference");
595 TEST_RESULT_UINT(result.copySize, 9, "copy size set");
596 TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo");
597 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultNoOp, "noop file");
598 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
599 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
600 TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "file not modified");
601
602 // -------------------------------------------------------------------------------------------------------------------------
603 TEST_TITLE("file exists in repo and db, checksum mismatch - COPY");
604
605 // File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY
606 TEST_ASSIGN(
607 result,
608 backupFile(
609 pgFile, false, 9, true, STRDEF("1234567890123456789012345678901234567890"), false, 0, pgFile, true,
610 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
611 "file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference");
612 TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
613 TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
614 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
615 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
616 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
617 TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile", .comment = "9 bytes copied");
618
619 // -------------------------------------------------------------------------------------------------------------------------
620 TEST_TITLE("file exists in repo and pg, copy only exact file even if size passed is greater - COPY");
621
622 // File exists in repo and pg, pg checksum same, pg size passed is different, delta set, ignoreMissing false, hasReference
623 TEST_ASSIGN(
624 result,
625 backupFile(
626 pgFile, false, 9999999, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true,
627 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
628 "db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
629 TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size");
630 TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size");
631 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
632 TEST_RESULT_STR_Z(result.copyChecksum, "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "copy checksum updated");
633 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
634 TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "confirm contents");
635
636 // -------------------------------------------------------------------------------------------------------------------------
637 TEST_TITLE("resumed file is missing in repo but present in resumed manifest, file same name in repo - RECOPY");
638
639 TEST_STORAGE_LIST(
640 storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F", "testfile\n", .comment = "resumed file is missing in repo");
641 TEST_ASSIGN(
642 result,
643 backupFile(
644 pgFile, false, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, STRDEF(BOGUS_STR), false,
645 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
646 "backup 9 bytes of pgfile to file to resume in repo");
647 TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
648 TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
649 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "check recopy result");
650 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
651 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
652 TEST_STORAGE_GET(
653 storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "existing file with same name as pgFile not modified");
654 TEST_STORAGE_GET(
655 storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F/" BOGUS_STR, "atestfile", .comment = "resumed file copied");
656
657 // -------------------------------------------------------------------------------------------------------------------------
658 TEST_TITLE("file exists in repo & db, checksum not same in repo - RECOPY");
659
660 HRN_STORAGE_PUT_Z(
661 storageRepoWrite(), strZ(backupPathFile), "adifferentfile",
662 .comment = "create different file (size and checksum) with same name in repo");
663
664 // Delta set, ignoreMissing false, no hasReference
665 TEST_ASSIGN(
666 result,
667 backupFile(
668 pgFile, false, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false,
669 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
670 "db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference");
671 TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
672 TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
673 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
674 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
675 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
676 TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile", .comment = "existing file recopied");
677
678 // -------------------------------------------------------------------------------------------------------------------------
679 TEST_TITLE("file exists in repo but missing from db, checksum same in repo - SKIP");
680
681 TEST_ASSIGN(
682 result,
683 backupFile(
684 missingFile, true, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false,
685 compressTypeNone, 1, backupLabel, true, cipherTypeNone, NULL),
686 "file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference");
687 TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=0 size");
688 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
689 TEST_RESULT_PTR(result.copyChecksum, NULL, "copy checksum NULL");
690 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
691 TEST_STORAGE_LIST(
692 storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F", BOGUS_STR "\n", .comment = "file removed from repo");
693
694 // -------------------------------------------------------------------------------------------------------------------------
695 TEST_TITLE("compression set, all other boolean parameters false - COPY");
696
697 TEST_ASSIGN(
698 result,
699 backupFile(
700 pgFile, false, 9, true, NULL, false, 0, pgFile, false, compressTypeGz, 3, backupLabel, false, cipherTypeNone, NULL),
701 "pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
702 TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
703 TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
704 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
705 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
706 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
707 TEST_STORAGE_EXISTS(
708 storageRepo(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(backupLabel), strZ(pgFile))),
709 .comment = "copy file to repo compress success");
710
711 // -------------------------------------------------------------------------------------------------------------------------
712 TEST_TITLE("pg and repo file exist & match, prior checksum, compression - COPY CHECKSUM");
713
714 TEST_ASSIGN(
715 result,
716 backupFile(
717 pgFile, false, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, compressTypeGz,
718 3, backupLabel, false, cipherTypeNone, NULL),
719 "pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
720 TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
721 TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
722 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultChecksum, "checksum file");
723 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "compressed repo file checksum matches");
724 TEST_STORAGE_EXISTS(
725 storageRepo(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(backupLabel), strZ(pgFile))),
726 .comment = "compressed file exists");
727
728 // -------------------------------------------------------------------------------------------------------------------------
729 TEST_TITLE("create a zero sized file - checksum will be set but in backupManifestUpdate it will not be copied");
730
731 // Create zero sized file in pg
732 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zerofile");
733
734 // No prior checksum, no compression, no pageChecksum, no delta, no hasReference
735 TEST_ASSIGN(
736 result,
737 backupFile(
738 STRDEF("zerofile"), false, 0, true, NULL, false, 0, STRDEF("zerofile"), false, compressTypeNone, 1, backupLabel,
739 false, cipherTypeNone, NULL),
740 "zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
741 TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=pgFile size 0");
742 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
743 TEST_RESULT_PTR_NE(result.copyChecksum, NULL, "checksum set");
744 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
745 TEST_STORAGE_LIST(
746 storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F",
747 BOGUS_STR "\n"
748 "testfile.gz\n"
749 "zerofile\n",
750 .comment = "copy zero file to repo success");
751 }
752
753 // *****************************************************************************************************************************
754 if (testBegin("backupFile() - encrypt"))
755 {
756 // Load Parameters
757 StringList *argList = strLstNew();
758 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
759 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
760 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
761 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
762 hrnCfgArgRawStrId(argList, cfgOptRepoCipherType, cipherTypeAes256Cbc);
763 hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS);
764 HRN_CFG_LOAD(cfgCmdBackup, argList);
765 hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass);
766
767 // Create the pg path and pg file to backup
768 HRN_STORAGE_PUT_Z(storagePgWrite(), strZ(pgFile), "atestfile");
769
770 // -------------------------------------------------------------------------------------------------------------------------
771 TEST_TITLE("copy file to encrypted repo");
772
773 // No prior checksum, no compression, no pageChecksum, no delta, no hasReference
774 TEST_ASSIGN(
775 result,
776 backupFile(
777 pgFile, false, 9, true, NULL, false, 0, pgFile, false, compressTypeNone, 1, backupLabel, false, cipherTypeAes256Cbc,
778 STRDEF(TEST_CIPHER_PASS)),
779 "pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
780 TEST_RESULT_UINT(result.copySize, 9, "copy size set");
781 TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
782 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
783 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
784 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
785 TEST_STORAGE_GET(
786 storageRepo(), strZ(backupPathFile), "atestfile", .cipherType = cipherTypeAes256Cbc,
787 .comment = "copy file to encrypted repo success");
788
789 // -------------------------------------------------------------------------------------------------------------------------
790 TEST_TITLE("delta, copy file (size mismatch) to encrypted repo");
791
792 // Delta but pgFile does not match size passed, prior checksum, no compression, no pageChecksum, delta, no hasReference
793 TEST_ASSIGN(
794 result,
795 backupFile(
796 pgFile, false, 8, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false,
797 compressTypeNone, 1, backupLabel, true, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS)),
798 "pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
799 TEST_RESULT_UINT(result.copySize, 8, "copy size set");
800 TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
801 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
802 TEST_RESULT_STR_Z(result.copyChecksum, "acc972a8319d4903b839c64ec217faa3e77b4fcb", "copy checksum for size passed");
803 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
804 TEST_STORAGE_GET(
805 storageRepo(), strZ(backupPathFile), "atestfil", .cipherType = cipherTypeAes256Cbc,
806 .comment = "delta, copy file (size missmatch) to encrypted repo success");
807
808 // -------------------------------------------------------------------------------------------------------------------------
809 TEST_TITLE("no delta, recopy (size mismatch) file to encrypted repo");
810
811 TEST_ASSIGN(
812 result,
813 backupFile(
814 pgFile, false, 9, true, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false,
815 compressTypeNone, 0, backupLabel, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS)),
816 "pg and repo file exists, checksum mismatch, no ignoreMissing, no pageChecksum, no delta, no hasReference");
817 TEST_RESULT_UINT(result.copySize, 9, "copy size set");
818 TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
819 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
820 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
821 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
822 TEST_STORAGE_GET(
823 storageRepoWrite(), strZ(backupPathFile), "atestfile", .cipherType = cipherTypeAes256Cbc,
824 .comment = "recopy file to encrypted repo success");
825
826 // -------------------------------------------------------------------------------------------------------------------------
827 TEST_TITLE("no delta, recopy (checksum mismatch), file to encrypted repo");
828
829 TEST_ASSIGN(
830 result,
831 backupFile(
832 pgFile, false, 9, true, STRDEF("1234567890123456789012345678901234567890"), false, 0, pgFile, false,
833 compressTypeNone, 0, backupLabel, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS)),
834 "backup file");
835
836 TEST_RESULT_UINT(result.copySize, 9, "copy size set");
837 TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
838 TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
839 TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for size passed");
840 TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
841 TEST_STORAGE_GET(
842 storageRepo(), strZ(backupPathFile), "atestfile",
843 .cipherType = cipherTypeAes256Cbc, .comment = "recopy file to encrypted repo, success");
844 }
845
846 // *****************************************************************************************************************************
847 if (testBegin("backupLabelCreate()"))
848 {
849 StringList *argList = strLstNew();
850 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
851 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
852 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
853 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
854 HRN_CFG_LOAD(cfgCmdBackup, argList);
855
856 time_t timestamp = 1575401652;
857 String *backupLabel = backupLabelFormat(backupTypeFull, NULL, timestamp);
858
859 // -------------------------------------------------------------------------------------------------------------------------
860 TEST_TITLE("assign label when no history");
861
862 HRN_STORAGE_PATH_CREATE(storageRepoWrite(), STORAGE_REPO_BACKUP "/backup.history/2019");
863
864 TEST_RESULT_STR(backupLabelCreate(backupTypeFull, NULL, timestamp), backupLabel, "create label");
865
866 // -------------------------------------------------------------------------------------------------------------------------
867 TEST_TITLE("assign label when history is older");
868
869 HRN_STORAGE_PUT_EMPTY(
870 storageRepoWrite(), strZ(
871 strNewFmt(STORAGE_REPO_BACKUP "/backup.history/2019/%s.manifest.gz",
872 strZ(backupLabelFormat(backupTypeFull, NULL, timestamp - 4)))));
873
874 TEST_RESULT_STR(backupLabelCreate(backupTypeFull, NULL, timestamp), backupLabel, "create label");
875
876 // -------------------------------------------------------------------------------------------------------------------------
877 TEST_TITLE("assign label when backup is older");
878
879 HRN_STORAGE_PUT_EMPTY(
880 storageRepoWrite(), strZ(
881 strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(backupLabelFormat(backupTypeFull, NULL, timestamp - 2)))));
882
883 TEST_RESULT_STR(backupLabelCreate(backupTypeFull, NULL, timestamp), backupLabel, "create label");
884
885 // -------------------------------------------------------------------------------------------------------------------------
886 TEST_TITLE("advance time when backup is same");
887
888 HRN_STORAGE_PUT_EMPTY(
889 storageRepoWrite(), strZ(
890 strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(backupLabelFormat(backupTypeFull, NULL, timestamp)))));
891
892 TEST_RESULT_STR_Z(backupLabelCreate(backupTypeFull, NULL, timestamp), "20191203-193413F", "create label");
893
894 // -------------------------------------------------------------------------------------------------------------------------
895 TEST_TITLE("error when new label is in the past even with advanced time");
896
897 HRN_STORAGE_PUT_EMPTY(
898 storageRepoWrite(), strZ(
899 strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(backupLabelFormat(backupTypeFull, NULL, timestamp + 1)))));
900
901 TEST_ERROR(
902 backupLabelCreate(backupTypeFull, NULL, timestamp), FormatError,
903 "new backup label '20191203-193413F' is not later than latest backup label '20191203-193413F'\n"
904 "HINT: has the timezone changed?\n"
905 "HINT: is there clock skew?");
906 }
907
908 // *****************************************************************************************************************************
909 if (testBegin("backupInit()"))
910 {
911 // Set log level to detail
912 harnessLogLevelSet(logLevelDetail);
913
914 // -------------------------------------------------------------------------------------------------------------------------
915 TEST_TITLE("error when backup from standby is not supported");
916
917 StringList *argList = strLstNew();
918 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
919 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
920 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
921 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
922 hrnCfgArgRawBool(argList, cfgOptBackupStandby, true);
923 HRN_CFG_LOAD(cfgCmdBackup, argList);
924
925 TEST_ERROR(
926 backupInit(infoBackupNew(PG_VERSION_91, 1000000000000000910, hrnPgCatalogVersion(PG_VERSION_91), NULL)), ConfigError,
927 "option 'backup-standby' not valid for PostgreSQL < 9.2");
928
929 // -------------------------------------------------------------------------------------------------------------------------
930 TEST_TITLE("warn and reset when backup from standby used in offline mode");
931
932 // Create pg_control
933 HRN_STORAGE_PUT(
934 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
935 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 1000000000000000920}));
936
937 argList = strLstNew();
938 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
939 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
940 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
941 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
942 hrnCfgArgRawBool(argList, cfgOptBackupStandby, true);
943 hrnCfgArgRawBool(argList, cfgOptOnline, false);
944 HRN_CFG_LOAD(cfgCmdBackup, argList);
945
946 TEST_RESULT_VOID(
947 backupInit(infoBackupNew(PG_VERSION_92, 1000000000000000920, hrnPgCatalogVersion(PG_VERSION_92), NULL)),
948 "backup init");
949 TEST_RESULT_BOOL(cfgOptionBool(cfgOptBackupStandby), false, "check backup-standby");
950
951 TEST_RESULT_LOG(
952 "P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary");
953
954 // -------------------------------------------------------------------------------------------------------------------------
955 TEST_TITLE("error when pg_control does not match stanza");
956
957 // Create pg_control
958 HRN_STORAGE_PUT(
959 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
960 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 1000000000000001000}));
961
962 argList = strLstNew();
963 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
964 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
965 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
966 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
967 hrnCfgArgRawBool(argList, cfgOptOnline, false);
968 HRN_CFG_LOAD(cfgCmdBackup, argList);
969
970 TEST_ERROR(
971 backupInit(infoBackupNew(PG_VERSION_11, 1000000000000001100, hrnPgCatalogVersion(PG_VERSION_11), NULL)),
972 BackupMismatchError,
973 "PostgreSQL version 10, system-id 1000000000000001000 do not match stanza version 11, system-id 1000000000000001100\n"
974 "HINT: is this the correct stanza?");
975 TEST_ERROR(
976 backupInit(infoBackupNew(PG_VERSION_10, 1000000000000001100, hrnPgCatalogVersion(PG_VERSION_10), NULL)),
977 BackupMismatchError,
978 "PostgreSQL version 10, system-id 1000000000000001000 do not match stanza version 10, system-id 1000000000000001100\n"
979 "HINT: is this the correct stanza?");
980
981 // -------------------------------------------------------------------------------------------------------------------------
982 TEST_TITLE("reset start-fast when PostgreSQL < 8.4");
983
984 // Create pg_control
985 HRN_STORAGE_PUT(
986 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
987 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_83, .systemId = 1000000000000000830}));
988
989 argList = strLstNew();
990 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
991 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
992 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
993 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
994 hrnCfgArgRawBool(argList, cfgOptOnline, false);
995 hrnCfgArgRawBool(argList, cfgOptStartFast, true);
996 HRN_CFG_LOAD(cfgCmdBackup, argList);
997
998 TEST_RESULT_VOID(
999 backupInit(infoBackupNew(PG_VERSION_83, 1000000000000000830, hrnPgCatalogVersion(PG_VERSION_83), NULL)),
1000 "backup init");
1001 TEST_RESULT_BOOL(cfgOptionBool(cfgOptStartFast), false, "check start-fast");
1002
1003 TEST_RESULT_LOG("P00 WARN: start-fast option is only available in PostgreSQL >= 8.4");
1004
1005 // -------------------------------------------------------------------------------------------------------------------------
1006 TEST_TITLE("reset stop-auto when PostgreSQL < 9.3");
1007
1008 // Create pg_control
1009 HRN_STORAGE_PUT(
1010 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1011 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_84, .systemId = 1000000000000000840}));
1012
1013 argList = strLstNew();
1014 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1015 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1016 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1017 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1018 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1019 hrnCfgArgRawBool(argList, cfgOptStopAuto, true);
1020 HRN_CFG_LOAD(cfgCmdBackup, argList);
1021
1022 TEST_RESULT_VOID(
1023 backupInit(infoBackupNew(PG_VERSION_84, 1000000000000000840, hrnPgCatalogVersion(PG_VERSION_84), NULL)),
1024 "backup init");
1025 TEST_RESULT_BOOL(cfgOptionBool(cfgOptStopAuto), false, "check stop-auto");
1026
1027 TEST_RESULT_LOG("P00 WARN: stop-auto option is only available in PostgreSQL >= 9.3");
1028
1029 // -------------------------------------------------------------------------------------------------------------------------
1030 TEST_TITLE("reset checksum-page when the cluster does not have checksums enabled");
1031
1032 // Create pg_control
1033 HRN_STORAGE_PUT(
1034 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1035 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93}));
1036
1037 argList = strLstNew();
1038 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1039 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1040 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1041 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1042 hrnCfgArgRawBool(argList, cfgOptChecksumPage, true);
1043 HRN_CFG_LOAD(cfgCmdBackup, argList);
1044
1045 harnessPqScriptSet((HarnessPq [])
1046 {
1047 // Connect to primary
1048 HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_93, TEST_PATH "/pg1", false, NULL, NULL),
1049
1050 HRNPQ_MACRO_DONE()
1051 });
1052
1053 TEST_RESULT_VOID(
1054 dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary),
1055 "backup init");
1056 TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page");
1057
1058 TEST_RESULT_LOG(
1059 "P00 WARN: checksum-page option set to true but checksums are not enabled on the cluster, resetting to false");
1060
1061 // -------------------------------------------------------------------------------------------------------------------------
1062 TEST_TITLE("ok if cluster checksums are enabled and checksum-page is any value");
1063
1064 // Create pg_control with page checksums
1065 HRN_STORAGE_PUT(
1066 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1067 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93, .pageChecksum = true}));
1068
1069 argList = strLstNew();
1070 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1071 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1072 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1073 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1074 hrnCfgArgRawBool(argList, cfgOptChecksumPage, false);
1075 HRN_CFG_LOAD(cfgCmdBackup, argList);
1076
1077 harnessPqScriptSet((HarnessPq [])
1078 {
1079 // Connect to primary
1080 HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_93, TEST_PATH "/pg1", false, NULL, NULL),
1081
1082 HRNPQ_MACRO_DONE()
1083 });
1084
1085 TEST_RESULT_VOID(
1086 dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary),
1087 "backup init");
1088 TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page");
1089
1090 // Create pg_control without page checksums
1091 HRN_STORAGE_PUT(
1092 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1093 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93}));
1094
1095 harnessPqScriptSet((HarnessPq [])
1096 {
1097 // Connect to primary
1098 HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_93, TEST_PATH "/pg1", false, NULL, NULL),
1099
1100 HRNPQ_MACRO_DONE()
1101 });
1102
1103 TEST_RESULT_VOID(
1104 dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary),
1105 "backup init");
1106 TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page");
1107 }
1108
1109 // *****************************************************************************************************************************
1110 if (testBegin("backupTime()"))
1111 {
1112 // -------------------------------------------------------------------------------------------------------------------------
1113 TEST_TITLE("sleep retries and stall error");
1114
1115 StringList *argList = strLstNew();
1116 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1117 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1118 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1119 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1120 HRN_CFG_LOAD(cfgCmdBackup, argList);
1121
1122 // Create pg_control
1123 HRN_STORAGE_PUT(
1124 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1125 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93}));
1126
1127 harnessPqScriptSet((HarnessPq [])
1128 {
1129 // Connect to primary
1130 HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_93, TEST_PATH "/pg1", false, NULL, NULL),
1131
1132 // Advance the time slowly to force retries
1133 HRNPQ_MACRO_TIME_QUERY(1, 1575392588998),
1134 HRNPQ_MACRO_TIME_QUERY(1, 1575392588999),
1135 HRNPQ_MACRO_TIME_QUERY(1, 1575392589001),
1136
1137 // Stall time to force an error
1138 HRNPQ_MACRO_TIME_QUERY(1, 1575392589998),
1139 HRNPQ_MACRO_TIME_QUERY(1, 1575392589997),
1140 HRNPQ_MACRO_TIME_QUERY(1, 1575392589998),
1141 HRNPQ_MACRO_TIME_QUERY(1, 1575392589999),
1142
1143 HRNPQ_MACRO_DONE()
1144 });
1145
1146 BackupData *backupData = backupInit(
1147 infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL));
1148
1149 TEST_RESULT_INT(backupTime(backupData, true), 1575392588, "multiple tries for sleep");
1150 TEST_ERROR(backupTime(backupData, true), KernelError, "PostgreSQL clock has not advanced to the next second after 3 tries");
1151
1152 dbFree(backupData->dbPrimary);
1153 }
1154
1155 // *****************************************************************************************************************************
1156 if (testBegin("backupResumeFind()"))
1157 {
1158 StringList *argList = strLstNew();
1159 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1160 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1161 hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
1162 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1163 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeFull);
1164 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1165 HRN_CFG_LOAD(cfgCmdBackup, argList);
1166
1167 // -------------------------------------------------------------------------------------------------------------------------
1168 TEST_TITLE("cannot resume when manifest and copy are missing");
1169
1170 HRN_STORAGE_PATH_CREATE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191003-105320F");
1171
1172 TEST_RESULT_PTR(backupResumeFind((Manifest *)1, NULL), NULL, "find resumable backup");
1173
1174 TEST_RESULT_LOG(
1175 "P00 WARN: backup '20191003-105320F' cannot be resumed: partially deleted by prior resume or invalid");
1176
1177 // -------------------------------------------------------------------------------------------------------------------------
1178 TEST_TITLE("cannot resume when resume is disabled");
1179
1180 HRN_STORAGE_PATH_CREATE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191003-105320F");
1181
1182 cfgOptionSet(cfgOptResume, cfgSourceParam, BOOL_FALSE_VAR);
1183
1184 HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT);
1185
1186 TEST_RESULT_PTR(backupResumeFind((Manifest *)1, NULL), NULL, "find resumable backup");
1187
1188 TEST_RESULT_LOG("P00 WARN: backup '20191003-105320F' cannot be resumed: resume is disabled");
1189
1190 TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed");
1191
1192 cfgOptionSet(cfgOptResume, cfgSourceParam, BOOL_TRUE_VAR);
1193
1194 // -------------------------------------------------------------------------------------------------------------------------
1195 TEST_TITLE("cannot resume when pgBackRest version has changed");
1196
1197 Manifest *manifestResume = manifestNewInternal();
1198 manifestResume->pub.info = infoNew(NULL);
1199 manifestResume->pub.data.backupType = backupTypeFull;
1200 manifestResume->pub.data.backupLabel = STRDEF("20191003-105320F");
1201 manifestResume->pub.data.pgVersion = PG_VERSION_12;
1202
1203 manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
1204 manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR});
1205 manifestFileAdd(manifestResume, &(ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
1206
1207 manifestSave(
1208 manifestResume,
1209 storageWriteIo(
1210 storageNewWriteP(
1211 storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT))));
1212
1213 Manifest *manifest = manifestNewInternal();
1214 manifest->pub.data.backupType = backupTypeFull;
1215 manifest->pub.data.backrestVersion = STRDEF("BOGUS");
1216
1217 TEST_RESULT_PTR(backupResumeFind(manifest, NULL), NULL, "find resumable backup");
1218
1219 TEST_RESULT_LOG(
1220 "P00 WARN: backup '20191003-105320F' cannot be resumed:"
1221 " new pgBackRest version 'BOGUS' does not match resumable pgBackRest version '" PROJECT_VERSION "'");
1222
1223 TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed");
1224
1225 manifest->pub.data.backrestVersion = STRDEF(PROJECT_VERSION);
1226
1227 // -------------------------------------------------------------------------------------------------------------------------
1228 TEST_TITLE("cannot resume when backup labels do not match (resumable is null)");
1229
1230 manifest->pub.data.backupType = backupTypeFull;
1231 manifest->pub.data.backupLabelPrior = STRDEF("20191003-105320F");
1232
1233 manifestSave(
1234 manifestResume,
1235 storageWriteIo(
1236 storageNewWriteP(
1237 storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT))));
1238
1239 TEST_RESULT_PTR(backupResumeFind(manifest, NULL), NULL, "find resumable backup");
1240
1241 TEST_RESULT_LOG(
1242 "P00 WARN: backup '20191003-105320F' cannot be resumed:"
1243 " new prior backup label '<undef>' does not match resumable prior backup label '20191003-105320F'");
1244
1245 TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed");
1246
1247 manifest->pub.data.backupLabelPrior = NULL;
1248
1249 // -------------------------------------------------------------------------------------------------------------------------
1250 TEST_TITLE("cannot resume when backup labels do not match (new is null)");
1251
1252 manifest->pub.data.backupType = backupTypeFull;
1253 manifestResume->pub.data.backupLabelPrior = STRDEF("20191003-105320F");
1254
1255 manifestSave(
1256 manifestResume,
1257 storageWriteIo(
1258 storageNewWriteP(
1259 storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT))));
1260
1261 TEST_RESULT_PTR(backupResumeFind(manifest, NULL), NULL, "find resumable backup");
1262
1263 TEST_RESULT_LOG(
1264 "P00 WARN: backup '20191003-105320F' cannot be resumed:"
1265 " new prior backup label '20191003-105320F' does not match resumable prior backup label '<undef>'");
1266
1267 TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed");
1268
1269 manifestResume->pub.data.backupLabelPrior = NULL;
1270
1271 // -------------------------------------------------------------------------------------------------------------------------
1272 TEST_TITLE("cannot resume when compression does not match");
1273
1274 manifestResume->pub.data.backupOptionCompressType = compressTypeGz;
1275
1276 manifestSave(
1277 manifestResume,
1278 storageWriteIo(
1279 storageNewWriteP(
1280 storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT))));
1281
1282 TEST_RESULT_PTR(backupResumeFind(manifest, NULL), NULL, "find resumable backup");
1283
1284 TEST_RESULT_LOG(
1285 "P00 WARN: backup '20191003-105320F' cannot be resumed:"
1286 " new compression 'none' does not match resumable compression 'gz'");
1287
1288 TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed");
1289
1290 manifestResume->pub.data.backupOptionCompressType = compressTypeNone;
1291 }
1292
1293 // *****************************************************************************************************************************
1294 if (testBegin("backupJobResult()"))
1295 {
1296 // Set log level to detail
1297 harnessLogLevelSet(logLevelDetail);
1298
1299 // -------------------------------------------------------------------------------------------------------------------------
1300 TEST_TITLE("report job error");
1301
1302 ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
1303 protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message"));
1304
1305 TEST_ERROR(backupJobResult((Manifest *)1, NULL, STRDEF("log"), strLstNew(), job, 0, 0), AssertError, "error message");
1306
1307 // -------------------------------------------------------------------------------------------------------------------------
1308 TEST_TITLE("report host/100% progress on noop result");
1309
1310 // Create job that skips file
1311 job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
1312
1313 PackWrite *const resultPack = protocolPackNew();
1314 pckWriteU32P(resultPack, backupCopyResultNoOp);
1315 pckWriteU64P(resultPack, 0);
1316 pckWriteU64P(resultPack, 0);
1317 pckWriteStrP(resultPack, NULL);
1318 pckWriteStrP(resultPack, NULL);
1319 pckWriteEndP(resultPack);
1320
1321 protocolParallelJobResultSet(job, pckReadNewBuf(pckWriteBuf(resultPack)));
1322
1323 // Create manifest with file
1324 Manifest *manifest = manifestNewInternal();
1325 manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")});
1326
1327 TEST_RESULT_UINT(
1328 backupJobResult(manifest, STRDEF("host"), STRDEF("log-test"), strLstNew(), job, 0, 0), 0, "log noop result");
1329
1330 TEST_RESULT_LOG("P00 DETAIL: match file from prior backup host:log-test (0B, 100%)");
1331 }
1332
1333 // Offline tests should only be used to test offline functionality and errors easily tested in offline mode
1334 // *****************************************************************************************************************************
1335 if (testBegin("cmdBackup() offline"))
1336 {
1337 // Set log level to detail
1338 harnessLogLevelSet(logLevelDetail);
1339
1340 // Replace backup labels since the times are not deterministic
1341 hrnLogReplaceAdd("[0-9]{8}-[0-9]{6}F_[0-9]{8}-[0-9]{6}I", NULL, "INCR", true);
1342 hrnLogReplaceAdd("[0-9]{8}-[0-9]{6}F_[0-9]{8}-[0-9]{6}D", NULL, "DIFF", true);
1343 hrnLogReplaceAdd("[0-9]{8}-[0-9]{6}F", NULL, "FULL", true);
1344
1345 // Create stanza
1346 StringList *argList = strLstNew();
1347 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1348 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1349 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1350 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1351 HRN_CFG_LOAD(cfgCmdStanzaCreate, argList);
1352
1353 // Create pg_control
1354 HRN_STORAGE_PUT(
1355 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1356 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_84, .systemId = 1000000000000000840}));
1357
1358 cmdStanzaCreate();
1359 TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1");
1360
1361 // -------------------------------------------------------------------------------------------------------------------------
1362 TEST_TITLE("error when pg appears to be running");
1363
1364 argList = strLstNew();
1365 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1366 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1367 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1368 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1369 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1370 HRN_CFG_LOAD(cfgCmdBackup, argList);
1371
1372 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_POSTMASTERPID, "PID");
1373
1374 TEST_ERROR(
1375 cmdBackup(), PgRunningError,
1376 "--no-online passed but postmaster.pid exists - looks like " PG_NAME " is running. Shut down " PG_NAME " and try"
1377 " again, or use --force.");
1378
1379 TEST_RESULT_LOG("P00 WARN: no prior backup exists, incr backup has been changed to full");
1380
1381 // -------------------------------------------------------------------------------------------------------------------------
1382 TEST_TITLE("offline full backup");
1383
1384 argList = strLstNew();
1385 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1386 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1387 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1388 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1389 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1390 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1391 hrnCfgArgRawBool(argList, cfgOptForce, true);
1392 HRN_CFG_LOAD(cfgCmdBackup, argList);
1393
1394 HRN_STORAGE_PUT_Z(storagePgWrite(), "postgresql.conf", "CONFIGSTUFF");
1395
1396 TEST_RESULT_VOID(cmdBackup(), "backup");
1397
1398 TEST_RESULT_LOG_FMT(
1399 "P00 WARN: no prior backup exists, incr backup has been changed to full\n"
1400 "P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it"
1401 " looks like " PG_NAME " is running and the backup will probably not be consistent\n"
1402 "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, 99%%) checksum %s\n"
1403 "P01 DETAIL: backup file " TEST_PATH "/pg1/postgresql.conf (11B, 100%%) checksum"
1404 " e3db315c260e79211b7b52587123b7aa060f30ab\n"
1405 "P00 INFO: new backup label = [FULL-1]\n"
1406 "P00 INFO: full backup size = 8KB, file total = 2",
1407 TEST_64BIT() ?
1408 (TEST_BIG_ENDIAN() ? "749acedef8f8d5fe35fc20c0375657f876ccc38e" : "21e2ddc99cdf4cfca272eee4f38891146092e358") :
1409 "8bb70506d988a8698d9e8cf90736ada23634571b");
1410
1411 // Make pg no longer appear to be running
1412 HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMASTERPID, .errorOnMissing = true);
1413
1414 // -------------------------------------------------------------------------------------------------------------------------
1415 TEST_TITLE("error when no files have changed");
1416
1417 argList = strLstNew();
1418 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1419 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1420 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1421 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1422 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1423 hrnCfgArgRawBool(argList, cfgOptCompress, true);
1424 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
1425 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff);
1426 HRN_CFG_LOAD(cfgCmdBackup, argList);
1427
1428 TEST_ERROR(cmdBackup(), FileMissingError, "no files have changed since the last backup - this seems unlikely");
1429
1430 TEST_RESULT_LOG(
1431 "P00 INFO: last backup label = [FULL-1], version = " PROJECT_VERSION "\n"
1432 "P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [FULL-1]\n"
1433 "P00 WARN: diff backup cannot alter hardlink option to 'true', reset to value in [FULL-1]");
1434
1435 // -------------------------------------------------------------------------------------------------------------------------
1436 TEST_TITLE("offline incr backup to test unresumable backup");
1437
1438 argList = strLstNew();
1439 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1440 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1441 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1442 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1443 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1444 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1445 hrnCfgArgRawBool(argList, cfgOptChecksumPage, true);
1446 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeIncr);
1447 HRN_CFG_LOAD(cfgCmdBackup, argList);
1448
1449 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, "VER");
1450
1451 TEST_RESULT_VOID(cmdBackup(), "backup");
1452
1453 TEST_RESULT_LOG(
1454 "P00 INFO: last backup label = [FULL-1], version = " PROJECT_VERSION "\n"
1455 "P00 WARN: incr backup cannot alter 'checksum-page' option to 'true', reset to 'false' from [FULL-1]\n"
1456 "P00 WARN: backup '[DIFF-1]' cannot be resumed: new backup type 'incr' does not match resumable backup type 'diff'\n"
1457 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (3B, 100%) checksum c8663c2525f44b6d9c687fbceb4aafc63ed8b451\n"
1458 "P00 DETAIL: reference pg_data/global/pg_control to [FULL-1]\n"
1459 "P00 DETAIL: reference pg_data/postgresql.conf to [FULL-1]\n"
1460 "P00 INFO: new backup label = [INCR-1]\n"
1461 "P00 INFO: incr backup size = 3B, file total = 3");
1462
1463 // -------------------------------------------------------------------------------------------------------------------------
1464 TEST_TITLE("offline diff backup to test prior backup must be full");
1465
1466 argList = strLstNew();
1467 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1468 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
1469 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1470 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1471 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1472 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1473 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff);
1474 HRN_CFG_LOAD(cfgCmdBackup, argList);
1475
1476 sleepMSec(MSEC_PER_SEC - (timeMSec() % MSEC_PER_SEC));
1477 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, "VR2");
1478
1479 TEST_RESULT_VOID(cmdBackup(), "backup");
1480
1481 TEST_RESULT_LOG(
1482 "P00 INFO: last backup label = [FULL-1], version = " PROJECT_VERSION "\n"
1483 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (3B, 100%) checksum 6f1894088c578e4f0b9888e8e8a997d93cbbc0c5\n"
1484 "P00 DETAIL: reference pg_data/global/pg_control to [FULL-1]\n"
1485 "P00 DETAIL: reference pg_data/postgresql.conf to [FULL-1]\n"
1486 "P00 INFO: new backup label = [DIFF-2]\n"
1487 "P00 INFO: diff backup size = 3B, file total = 3");
1488
1489 // -------------------------------------------------------------------------------------------------------------------------
1490 TEST_TITLE("only repo2 configured");
1491
1492 // Create stanza on a second repo
1493 argList = strLstNew();
1494 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1495 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2");
1496 hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 2, cipherTypeAes256Cbc);
1497 hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, TEST_CIPHER_PASS);
1498 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1");
1499 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1500 HRN_CFG_LOAD(cfgCmdStanzaCreate, argList);
1501
1502 cmdStanzaCreate();
1503 TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo2");
1504
1505 // Set log level to warn
1506 harnessLogLevelSet(logLevelWarn);
1507
1508 // With repo2 the only repo configured, ensure it is chosen by confirming diff is changed to full due to no prior backups
1509 hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "1");
1510 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff);
1511 HRN_CFG_LOAD(cfgCmdBackup, argList);
1512
1513 TEST_RESULT_VOID(cmdBackup(), "backup");
1514 TEST_RESULT_LOG("P00 WARN: no prior backup exists, diff backup has been changed to full");
1515
1516 // -------------------------------------------------------------------------------------------------------------------------
1517 TEST_TITLE("multi-repo");
1518
1519 // Set log level to detail
1520 harnessLogLevelSet(logLevelDetail);
1521
1522 // Add repo1 to the configuration
1523 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, TEST_PATH "/repo");
1524 hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 1, "1");
1525 HRN_CFG_LOAD(cfgCmdBackup, argList);
1526
1527 TEST_RESULT_VOID(cmdBackup(), "backup");
1528 TEST_RESULT_LOG(
1529 "P00 INFO: repo option not specified, defaulting to repo1\n"
1530 "P00 INFO: last backup label = [FULL-1], version = " PROJECT_VERSION "\n"
1531 "P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [FULL-1]\n"
1532 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (3B, 100%) checksum 6f1894088c578e4f0b9888e8e8a997d93cbbc0c5\n"
1533 "P00 DETAIL: reference pg_data/global/pg_control to [FULL-1]\n"
1534 "P00 DETAIL: reference pg_data/postgresql.conf to [FULL-1]\n"
1535 "P00 INFO: new backup label = [DIFF-3]\n"
1536 "P00 INFO: diff backup size = 3B, file total = 3");
1537
1538 // -------------------------------------------------------------------------------------------------------------------------
1539 TEST_TITLE("multi-repo - specify repo");
1540
1541 hrnCfgArgRawZ(argList, cfgOptRepo, "2");
1542
1543 HRN_CFG_LOAD(cfgCmdBackup, argList);
1544
1545 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, "VER");
1546
1547 unsigned int backupCount = strLstSize(storageListP(storageRepoIdx(1), strNewFmt(STORAGE_PATH_BACKUP "/test1")));
1548
1549 TEST_RESULT_VOID(cmdBackup(), "backup");
1550 TEST_RESULT_LOG(
1551 "P00 INFO: last backup label = [FULL-2], version = " PROJECT_VERSION "\n"
1552 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (3B, 100%) checksum c8663c2525f44b6d9c687fbceb4aafc63ed8b451\n"
1553 "P00 DETAIL: reference pg_data/global/pg_control to [FULL-2]\n"
1554 "P00 DETAIL: reference pg_data/postgresql.conf to [FULL-2]\n"
1555 "P00 INFO: new backup label = [DIFF-4]\n"
1556 "P00 INFO: diff backup size = 3B, file total = 3");
1557 TEST_RESULT_UINT(
1558 strLstSize(storageListP(storageRepoIdx(1), strNewFmt(STORAGE_PATH_BACKUP "/test1"))), backupCount + 1,
1559 "new backup repo2");
1560
1561 // Cleanup
1562 hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
1563 harnessLogLevelReset();
1564 }
1565
1566 // *****************************************************************************************************************************
1567 if (testBegin("cmdBackup() online"))
1568 {
1569 const String *pg1Path = STRDEF(TEST_PATH "/pg1");
1570 const String *repoPath = STRDEF(TEST_PATH "/repo");
1571 const String *pg2Path = STRDEF(TEST_PATH "/pg2");
1572
1573 // Set log level to detail
1574 harnessLogLevelSet(logLevelDetail);
1575
1576 // Replace percent complete and backup size since they can cause a lot of churn when files are added/removed
1577 hrnLogReplaceAdd(", [0-9]{1,3}%\\)", "[0-9]+%", "PCT", false);
1578 hrnLogReplaceAdd(" backup size = [0-9]+[A-Z]+", "[^ ]+$", "SIZE", false);
1579
1580 // Replace checksums since they can differ between architectures (e.g. 32/64 bit)
1581 hrnLogReplaceAdd("\\) checksum [a-f0-9]{40}", "[a-f0-9]{40}$", "SHA1", false);
1582
1583 // Backup start time epoch. The idea is to not have backup times (and therefore labels) ever change. Each backup added
1584 // should be separated by 100,000 seconds (1,000,000 after stanza-upgrade) but after the initial assignments this will only
1585 // be possible at the beginning and the end, so new backups added in the middle will average the start times of the prior
1586 // and next backup to get their start time. Backups added to the beginning of the test will need to subtract from the
1587 // epoch.
1588 #define BACKUP_EPOCH 1570000000
1589
1590 // -------------------------------------------------------------------------------------------------------------------------
1591 TEST_TITLE("online 9.5 resume uncompressed full backup");
1592
1593 time_t backupTimeStart = BACKUP_EPOCH;
1594
1595 {
1596 // Create stanza
1597 StringList *argList = strLstNew();
1598 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1599 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
1600 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
1601 hrnCfgArgRawBool(argList, cfgOptOnline, false);
1602 HRN_CFG_LOAD(cfgCmdStanzaCreate, argList);
1603
1604 // Create pg_control
1605 HRN_STORAGE_PUT(
1606 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
1607 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_95, .systemId = 1000000000000000950}),
1608 .timeModified = backupTimeStart);
1609
1610 cmdStanzaCreate();
1611 TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1");
1612
1613 // Load options
1614 argList = strLstNew();
1615 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1616 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
1617 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
1618 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1619 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeFull);
1620 hrnCfgArgRawBool(argList, cfgOptStopAuto, true);
1621 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1622 hrnCfgArgRawBool(argList, cfgOptArchiveCheck, false);
1623 HRN_CFG_LOAD(cfgCmdBackup, argList);
1624
1625 // Add files
1626 HRN_STORAGE_PUT_Z(storagePgWrite(), "postgresql.conf", "CONFIGSTUFF", .timeModified = backupTimeStart);
1627 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_95_STR, .timeModified = backupTimeStart);
1628 HRN_STORAGE_PATH_CREATE(storagePgWrite(), strZ(pgWalPath(PG_VERSION_95)), .noParentCreate = true);
1629
1630 // Create a backup manifest that looks like a halted backup manifest
1631 Manifest *manifestResume = manifestNewBuild(
1632 storagePg(), PG_VERSION_95, hrnPgCatalogVersion(PG_VERSION_95), true, false, NULL, NULL);
1633 ManifestData *manifestResumeData = (ManifestData *)manifestData(manifestResume);
1634
1635 manifestResumeData->backupType = backupTypeFull;
1636 const String *resumeLabel = backupLabelCreate(backupTypeFull, NULL, backupTimeStart);
1637 manifestBackupLabelSet(manifestResume, resumeLabel);
1638
1639 // Copy a file to be resumed that has not changed in the repo
1640 HRN_STORAGE_COPY(
1641 storagePg(), PG_FILE_PGVERSION, storageRepoWrite(),
1642 strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel))));
1643
1644 strcpy(
1645 ((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/PG_VERSION")))->checksumSha1,
1646 "06d06bb31b570b94d7b4325f511f853dbe771c21");
1647
1648 // Save the resume manifest
1649 manifestSave(
1650 manifestResume,
1651 storageWriteIo(
1652 storageNewWriteP(
1653 storageRepoWrite(),
1654 strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(resumeLabel)))));
1655
1656 // Run backup
1657 testBackupPqScriptP(PG_VERSION_95, backupTimeStart);
1658 TEST_RESULT_VOID(cmdBackup(), "backup");
1659
1660 TEST_RESULT_LOG(
1661 "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
1662 "P00 INFO: backup start archive = 0000000105D944C000000000, lsn = 5d944c0/0\n"
1663 "P00 WARN: resumable backup 20191002-070640F of same type exists -- remove invalid files and resume\n"
1664 "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n"
1665 "P01 DETAIL: backup file " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n"
1666 "P01 DETAIL: checksum resumed file " TEST_PATH "/pg1/PG_VERSION (3B, [PCT]) checksum [SHA1]\n"
1667 "P00 INFO: execute exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
1668 "P00 INFO: backup stop archive = 0000000105D944C000000000, lsn = 5d944c0/800000\n"
1669 "P00 INFO: new backup label = 20191002-070640F\n"
1670 "P00 INFO: full backup size = [SIZE], file total = 3");
1671
1672 TEST_RESULT_STR_Z(
1673 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
1674 ". {link, d=20191002-070640F}\n"
1675 "pg_data {path}\n"
1676 "pg_data/PG_VERSION {file, s=3}\n"
1677 "pg_data/global {path}\n"
1678 "pg_data/global/pg_control {file, s=8192}\n"
1679 "pg_data/pg_xlog {path}\n"
1680 "pg_data/postgresql.conf {file, s=11}\n"
1681 "--------\n"
1682 "[backup:target]\n"
1683 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
1684 "\n"
1685 "[target:file]\n"
1686 "pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"size\":3"
1687 ",\"timestamp\":1570000000}\n"
1688 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570000000}\n"
1689 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
1690 ",\"timestamp\":1570000000}\n"
1691 "\n"
1692 "[target:path]\n"
1693 "pg_data={}\n"
1694 "pg_data/global={}\n"
1695 "pg_data/pg_xlog={}\n",
1696 "compare file list");
1697 }
1698
1699 // -------------------------------------------------------------------------------------------------------------------------
1700 TEST_TITLE("online resumed compressed 9.5 full backup");
1701
1702 // Backup start time
1703 backupTimeStart = BACKUP_EPOCH + 100000;
1704
1705 {
1706 // Load options
1707 StringList *argList = strLstNew();
1708 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1709 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
1710 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
1711 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1712 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeFull);
1713 hrnCfgArgRawBool(argList, cfgOptStopAuto, true);
1714 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
1715 hrnCfgArgRawBool(argList, cfgOptArchiveCopy, true);
1716 HRN_CFG_LOAD(cfgCmdBackup, argList);
1717
1718 // Create a backup manifest that looks like a halted backup manifest
1719 Manifest *manifestResume = manifestNewBuild(
1720 storagePg(), PG_VERSION_95, hrnPgCatalogVersion(PG_VERSION_95), true, false, NULL, NULL);
1721 ManifestData *manifestResumeData = (ManifestData *)manifestData(manifestResume);
1722
1723 manifestResumeData->backupType = backupTypeFull;
1724 manifestResumeData->backupOptionCompressType = compressTypeGz;
1725 const String *resumeLabel = backupLabelCreate(backupTypeFull, NULL, backupTimeStart);
1726 manifestBackupLabelSet(manifestResume, resumeLabel);
1727
1728 // File exists in cluster and repo but not in the resume manifest
1729 HRN_STORAGE_PUT_Z(storagePgWrite(), "not-in-resume", "TEST", .timeModified = backupTimeStart);
1730 HRN_STORAGE_PUT_EMPTY(
1731 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/not-in-resume.gz", strZ(resumeLabel))));
1732
1733 // Remove checksum from file so it won't be resumed
1734 HRN_STORAGE_PUT_EMPTY(
1735 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel))));
1736
1737 ((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/global/pg_control")))->checksumSha1[0] = 0;
1738
1739 // Size does not match between cluster and resume manifest
1740 HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart);
1741 HRN_STORAGE_PUT_EMPTY(
1742 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel))));
1743 manifestFileAdd(
1744 manifestResume, &(ManifestFile){
1745 .name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
1746 .size = 33});
1747
1748 // Time does not match between cluster and resume manifest
1749 HRN_STORAGE_PUT_Z(storagePgWrite(), "time-mismatch", "TEST", .timeModified = backupTimeStart);
1750 HRN_STORAGE_PUT_EMPTY(
1751 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel))));
1752 manifestFileAdd(
1753 manifestResume, &(ManifestFile){
1754 .name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
1755 .timestamp = backupTimeStart - 1});
1756
1757 // Size is zero in cluster and resume manifest. ??? We'd like to remove this requirement after the migration.
1758 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero-size", .timeModified = backupTimeStart);
1759 HRN_STORAGE_PUT_Z(
1760 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))),
1761 "ZERO-SIZE");
1762 manifestFileAdd(
1763 manifestResume, &(ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
1764
1765 // Path is not in manifest
1766 HRN_STORAGE_PATH_CREATE(
1767 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/bogus_path", strZ(resumeLabel))));
1768
1769 // File is not in manifest
1770 HRN_STORAGE_PUT_EMPTY(
1771 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/bogus.gz", strZ(resumeLabel))));
1772
1773 // File has incorrect compression type
1774 HRN_STORAGE_PUT_EMPTY(
1775 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/bogus", strZ(resumeLabel))));
1776
1777 // Save the resume manifest
1778 manifestSave(
1779 manifestResume,
1780 storageWriteIo(
1781 storageNewWriteP(
1782 storageRepoWrite(),
1783 strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(resumeLabel)))));
1784
1785 // Disable storageFeaturePath so paths will not be created before files are copied
1786 ((Storage *)storageRepoWrite())->pub.interface.feature ^= 1 << storageFeaturePath;
1787
1788 // Disable storageFeaturePathSync so paths will not be synced
1789 ((Storage *)storageRepoWrite())->pub.interface.feature ^= 1 << storageFeaturePathSync;
1790
1791 // Run backup
1792 testBackupPqScriptP(PG_VERSION_95, backupTimeStart);
1793 TEST_RESULT_VOID(cmdBackup(), "backup");
1794
1795 // Enable storage features
1796 ((Storage *)storageRepoWrite())->pub.interface.feature |= 1 << storageFeaturePath;
1797 ((Storage *)storageRepoWrite())->pub.interface.feature |= 1 << storageFeaturePathSync;
1798
1799 TEST_RESULT_LOG(
1800 "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
1801 "P00 INFO: backup start archive = 0000000105D95D3000000000, lsn = 5d95d30/0\n"
1802 "P00 WARN: resumable backup 20191003-105320F of same type exists -- remove invalid files and resume\n"
1803 "P00 DETAIL: remove path '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/bogus_path' from resumed"
1804 " backup\n"
1805 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/global/bogus' from resumed"
1806 " backup (mismatched compression type)\n"
1807 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/global/bogus.gz' from resumed"
1808 " backup (missing in manifest)\n"
1809 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/global/pg_control.gz' from"
1810 " resumed backup (no checksum in resumed manifest)\n"
1811 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/not-in-resume.gz' from resumed"
1812 " backup (missing in resumed manifest)\n"
1813 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/size-mismatch.gz' from resumed"
1814 " backup (mismatched size)\n"
1815 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/time-mismatch.gz' from resumed"
1816 " backup (mismatched timestamp)\n"
1817 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/zero-size.gz' from resumed"
1818 " backup (zero size)\n"
1819 "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n"
1820 "P01 DETAIL: backup file " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n"
1821 "P01 DETAIL: backup file " TEST_PATH "/pg1/time-mismatch (4B, [PCT]) checksum [SHA1]\n"
1822 "P01 DETAIL: backup file " TEST_PATH "/pg1/size-mismatch (4B, [PCT]) checksum [SHA1]\n"
1823 "P01 DETAIL: backup file " TEST_PATH "/pg1/not-in-resume (4B, [PCT]) checksum [SHA1]\n"
1824 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (3B, [PCT]) checksum [SHA1]\n"
1825 "P01 DETAIL: backup file " TEST_PATH "/pg1/zero-size (0B, [PCT])\n"
1826 "P00 INFO: execute exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
1827 "P00 INFO: backup stop archive = 0000000105D95D3000000000, lsn = 5d95d30/800000\n"
1828 "P00 INFO: check archive for segment(s) 0000000105D95D3000000000:0000000105D95D3000000000\n"
1829 "P00 DETAIL: copy segment 0000000105D95D3000000000 to backup\n"
1830 "P00 INFO: new backup label = 20191003-105320F\n"
1831 "P00 INFO: full backup size = [SIZE], file total = 8");
1832
1833 TEST_RESULT_STR_Z(
1834 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
1835 ". {link, d=20191003-105320F}\n"
1836 "pg_data {path}\n"
1837 "pg_data/PG_VERSION.gz {file, s=3}\n"
1838 "pg_data/global {path}\n"
1839 "pg_data/global/pg_control.gz {file, s=8192}\n"
1840 "pg_data/not-in-resume.gz {file, s=4}\n"
1841 "pg_data/pg_xlog {path}\n"
1842 "pg_data/pg_xlog/0000000105D95D3000000000.gz {file, s=16777216}\n"
1843 "pg_data/postgresql.conf.gz {file, s=11}\n"
1844 "pg_data/size-mismatch.gz {file, s=4}\n"
1845 "pg_data/time-mismatch.gz {file, s=4}\n"
1846 "pg_data/zero-size.gz {file, s=0}\n"
1847 "--------\n"
1848 "[backup:target]\n"
1849 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
1850 "\n"
1851 "[target:file]\n"
1852 "pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"size\":3"
1853 ",\"timestamp\":1570000000}\n"
1854 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570000000}\n"
1855 "pg_data/not-in-resume={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
1856 ",\"timestamp\":1570100000}\n"
1857 "pg_data/pg_xlog/0000000105D95D3000000000={\"size\":16777216,\"timestamp\":1570100002}\n"
1858 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
1859 ",\"timestamp\":1570000000}\n"
1860 "pg_data/size-mismatch={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
1861 ",\"timestamp\":1570100000}\n"
1862 "pg_data/time-mismatch={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
1863 ",\"timestamp\":1570100000}\n"
1864 "pg_data/zero-size={\"size\":0,\"timestamp\":1570100000}\n"
1865 "\n"
1866 "[target:path]\n"
1867 "pg_data={}\n"
1868 "pg_data/global={}\n"
1869 "pg_data/pg_xlog={}\n",
1870 "compare file list");
1871
1872 // Remove test files
1873 HRN_STORAGE_REMOVE(storagePgWrite(), "not-in-resume", .errorOnMissing = true);
1874 HRN_STORAGE_REMOVE(storagePgWrite(), "size-mismatch", .errorOnMissing = true);
1875 HRN_STORAGE_REMOVE(storagePgWrite(), "time-mismatch", .errorOnMissing = true);
1876 HRN_STORAGE_REMOVE(storagePgWrite(), "zero-size", .errorOnMissing = true);
1877 }
1878
1879 // -------------------------------------------------------------------------------------------------------------------------
1880 TEST_TITLE("online resumed compressed 9.5 diff backup");
1881
1882 backupTimeStart = BACKUP_EPOCH + 200000;
1883
1884 {
1885 StringList *argList = strLstNew();
1886 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
1887 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
1888 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
1889 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
1890 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff);
1891 hrnCfgArgRawBool(argList, cfgOptCompress, false);
1892 hrnCfgArgRawBool(argList, cfgOptStopAuto, true);
1893 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
1894 HRN_CFG_LOAD(cfgCmdBackup, argList);
1895
1896 // Load the previous manifest and null out the checksum-page option to be sure it gets set to false in this backup
1897 const String *manifestPriorFile = STRDEF(STORAGE_REPO_BACKUP "/latest/" BACKUP_MANIFEST_FILE);
1898 Manifest *manifestPrior = manifestNewLoad(storageReadIo(storageNewReadP(storageRepo(), manifestPriorFile)));
1899 ((ManifestData *)manifestData(manifestPrior))->backupOptionChecksumPage = NULL;
1900 manifestSave(manifestPrior, storageWriteIo(storageNewWriteP(storageRepoWrite(), manifestPriorFile)));
1901
1902 // Create a backup manifest that looks like a halted backup manifest
1903 Manifest *manifestResume = manifestNewBuild(
1904 storagePg(), PG_VERSION_95, hrnPgCatalogVersion(PG_VERSION_95), true, false, NULL, NULL);
1905 ManifestData *manifestResumeData = (ManifestData *)manifestData(manifestResume);
1906
1907 manifestResumeData->backupType = backupTypeDiff;
1908 manifestResumeData->backupLabelPrior = manifestData(manifestPrior)->backupLabel;
1909 manifestResumeData->backupOptionCompressType = compressTypeGz;
1910 const String *resumeLabel = backupLabelCreate(
1911 backupTypeDiff, manifestData(manifestPrior)->backupLabel, backupTimeStart);
1912 manifestBackupLabelSet(manifestResume, resumeLabel);
1913
1914 // Reference in manifest
1915 HRN_STORAGE_PUT_EMPTY(
1916 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION.gz", strZ(resumeLabel))));
1917
1918 // Reference in resumed manifest
1919 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "resume-ref", .timeModified = backupTimeStart);
1920 HRN_STORAGE_PUT_EMPTY(
1921 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel))));
1922 manifestFileAdd(
1923 manifestResume, &(ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
1924
1925 // Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note
1926 // also that the repo file is intenionally corrupt to generate a warning about corruption in the repository.
1927 HRN_STORAGE_PUT_Z(storagePgWrite(), "time-mismatch2", "TEST", .timeModified = backupTimeStart + 100);
1928 HRN_STORAGE_PUT_EMPTY(
1929 storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel))));
1930 manifestFileAdd(
1931 manifestResume, &(ManifestFile){
1932 .name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
1933 .timestamp = backupTimeStart});
1934
1935 // Links are always removed on resume
1936 THROW_ON_SYS_ERROR(
1937 symlink(
1938 "..",
1939 strZ(storagePathP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/link", strZ(resumeLabel))))) == -1,
1940 FileOpenError, "unable to create symlink");
1941
1942 // Special files should not be in the repo
1943 HRN_SYSTEM_FMT(
1944 "mkfifo -m 666 %s",
1945 strZ(storagePathP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/pipe", strZ(resumeLabel)))));
1946
1947 // Save the resume manifest
1948 manifestSave(
1949 manifestResume,
1950 storageWriteIo(
1951 storageNewWriteP(
1952 storageRepoWrite(),
1953 strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(resumeLabel)))));
1954
1955 // Run backup
1956 testBackupPqScriptP(PG_VERSION_95, backupTimeStart);
1957 TEST_RESULT_VOID(cmdBackup(), "backup");
1958
1959 // Check log
1960 TEST_RESULT_LOG(
1961 "P00 INFO: last backup label = 20191003-105320F, version = " PROJECT_VERSION "\n"
1962 "P00 WARN: diff backup cannot alter compress-type option to 'none', reset to value in 20191003-105320F\n"
1963 "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
1964 "P00 INFO: backup start archive = 0000000105D9759000000000, lsn = 5d97590/0\n"
1965 "P00 WARN: file 'time-mismatch2' has timestamp in the future, enabling delta checksum\n"
1966 "P00 WARN: resumable backup 20191003-105320F_20191004-144000D of same type exists"
1967 " -- remove invalid files and resume\n"
1968 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F_20191004-144000D/pg_data/PG_VERSION.gz'"
1969 " from resumed backup (reference in manifest)\n"
1970 "P00 WARN: remove special file '" TEST_PATH "/repo/backup/test1/20191003-105320F_20191004-144000D/pg_data/pipe'"
1971 " from resumed backup\n"
1972 "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F_20191004-144000D/pg_data/resume-ref.gz'"
1973 " from resumed backup (reference in resumed manifest)\n"
1974 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n"
1975 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n"
1976 "P00 WARN: resumed backup file pg_data/time-mismatch2 does not have expected checksum"
1977 " 984816fd329622876e14907634264e6f332e9fb3. The file will be recopied and backup will continue but this may be"
1978 " an issue unless the resumed backup path in the repository is known to be corrupted.\n"
1979 " NOTE: this does not indicate a problem with the PostgreSQL page checksums.\n"
1980 "P01 DETAIL: backup file " TEST_PATH "/pg1/time-mismatch2 (4B, [PCT]) checksum [SHA1]\n"
1981 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (3B, [PCT]) checksum [SHA1]\n"
1982 "P01 DETAIL: backup file " TEST_PATH "/pg1/resume-ref (0B, [PCT])\n"
1983 "P00 DETAIL: hardlink pg_data/PG_VERSION to 20191003-105320F\n"
1984 "P00 DETAIL: hardlink pg_data/global/pg_control to 20191003-105320F\n"
1985 "P00 DETAIL: hardlink pg_data/postgresql.conf to 20191003-105320F\n"
1986 "P00 INFO: execute exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
1987 "P00 INFO: backup stop archive = 0000000105D9759000000000, lsn = 5d97590/800000\n"
1988 "P00 INFO: check archive for segment(s) 0000000105D9759000000000:0000000105D9759000000000\n"
1989 "P00 INFO: new backup label = 20191003-105320F_20191004-144000D\n"
1990 "P00 INFO: diff backup size = [SIZE], file total = 5");
1991
1992 // Check repo directory
1993 TEST_RESULT_STR_Z(
1994 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
1995 ". {link, d=20191003-105320F_20191004-144000D}\n"
1996 "pg_data {path}\n"
1997 "pg_data/PG_VERSION.gz {file, s=3}\n"
1998 "pg_data/global {path}\n"
1999 "pg_data/global/pg_control.gz {file, s=8192}\n"
2000 "pg_data/pg_xlog {path}\n"
2001 "pg_data/postgresql.conf.gz {file, s=11}\n"
2002 "pg_data/resume-ref.gz {file, s=0}\n"
2003 "pg_data/time-mismatch2.gz {file, s=4}\n"
2004 "--------\n"
2005 "[backup:target]\n"
2006 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
2007 "\n"
2008 "[target:file]\n"
2009 "pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"reference\":\"20191003-105320F\""
2010 ",\"size\":3,\"timestamp\":1570000000}\n"
2011 "pg_data/global/pg_control={\"reference\":\"20191003-105320F\",\"size\":8192,\"timestamp\":1570000000}\n"
2012 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\""
2013 ",\"reference\":\"20191003-105320F\",\"size\":11,\"timestamp\":1570000000}\n"
2014 "pg_data/resume-ref={\"size\":0,\"timestamp\":1570200000}\n"
2015 "pg_data/time-mismatch2={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
2016 ",\"timestamp\":1570200100}\n"
2017 "\n"
2018 "[target:path]\n"
2019 "pg_data={}\n"
2020 "pg_data/global={}\n"
2021 "pg_data/pg_xlog={}\n",
2022 "compare file list");
2023
2024 // Remove test files
2025 HRN_STORAGE_REMOVE(storagePgWrite(), "resume-ref", .errorOnMissing = true);
2026 HRN_STORAGE_REMOVE(storagePgWrite(), "time-mismatch2", .errorOnMissing = true);
2027 }
2028
2029 // -------------------------------------------------------------------------------------------------------------------------
2030 TEST_TITLE("online 9.6 backup-standby full backup");
2031
2032 backupTimeStart = BACKUP_EPOCH + 1200000;
2033
2034 {
2035 // Update pg_control
2036 HRN_STORAGE_PUT(
2037 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
2038 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 1000000000000000960}),
2039 .timeModified = backupTimeStart);
2040
2041 // Update version
2042 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_96_STR, .timeModified = backupTimeStart);
2043
2044 // Upgrade stanza
2045 StringList *argList = strLstNew();
2046 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2047 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
2048 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
2049 hrnCfgArgRawBool(argList, cfgOptOnline, false);
2050 HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList);
2051
2052 cmdStanzaUpgrade();
2053 TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'test1' on repo1");
2054
2055 // Load options
2056 argList = strLstNew();
2057 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2058 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
2059 hrnCfgArgKeyRaw(argList, cfgOptPgPath, 1, pg1Path);
2060 hrnCfgArgKeyRaw(argList, cfgOptPgPath, 2, pg2Path);
2061 hrnCfgArgKeyRawZ(argList, cfgOptPgPort, 2, "5433");
2062 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
2063 hrnCfgArgRawBool(argList, cfgOptCompress, false);
2064 hrnCfgArgRawBool(argList, cfgOptBackupStandby, true);
2065 hrnCfgArgRawBool(argList, cfgOptStartFast, true);
2066 hrnCfgArgRawBool(argList, cfgOptArchiveCopy, true);
2067 HRN_CFG_LOAD(cfgCmdBackup, argList);
2068
2069 // Create file to copy from the standby. This file will be zero-length on the primary and non-zero-length on the standby
2070 // but no bytes will be copied.
2071 HRN_STORAGE_PUT_EMPTY(storagePgIdxWrite(0), PG_PATH_BASE "/1/1", .timeModified = backupTimeStart);
2072 HRN_STORAGE_PUT_Z(storagePgIdxWrite(1), PG_PATH_BASE "/1/1", "1234");
2073
2074 // Create file to copy from the standby. This file will be smaller on the primary than the standby and have no common
2075 // data in the bytes that exist on primary and standby. If the file is copied from the primary instead of the standby
2076 // the checksum will change but not the size.
2077 HRN_STORAGE_PUT_Z(storagePgIdxWrite(0), PG_PATH_BASE "/1/2", "DA", .timeModified = backupTimeStart);
2078 HRN_STORAGE_PUT_Z(storagePgIdxWrite(1), PG_PATH_BASE "/1/2", "5678");
2079
2080 // Create file to copy from the standby. This file will be larger on the primary than the standby and have no common
2081 // data in the bytes that exist on primary and standby. If the file is copied from the primary instead of the standby
2082 // the checksum and size will change.
2083 HRN_STORAGE_PUT_Z(storagePgIdxWrite(0), PG_PATH_BASE "/1/3", "TEST", .timeModified = backupTimeStart);
2084 HRN_STORAGE_PUT_Z(storagePgIdxWrite(1), PG_PATH_BASE "/1/3", "ABC");
2085
2086 // Create a file on the primary that does not exist on the standby to test that the file is removed from the manifest
2087 HRN_STORAGE_PUT_Z(storagePgIdxWrite(0), PG_PATH_BASE "/1/0", "DATA", .timeModified = backupTimeStart);
2088
2089 // Set log level to warn because the following test uses multiple processes so the log order will not be deterministic
2090 harnessLogLevelSet(logLevelWarn);
2091
2092 // Run backup but error on archive check
2093 testBackupPqScriptP(PG_VERSION_96, backupTimeStart, .noWal = true, .backupStandby = true);
2094 TEST_ERROR(
2095 cmdBackup(), ArchiveTimeoutError,
2096 "WAL segment 0000000105DA69C000000000 was not archived before the 100ms timeout\n"
2097 "HINT: check the archive_command to ensure that all options are correct (especially --stanza).\n"
2098 "HINT: check the PostgreSQL server log for errors.\n"
2099 "HINT: run the 'start' command if the stanza was previously stopped.");
2100
2101 // Remove halted backup so there's no resume
2102 HRN_STORAGE_PATH_REMOVE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191016-042640F", .recurse = true);
2103
2104 // Run backup
2105 testBackupPqScriptP(PG_VERSION_96, backupTimeStart, .backupStandby = true, .walCompressType = compressTypeGz);
2106 TEST_RESULT_VOID(cmdBackup(), "backup");
2107
2108 // Set log level back to detail
2109 harnessLogLevelSet(logLevelDetail);
2110
2111 TEST_RESULT_LOG(
2112 "P00 WARN: no prior backup exists, incr backup has been changed to full");
2113
2114 TEST_RESULT_STR_Z(
2115 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
2116 ". {link, d=20191016-042640F}\n"
2117 "pg_data {path}\n"
2118 "pg_data/PG_VERSION {file, s=3}\n"
2119 "pg_data/backup_label {file, s=17}\n"
2120 "pg_data/base {path}\n"
2121 "pg_data/base/1 {path}\n"
2122 "pg_data/base/1/1 {file, s=0}\n"
2123 "pg_data/base/1/2 {file, s=2}\n"
2124 "pg_data/base/1/3 {file, s=3}\n"
2125 "pg_data/global {path}\n"
2126 "pg_data/global/pg_control {file, s=8192}\n"
2127 "pg_data/pg_xlog {path}\n"
2128 "pg_data/pg_xlog/0000000105DA69C000000000 {file, s=16777216}\n"
2129 "pg_data/postgresql.conf {file, s=11}\n"
2130 "--------\n"
2131 "[backup:target]\n"
2132 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
2133 "\n"
2134 "[target:file]\n"
2135 "pg_data/PG_VERSION={\"checksum\":\"f5b7e6d36dc0113f61b36c700817d42b96f7b037\",\"size\":3"
2136 ",\"timestamp\":1571200000}\n"
2137 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
2138 ",\"timestamp\":1571200002}\n"
2139 "pg_data/base/1/1={\"master\":false,\"size\":0,\"timestamp\":1571200000}\n"
2140 "pg_data/base/1/2={\"checksum\":\"54ceb91256e8190e474aa752a6e0650a2df5ba37\",\"master\":false,\"size\":2"
2141 ",\"timestamp\":1571200000}\n"
2142 "pg_data/base/1/3={\"checksum\":\"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8\",\"master\":false,\"size\":3"
2143 ",\"timestamp\":1571200000}\n"
2144 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1571200000}\n"
2145 "pg_data/pg_xlog/0000000105DA69C000000000={\"size\":16777216,\"timestamp\":1571200002}\n"
2146 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
2147 ",\"timestamp\":1570000000}\n"
2148 "\n"
2149 "[target:path]\n"
2150 "pg_data={}\n"
2151 "pg_data/base={}\n"
2152 "pg_data/base/1={}\n"
2153 "pg_data/global={}\n"
2154 "pg_data/pg_xlog={}\n",
2155 "compare file list");
2156
2157 // Remove test files
2158 HRN_STORAGE_PATH_REMOVE(storagePgIdxWrite(1), NULL, .recurse = true);
2159 HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "base/1", .recurse = true);
2160 }
2161
2162 // -------------------------------------------------------------------------------------------------------------------------
2163 TEST_TITLE("online 11 full backup with tablespaces and page checksums");
2164
2165 backupTimeStart = BACKUP_EPOCH + 2200000;
2166
2167 {
2168 // Update pg_control
2169 HRN_STORAGE_PUT(
2170 storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
2171 hrnPgControlToBuffer(
2172 (PgControl){
2173 .version = PG_VERSION_11, .systemId = 1000000000000001100, .pageChecksum = true,
2174 .walSegmentSize = 1024 * 1024}),
2175 .timeModified = backupTimeStart);
2176
2177 // Update version
2178 HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_11_STR, .timeModified = backupTimeStart);
2179
2180 // Update wal path
2181 HRN_STORAGE_PATH_REMOVE(storagePgWrite(), strZ(pgWalPath(PG_VERSION_95)));
2182 HRN_STORAGE_PATH_CREATE(storagePgWrite(), strZ(pgWalPath(PG_VERSION_11)), .noParentCreate = true);
2183
2184 // Upgrade stanza
2185 StringList *argList = strLstNew();
2186 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2187 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
2188 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
2189 hrnCfgArgRawBool(argList, cfgOptOnline, false);
2190 HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList);
2191
2192 cmdStanzaUpgrade();
2193 TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'test1' on repo1");
2194
2195 // Load options
2196 argList = strLstNew();
2197 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2198 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
2199 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
2200 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
2201 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeFull);
2202 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
2203 hrnCfgArgRawZ(argList, cfgOptManifestSaveThreshold, "1");
2204 hrnCfgArgRawBool(argList, cfgOptArchiveCopy, true);
2205 HRN_CFG_LOAD(cfgCmdBackup, argList);
2206
2207 // Move pg1-path and put a link in its place. This tests that backup works when pg1-path is a symlink yet should be
2208 // completely invisible in the manifest and logging.
2209 HRN_SYSTEM_FMT("mv %s %s-data", strZ(pg1Path), strZ(pg1Path));
2210 HRN_SYSTEM_FMT("ln -s %s-data %s ", strZ(pg1Path), strZ(pg1Path));
2211
2212 // Zeroed file which passes page checksums
2213 Buffer *relation = bufNew(PG_PAGE_SIZE_DEFAULT);
2214 memset(bufPtr(relation), 0, bufSize(relation));
2215 bufUsedSet(relation, bufSize(relation));
2216
2217 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0};
2218
2219 HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/1", relation, .timeModified = backupTimeStart);
2220
2221 // Zeroed file which will fail on alignment
2222 relation = bufNew(PG_PAGE_SIZE_DEFAULT + 1);
2223 memset(bufPtr(relation), 0, bufSize(relation));
2224 bufUsedSet(relation, bufSize(relation));
2225
2226 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0};
2227
2228 HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/2", relation, .timeModified = backupTimeStart);
2229
2230 // File with bad page checksums
2231 relation = bufNew(PG_PAGE_SIZE_DEFAULT * 4);
2232 memset(bufPtr(relation), 0, bufSize(relation));
2233 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0xFF};
2234 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x01)) = (PageHeaderData){.pd_upper = 0x00};
2235 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x02)) = (PageHeaderData){.pd_upper = 0xFE};
2236 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x03)) = (PageHeaderData){.pd_upper = 0xEF};
2237 bufUsedSet(relation, bufSize(relation));
2238
2239 HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/3", relation, .timeModified = backupTimeStart);
2240 const char *rel1_3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, relation)));
2241
2242 // File with bad page checksum
2243 relation = bufNew(PG_PAGE_SIZE_DEFAULT * 3);
2244 memset(bufPtr(relation), 0, bufSize(relation));
2245 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0x00};
2246 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x01)) = (PageHeaderData){.pd_upper = 0x08};
2247 *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x02)) = (PageHeaderData){.pd_upper = 0x00};
2248 bufUsedSet(relation, bufSize(relation));
2249
2250 HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/4", relation, .timeModified = backupTimeStart);
2251 const char *rel1_4Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, relation)));
2252
2253 // Add a tablespace
2254 HRN_STORAGE_PATH_CREATE(storagePgWrite(), PG_PATH_PGTBLSPC);
2255 THROW_ON_SYS_ERROR(
2256 symlink("../../pg1-tblspc/32768", strZ(storagePathP(storagePg(), STRDEF(PG_PATH_PGTBLSPC "/32768")))) == -1,
2257 FileOpenError, "unable to create symlink");
2258
2259 HRN_STORAGE_PUT_EMPTY(
2260 storageTest,
2261 strZ(strNewFmt("pg1-tblspc/32768/%s/1/5", strZ(pgTablespaceId(PG_VERSION_11, hrnPgCatalogVersion(PG_VERSION_11))))),
2262 .timeModified = backupTimeStart);
2263
2264 // Disable storageFeatureSymLink so tablespace (and latest) symlinks will not be created
2265 ((Storage *)storageRepoWrite())->pub.interface.feature ^= 1 << storageFeatureSymLink;
2266
2267 // Disable storageFeatureHardLink so hardlinks will not be created
2268 ((Storage *)storageRepoWrite())->pub.interface.feature ^= 1 << storageFeatureHardLink;
2269
2270 // Run backup
2271 testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 3);
2272 TEST_RESULT_VOID(cmdBackup(), "backup");
2273
2274 // Reset storage features
2275 ((Storage *)storageRepoWrite())->pub.interface.feature |= 1 << storageFeatureSymLink;
2276 ((Storage *)storageRepoWrite())->pub.interface.feature |= 1 << storageFeatureHardLink;
2277
2278 TEST_RESULT_LOG(
2279 "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
2280 "P00 INFO: backup start archive = 0000000105DB5DE000000000, lsn = 5db5de0/0\n"
2281 "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/3 (32KB, [PCT]) checksum [SHA1]\n"
2282 "P00 WARN: invalid page checksums found in file " TEST_PATH "/pg1/base/1/3 at pages 0, 2-3\n"
2283 "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/4 (24KB, [PCT]) checksum [SHA1]\n"
2284 "P00 WARN: invalid page checksum found in file " TEST_PATH "/pg1/base/1/4 at page 1\n"
2285 "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/2 (8KB, [PCT]) checksum [SHA1]\n"
2286 "P00 WARN: page misalignment in file " TEST_PATH "/pg1/base/1/2: file size 8193 is not divisible by page size"
2287 " 8192\n"
2288 "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n"
2289 "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/1 (8KB, [PCT]) checksum [SHA1]\n"
2290 "P01 DETAIL: backup file " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n"
2291 "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (2B, [PCT]) checksum [SHA1]\n"
2292 "P01 DETAIL: backup file " TEST_PATH "/pg1/pg_tblspc/32768/PG_11_201809051/1/5 (0B, [PCT])\n"
2293 "P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
2294 "P00 INFO: backup stop archive = 0000000105DB5DE000000002, lsn = 5db5de0/280000\n"
2295 "P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n"
2296 "P00 INFO: check archive for segment(s) 0000000105DB5DE000000000:0000000105DB5DE000000002\n"
2297 "P00 DETAIL: copy segment 0000000105DB5DE000000000 to backup\n"
2298 "P00 DETAIL: copy segment 0000000105DB5DE000000001 to backup\n"
2299 "P00 DETAIL: copy segment 0000000105DB5DE000000002 to backup\n"
2300 "P00 INFO: new backup label = 20191027-181320F\n"
2301 "P00 INFO: full backup size = [SIZE], file total = 12");
2302
2303 TEST_RESULT_STR(
2304 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191027-181320F")),
2305 strNewFmt(
2306 "pg_data {path}\n"
2307 "pg_data/PG_VERSION.gz {file, s=2}\n"
2308 "pg_data/backup_label.gz {file, s=17}\n"
2309 "pg_data/base {path}\n"
2310 "pg_data/base/1 {path}\n"
2311 "pg_data/base/1/1.gz {file, s=8192}\n"
2312 "pg_data/base/1/2.gz {file, s=8193}\n"
2313 "pg_data/base/1/3.gz {file, s=32768}\n"
2314 "pg_data/base/1/4.gz {file, s=24576}\n"
2315 "pg_data/global {path}\n"
2316 "pg_data/global/pg_control.gz {file, s=8192}\n"
2317 "pg_data/pg_tblspc {path}\n"
2318 "pg_data/pg_wal {path}\n"
2319 "pg_data/pg_wal/0000000105DB5DE000000000.gz {file, s=1048576}\n"
2320 "pg_data/pg_wal/0000000105DB5DE000000001.gz {file, s=1048576}\n"
2321 "pg_data/pg_wal/0000000105DB5DE000000002.gz {file, s=1048576}\n"
2322 "pg_data/postgresql.conf.gz {file, s=11}\n"
2323 "pg_tblspc {path}\n"
2324 "pg_tblspc/32768 {path}\n"
2325 "pg_tblspc/32768/PG_11_201809051 {path}\n"
2326 "pg_tblspc/32768/PG_11_201809051/1 {path}\n"
2327 "pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n"
2328 "--------\n"
2329 "[backup:target]\n"
2330 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
2331 "pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\""
2332 ",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n"
2333 "\n"
2334 "[target:file]\n"
2335 "pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"size\":2"
2336 ",\"timestamp\":1572200000}\n"
2337 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
2338 ",\"timestamp\":1572200002}\n"
2339 "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true"
2340 ",\"master\":false,\"size\":8192,\"timestamp\":1572200000}\n"
2341 "pg_data/base/1/2={\"checksum\":\"8beb58e08394fe665fb04a17b4003faa3802760b\",\"checksum-page\":false"
2342 ",\"master\":false,\"size\":8193,\"timestamp\":1572200000}\n"
2343 "pg_data/base/1/3={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[0,[2,3]]"
2344 ",\"master\":false,\"size\":32768,\"timestamp\":1572200000}\n"
2345 "pg_data/base/1/4={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[1],\"master\":false"
2346 ",\"size\":24576,\"timestamp\":1572200000}\n"
2347 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572200000}\n"
2348 "pg_data/pg_wal/0000000105DB5DE000000000={\"size\":1048576,\"timestamp\":1572200002}\n"
2349 "pg_data/pg_wal/0000000105DB5DE000000001={\"size\":1048576,\"timestamp\":1572200002}\n"
2350 "pg_data/pg_wal/0000000105DB5DE000000002={\"size\":1048576,\"timestamp\":1572200002}\n"
2351 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
2352 ",\"timestamp\":1570000000}\n"
2353 "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"size\":0"
2354 ",\"timestamp\":1572200000}\n"
2355 "\n"
2356 "[target:link]\n"
2357 "pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n"
2358 "\n"
2359 "[target:path]\n"
2360 "pg_data={}\n"
2361 "pg_data/base={}\n"
2362 "pg_data/base/1={}\n"
2363 "pg_data/global={}\n"
2364 "pg_data/pg_tblspc={}\n"
2365 "pg_data/pg_wal={}\n"
2366 "pg_tblspc={}\n"
2367 "pg_tblspc/32768={}\n"
2368 "pg_tblspc/32768/PG_11_201809051={}\n"
2369 "pg_tblspc/32768/PG_11_201809051/1={}\n",
2370 rel1_3Sha1, rel1_4Sha1),
2371 "compare file list");
2372
2373 // Remove test files
2374 HRN_STORAGE_REMOVE(storagePgWrite(), "base/1/2", .errorOnMissing = true);
2375 HRN_STORAGE_REMOVE(storagePgWrite(), "base/1/3", .errorOnMissing = true);
2376 HRN_STORAGE_REMOVE(storagePgWrite(), "base/1/4", .errorOnMissing = true);
2377 }
2378
2379 // -------------------------------------------------------------------------------------------------------------------------
2380 TEST_TITLE("error when pg_control not present");
2381
2382 backupTimeStart = BACKUP_EPOCH + 2300000;
2383
2384 {
2385 // Load options
2386 StringList *argList = strLstNew();
2387 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2388 hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
2389 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
2390 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
2391 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeIncr);
2392 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
2393 HRN_CFG_LOAD(cfgCmdBackup, argList);
2394
2395 // Run backup
2396 testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .errorAfterStart = true);
2397 TEST_ERROR(
2398 cmdBackup(), FileMissingError,
2399 "pg_control must be present in all online backups\n"
2400 "HINT: is something wrong with the clock or filesystem timestamps?");
2401
2402 // Check log
2403 TEST_RESULT_LOG(
2404 "P00 INFO: last backup label = 20191027-181320F, version = " PROJECT_VERSION "\n"
2405 "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
2406 "P00 INFO: backup start archive = 0000000105DB764000000000, lsn = 5db7640/0");
2407
2408 // Remove partial backup so it won't be resumed (since it errored before any checksums were written)
2409 HRN_STORAGE_PATH_REMOVE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191027-181320F_20191028-220000I", .recurse = true);
2410 }
2411
2412 // -------------------------------------------------------------------------------------------------------------------------
2413 TEST_TITLE("online 11 incr backup with tablespaces");
2414
2415 backupTimeStart = BACKUP_EPOCH + 2400000;
2416
2417 {
2418 // Load options
2419 StringList *argList = strLstNew();
2420 hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
2421 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo-bogus");
2422 hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath);
2423 hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "1");
2424 hrnCfgArgKeyRawBool(argList, cfgOptRepoHardlink, 2, true);
2425 hrnCfgArgRawZ(argList, cfgOptRepo, "2");
2426 hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
2427 hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
2428 hrnCfgArgRawStrId(argList, cfgOptType, backupTypeIncr);
2429 hrnCfgArgRawBool(argList, cfgOptDelta, true);
2430 hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true);
2431 HRN_CFG_LOAD(cfgCmdBackup, argList);
2432
2433 // Update pg_control timestamp
2434 HRN_STORAGE_TIME(storagePg(), "global/pg_control", backupTimeStart);
2435
2436 // Run backup. Make sure that the timeline selected converts to hexdecimal that can't be interpreted as decimal.
2437 testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .timeline = 0x2C);
2438 TEST_RESULT_VOID(cmdBackup(), "backup");
2439
2440 TEST_RESULT_LOG(
2441 "P00 INFO: last backup label = 20191027-181320F, version = " PROJECT_VERSION "\n"
2442 "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
2443 "P00 INFO: backup start archive = 0000002C05DB8EB000000000, lsn = 5db8eb0/0\n"
2444 "P00 WARN: a timeline switch has occurred since the 20191027-181320F backup, enabling delta checksum\n"
2445 " HINT: this is normal after restoring from backup or promoting a standby.\n"
2446 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n"
2447 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/base/1/1 (8KB, [PCT]) checksum [SHA1]\n"
2448 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n"
2449 "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (2B, [PCT]) checksum [SHA1]\n"
2450 "P00 DETAIL: hardlink pg_data/PG_VERSION to 20191027-181320F\n"
2451 "P00 DETAIL: hardlink pg_data/base/1/1 to 20191027-181320F\n"
2452 "P00 DETAIL: hardlink pg_data/global/pg_control to 20191027-181320F\n"
2453 "P00 DETAIL: hardlink pg_data/postgresql.conf to 20191027-181320F\n"
2454 "P00 DETAIL: hardlink pg_tblspc/32768/PG_11_201809051/1/5 to 20191027-181320F\n"
2455 "P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
2456 "P00 INFO: backup stop archive = 0000002C05DB8EB000000000, lsn = 5db8eb0/80000\n"
2457 "P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n"
2458 "P00 INFO: check archive for segment(s) 0000002C05DB8EB000000000:0000002C05DB8EB000000000\n"
2459 "P00 INFO: new backup label = 20191027-181320F_20191030-014640I\n"
2460 "P00 INFO: incr backup size = [SIZE], file total = 6");
2461
2462 TEST_RESULT_STR_Z(
2463 testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
2464 ". {link, d=20191027-181320F_20191030-014640I}\n"
2465 "pg_data {path}\n"
2466 "pg_data/PG_VERSION.gz {file, s=2}\n"
2467 "pg_data/backup_label.gz {file, s=17}\n"
2468 "pg_data/base {path}\n"
2469 "pg_data/base/1 {path}\n"
2470 "pg_data/base/1/1.gz {file, s=8192}\n"
2471 "pg_data/global {path}\n"
2472 "pg_data/global/pg_control.gz {file, s=8192}\n"
2473 "pg_data/pg_tblspc {path}\n"
2474 "pg_data/pg_tblspc/32768 {link, d=../../pg_tblspc/32768}\n"
2475 "pg_data/pg_wal {path}\n"
2476 "pg_data/postgresql.conf.gz {file, s=11}\n"
2477 "pg_tblspc {path}\n"
2478 "pg_tblspc/32768 {path}\n"
2479 "pg_tblspc/32768/PG_11_201809051 {path}\n"
2480 "pg_tblspc/32768/PG_11_201809051/1 {path}\n"
2481 "pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n"
2482 "--------\n"
2483 "[backup:target]\n"
2484 "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
2485 "pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\""
2486 ",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n"
2487 "\n"
2488 "[target:file]\n"
2489 "pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"reference\":\"20191027-181320F\""
2490 ",\"size\":2,\"timestamp\":1572200000}\n"
2491 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
2492 ",\"timestamp\":1572400002}\n"
2493 "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true"
2494 ",\"master\":false,\"reference\":\"20191027-181320F\",\"size\":8192,\"timestamp\":1572200000}\n"
2495 "pg_data/global/pg_control={\"reference\":\"20191027-181320F\",\"size\":8192,\"timestamp\":1572400000}\n"
2496 "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\""
2497 ",\"reference\":\"20191027-181320F\",\"size\":11,\"timestamp\":1570000000}\n"
2498 "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"reference\":\"20191027-181320F\""
2499 ",\"size\":0,\"timestamp\":1572200000}\n"
2500 "\n"
2501 "[target:link]\n"
2502 "pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n"
2503 "\n"
2504 "[target:path]\n"
2505 "pg_data={}\n"
2506 "pg_data/base={}\n"
2507 "pg_data/base/1={}\n"
2508 "pg_data/global={}\n"
2509 "pg_data/pg_tblspc={}\n"
2510 "pg_data/pg_wal={}\n"
2511 "pg_tblspc={}\n"
2512 "pg_tblspc/32768={}\n"
2513 "pg_tblspc/32768/PG_11_201809051={}\n"
2514 "pg_tblspc/32768/PG_11_201809051/1={}\n",
2515 "compare file list");
2516 }
2517 }
2518
2519 FUNCTION_HARNESS_RETURN_VOID();
2520 }
2521