1 /***********************************************************************************************************************************
2 Test Archive Push Command
3 ***********************************************************************************************************************************/
4 #include "common/io/fdRead.h"
5 #include "common/io/fdWrite.h"
6 #include "common/time.h"
7 #include "postgres/version.h"
8 #include "storage/posix/storage.h"
9
10 #include "common/harnessConfig.h"
11 #include "common/harnessFork.h"
12 #include "common/harnessInfo.h"
13 #include "common/harnessPostgres.h"
14 #include "common/harnessProtocol.h"
15
16 /***********************************************************************************************************************************
17 Test Run
18 ***********************************************************************************************************************************/
19 void
testRun(void)20 testRun(void)
21 {
22 FUNCTION_HARNESS_VOID();
23
24 // Create default storage object for testing
25 Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
26
27 // *****************************************************************************************************************************
28 if (testBegin("archivePushReadyList(), archivePushProcessList(), and archivePushDrop()"))
29 {
30 StringList *argList = strLstNew();
31 hrnCfgArgRawZ(argList, cfgOptStanza, "db");
32 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/db");
33 hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
34 hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
35 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
36
37 HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_wal/archive_status");
38 HRN_STORAGE_PATH_CREATE(storageTest, "spool/archive/db/out");
39
40 // Create ok files to indicate WAL that has already been archived
41 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok");
42 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000003.ok");
43 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000004.ok");
44 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000005.error");
45 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000006.error");
46 HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error");
47
48 // Create ready files for wal that still needs to be archived
49 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000002.ready");
50 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000003.ready");
51 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000005.ready");
52 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000006.ready");
53
54 // -------------------------------------------------------------------------------------------------------------------------
55 TEST_TITLE("ready list");
56
57 TEST_RESULT_STRLST_Z(
58 archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal")),
59 "000000010000000100000002\n000000010000000100000005\n000000010000000100000006\n", "ready list");
60
61 TEST_STORAGE_LIST(
62 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "000000010000000100000003.ok\n", .comment = "remaining status list");
63
64 // -------------------------------------------------------------------------------------------------------------------------
65 TEST_TITLE("WAL drop");
66
67 StringList *argListDrop = strLstDup(argList);
68 hrnCfgArgRawFmt(argListDrop, cfgOptArchivePushQueueMax, "%zu", (size_t)1024 * 1024 * 1024);
69 HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync);
70
71 // Write the files that we claim are in pg_wal
72 Buffer *walBuffer = bufNew((size_t)16 * 1024 * 1024);
73 bufUsedSet(walBuffer, bufSize(walBuffer));
74 memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
75 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer);
76
77 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000002", walBuffer);
78 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000003", walBuffer);
79 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000005", walBuffer);
80 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000006", walBuffer);
81
82 // Queue max is high enough that no WAL will be dropped
83 TEST_RESULT_BOOL(
84 archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), false, "wal is not dropped");
85
86 // Now set queue max low enough that WAL will be dropped
87 argListDrop = strLstDup(argList);
88 hrnCfgArgRawFmt(argListDrop, cfgOptArchivePushQueueMax, "%zu", (size_t)16 * 1024 * 1024 * 2);
89 HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync);
90
91 TEST_RESULT_BOOL(
92 archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), true, "wal is dropped");
93 }
94
95 // *****************************************************************************************************************************
96 if (testBegin("archivePushCheck()"))
97 {
98 StringList *argList = strLstNew();
99 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
100 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
101 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
102 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
103
104 // -------------------------------------------------------------------------------------------------------------------------
105 TEST_TITLE("mismatched pg_control and archive.info - pg version");
106
107 HRN_STORAGE_PUT(
108 storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
109 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 0xFACEFACEFACEFACE}));
110
111 // Create incorrect archive info
112 HRN_INFO_PUT(
113 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
114 "[db]\n"
115 "db-id=1\n"
116 "\n"
117 "[db:history]\n"
118 "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n");
119
120 TEST_ERROR(
121 archivePushCheck(true), RepoInvalidError,
122 "unable to find a valid repository:\n"
123 "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version"
124 " 9.4, system-id 5555555555555555555"
125 "\nHINT: are you archiving to the correct stanza?");
126
127 // -------------------------------------------------------------------------------------------------------------------------
128 TEST_TITLE("mismatched pg_control and archive.info - system-id");
129
130 // Fix the version
131 HRN_INFO_PUT(
132 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
133 "[db]\n"
134 "db-id=1\n"
135 "\n"
136 "[db:history]\n"
137 "1={\"db-id\":5555555555555555555,\"db-version\":\"9.6\"}\n");
138
139 TEST_ERROR(
140 archivePushCheck(true), RepoInvalidError,
141 "unable to find a valid repository:\n"
142 "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version"
143 " 9.6, system-id 5555555555555555555"
144 "\nHINT: are you archiving to the correct stanza?");
145
146 // -------------------------------------------------------------------------------------------------------------------------
147 TEST_TITLE("pg_control and archive.info match");
148
149 // Fix archive info
150 HRN_INFO_PUT(
151 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
152 "[db]\n"
153 "db-id=1\n"
154 "\n"
155 "[db:history]\n"
156 "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
157
158 ArchivePushCheckResult result = {0};
159 TEST_ASSIGN(result, archivePushCheck(true), "get archive check result");
160
161 TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
162 TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
163
164 ArchivePushFileRepoData *repoData = lstGet(result.repoList, 0);
165 TEST_RESULT_UINT(repoData->repoIdx, 0, "check repo idx");
166 TEST_RESULT_STR_Z(repoData->archiveId, "9.6-1", "check archive id");
167 TEST_RESULT_UINT(repoData->cipherType, cipherTypeNone, "check cipher type");
168 TEST_RESULT_STR_Z(repoData->cipherPass, NULL, "check cipher pass (not set in this test)");
169
170 // -------------------------------------------------------------------------------------------------------------------------
171 TEST_TITLE("mismatched repos when pg-path not present");
172
173 argList = strLstNew();
174 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
175 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2");
176 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 4, TEST_PATH "/repo4");
177 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
178
179 // repo2 has correct info
180 HRN_INFO_PUT(
181 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
182 "[db]\n"
183 "db-id=1\n"
184 "\n"
185 "[db:history]\n"
186 "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
187
188 // repo4 has incorrect info
189 HRN_INFO_PUT(
190 storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
191 "[db]\n"
192 "db-id=1\n"
193 "\n"
194 "[db:history]\n"
195 "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n");
196
197 TEST_ASSIGN(result, archivePushCheck(false), "get archive check result");
198
199 TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
200 TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
201 TEST_RESULT_STRLST_Z(
202 result.errorList,
203 "repo4: [ArchiveMismatchError] repo2 stanza version 9.6, system-id 18072658121562454734 do not match repo4 stanza"
204 " version 9.4, system-id 5555555555555555555\n"
205 "HINT: are you archiving to the correct stanza?\n",
206 "check error list");
207
208 repoData = lstGet(result.repoList, 0);
209 TEST_RESULT_UINT(repoData->repoIdx, 0, "check repo idx");
210 TEST_RESULT_STR_Z(repoData->archiveId, "9.6-1", "check archive id");
211 TEST_RESULT_UINT(repoData->cipherType, cipherTypeNone, "check cipher type");
212 TEST_RESULT_STR_Z(repoData->cipherPass, NULL, "check cipher pass (not set in this test)");
213
214 // -------------------------------------------------------------------------------------------------------------------------
215 TEST_TITLE("matched repos when pg-path not present");
216
217 // repo4 has correct info
218 HRN_INFO_PUT(
219 storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
220 "[db]\n"
221 "db-id=2\n"
222 "\n"
223 "[db:history]\n"
224 "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"
225 "2={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
226
227 TEST_ASSIGN(result, archivePushCheck(false), "get archive check result");
228
229 TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
230 TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
231
232 repoData = lstGet(result.repoList, 0);
233 TEST_RESULT_UINT(repoData->repoIdx, 0, "check repo idx");
234 TEST_RESULT_STR_Z(repoData->archiveId, "9.6-1", "check repo2 archive id");
235 TEST_RESULT_UINT(repoData->cipherType, cipherTypeNone, "check repo2 cipher pass");
236 TEST_RESULT_STR_Z(repoData->cipherPass, NULL, "check repo2 cipher pass (not set in this test)");
237
238 repoData = lstGet(result.repoList, 1);
239 TEST_RESULT_UINT(repoData->repoIdx, 1, "check repo idx");
240 TEST_RESULT_STR_Z(repoData->archiveId, "9.6-2", "check repo4 archive id");
241 TEST_RESULT_UINT(repoData->cipherType, cipherTypeNone, "check repo4 cipher type");
242 TEST_RESULT_STR_Z(repoData->cipherPass, NULL, "check repo4 cipher pass (not set in this test)");
243 }
244
245 // *****************************************************************************************************************************
246 if (testBegin("Synchronous cmdArchivePush() and archivePushFile()"))
247 {
248 // -------------------------------------------------------------------------------------------------------------------------
249 TEST_TITLE("command must be run on the pg host");
250
251 StringList *argList = strLstNew();
252 hrnCfgArgRawZ(argList, cfgOptPgHost, "host");
253 hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
254 hrnCfgArgRawZ(argList, cfgOptStanza, "test2");
255 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleMain);
256
257 TEST_ERROR(cmdArchivePush(), HostInvalidError, "archive-push command must be run on the PostgreSQL host");
258
259 // -------------------------------------------------------------------------------------------------------------------------
260 TEST_TITLE("WAL segment not specified");
261
262 argList = strLstNew();
263 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
264 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
265
266 TEST_ERROR(cmdArchivePush(), ParamRequiredError, "WAL segment to push required");
267
268 // -------------------------------------------------------------------------------------------------------------------------
269 TEST_TITLE("pg-path not specified");
270
271 StringList *argListTemp = strLstDup(argList);
272 strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
273 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
274
275 TEST_ERROR(
276 cmdArchivePush(), OptionRequiredError,
277 "option 'pg1-path' must be specified when relative wal paths are used"
278 "\nHINT: is %f passed to archive-push instead of %p?"
279 "\nHINT: PostgreSQL may pass relative paths even with %p depending on the environment.");
280
281 // -------------------------------------------------------------------------------------------------------------------------
282 TEST_TITLE("attempt to push WAL with incorrect headers");
283
284 // Create pg_control and archive.info
285 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
286 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
287
288 argListTemp = strLstDup(argList);
289 strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
290 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
291
292 HRN_STORAGE_PUT(
293 storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
294 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}));
295
296 HRN_INFO_PUT(
297 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
298 "[db]\n"
299 "db-id=1\n"
300 "\n"
301 "[db:history]\n"
302 "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n");
303
304 // Generate WAL with incorrect headers and try to push them
305 Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
306 bufUsedSet(walBuffer1, bufSize(walBuffer1));
307 memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
308 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
309
310 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
311
312 THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
313
314 TEST_ERROR(
315 cmdArchivePush(), ArchiveMismatchError,
316 "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 10, system-id 18072658121562454734 do not match"
317 " stanza version 11, system-id 18072658121562454734");
318
319 memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
320 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1);
321 const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
322
323 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
324
325 TEST_ERROR(
326 cmdArchivePush(), ArchiveMismatchError,
327 "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 11, system-id 17055110554209741999 do not match"
328 " stanza version 11, system-id 18072658121562454734");
329
330 // -------------------------------------------------------------------------------------------------------------------------
331 TEST_TITLE("push by ignoring the invalid header");
332
333 argListTemp = strLstDup(argList);
334 hrnCfgArgRawBool(argListTemp, cfgOptArchiveHeaderCheck, false);
335 strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
336 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
337
338 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
339 TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
340
341 TEST_STORAGE_EXISTS(
342 storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
343 .remove = true, .comment = "check repo for WAL file, then remove");
344
345 // -------------------------------------------------------------------------------------------------------------------------
346 TEST_TITLE("generate valid WAL and push them");
347
348 argListTemp = strLstDup(argList);
349 strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
350 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
351
352 memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
353 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
354
355 // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them
356 // directly from the buffers to reduce the cost of maintaining checksums.
357 walBuffer1Sha1 = TEST_64BIT() ?
358 (TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") :
359 "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a";
360
361 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
362
363 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
364 TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
365
366 TEST_STORAGE_EXISTS(
367 storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
368 .comment = "check repo for WAL file");
369
370 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
371 TEST_RESULT_LOG(
372 "P00 WARN: WAL file '000000010000000100000001' already exists in the repo1 archive with the same checksum\n"
373 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
374 "P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
375
376 // Now create a new WAL buffer with a different checksum to test checksum errors
377 Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024);
378 bufUsedSet(walBuffer2, bufSize(walBuffer2));
379 memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2));
380 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2);
381 const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2)));
382
383 HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer2);
384
385 TEST_ERROR(
386 cmdArchivePush(), ArchiveDuplicateError,
387 "WAL file '000000010000000100000001' already exists in the repo1 archive with a different checksum");
388
389 // -------------------------------------------------------------------------------------------------------------------------
390 TEST_TITLE("WAL with absolute path and no pg1-path");
391
392 argListTemp = strLstNew();
393 hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test");
394 hrnCfgArgRawZ(argListTemp, cfgOptRepoPath, TEST_PATH "/repo");
395 strLstAddZ(argListTemp, TEST_PATH "/pg/pg_wal/000000010000000100000002");
396 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
397
398 HRN_STORAGE_PUT(storageTest, "pg/pg_wal/000000010000000100000002", walBuffer2, .comment = "write WAL");
399
400 // Create tmp file to make it look like a prior push failed partway through to ensure that retries work
401 HRN_STORAGE_PUT_Z(
402 storageTest,
403 strZ(
404 strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)),
405 "PARTIAL", .comment = "write WAL tmp file");
406
407 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
408 TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
409
410 TEST_STORAGE_EXISTS(
411 storageRepoIdxWrite(0),
412 strZ(
413 strNewFmt(
414 STORAGE_REPO_ARCHIVE "/11-1/0000000100000001/000000010000000100000002-%s.gz", walBuffer2Sha1)),
415 .comment = "check repo for WAL file");
416 TEST_RESULT_BOOL(
417 storageExistsP(
418 storageTest,
419 strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)),
420 false, "check WAL tmp file is gone");
421
422 // -------------------------------------------------------------------------------------------------------------------------
423 TEST_TITLE("push a history file");
424
425 argListTemp = strLstDup(argList);
426 strLstAddZ(argListTemp, "pg_wal/00000001.history");
427 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
428
429 HRN_STORAGE_PUT_Z(storagePgWrite(), "pg_wal/00000001.history", "FAKEHISTORY");
430
431 TEST_RESULT_VOID(cmdArchivePush(), "push a history file");
432 TEST_RESULT_LOG("P00 INFO: pushed WAL file '00000001.history' to the archive");
433
434 TEST_STORAGE_EXISTS(
435 storageRepoIdx(0), STORAGE_REPO_ARCHIVE "/11-1/00000001.history", .comment = "check repo for history file");
436
437 // -------------------------------------------------------------------------------------------------------------------------
438 TEST_TITLE("check drop functionality");
439
440 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000001.ready");
441 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000002.ready");
442
443 argListTemp = strLstDup(argList);
444 hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "16m");
445 strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
446 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
447
448 TEST_RESULT_VOID(cmdArchivePush(), "drop WAL file");
449 TEST_RESULT_LOG("P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
450
451 argListTemp = strLstDup(argList);
452 hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "1GB");
453 strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
454 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
455
456 TEST_RESULT_VOID(cmdArchivePush(), "push WAL file again");
457 TEST_RESULT_LOG(
458 "P00 WARN: WAL file '000000010000000100000002' already exists in the repo1 archive with the same checksum\n"
459 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
460 "P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
461
462 // -------------------------------------------------------------------------------------------------------------------------
463 TEST_TITLE("multiple repos, one encrypted");
464
465 // Remove old repo
466 HRN_STORAGE_PATH_REMOVE(storageTest, "repo", .errorOnMissing = true, .recurse = true);
467
468 argListTemp = strLstNew();
469 hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test");
470 hrnCfgArgKeyRawZ(argListTemp, cfgOptPgPath, 1, TEST_PATH "/pg");
471 hrnCfgArgKeyRawZ(argListTemp, cfgOptRepoPath, 2, TEST_PATH "/repo2");
472 hrnCfgArgKeyRawStrId(argListTemp, cfgOptRepoCipherType, 2, cipherTypeAes256Cbc);
473 hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, "badpassphrase");
474 hrnCfgArgKeyRawZ(argListTemp, cfgOptRepoPath, 3, TEST_PATH "/repo3");
475 hrnCfgArgRawNegate(argListTemp, cfgOptCompress);
476 strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
477 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
478 hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
479
480 // repo2 is encrypted
481 HRN_INFO_PUT(
482 storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
483 "[cipher]\n"
484 "cipher-pass=\"badsubpassphrase\"\n"
485 "\n"
486 "[db]\n"
487 "db-id=1\n"
488 "\n"
489 "[db:history]\n"
490 "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}",
491 .cipherType = cipherTypeAes256Cbc, .cipherPass = "badpassphrase");
492
493 // repo3 is not encrypted
494 HRN_INFO_PUT(
495 storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
496 "[db]\n"
497 "db-id=1\n"
498 "\n"
499 "[db:history]\n"
500 "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}");
501
502 // Push encrypted WAL segment
503 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
504 TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
505
506 TEST_STORAGE_EXISTS(
507 storageTest, strZ(strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
508 .remove = true, .comment = "check repo2 for WAL file then remove");
509
510 TEST_STORAGE_EXISTS(
511 storageTest, strZ(strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
512 .remove = true, .comment = "check repo3 for WAL file then remove");
513
514 // -------------------------------------------------------------------------------------------------------------------------
515 TEST_TITLE("write error on one repo but other repo succeeds");
516
517 HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001", .mode = 0500);
518
519 TEST_ERROR(
520 cmdArchivePush(), CommandError,
521 strZ(
522 strNewFmt(
523 "archive-push command encountered error(s):\n"
524 "repo2: [FileOpenError] unable to open file '" TEST_PATH "/repo2/archive/test/11-1/0000000100000001"
525 "/000000010000000100000002-%s' for write: [13] Permission denied", walBuffer2Sha1)));
526
527 TEST_STORAGE_LIST_EMPTY(storageTest, "repo2/archive/test/11-1/0000000100000001", .comment = "check repo2 for no WAL file");
528 TEST_STORAGE_LIST(
529 storageTest, "repo3/archive/test/11-1/0000000100000001",
530 strZ(strNewFmt("000000010000000100000002-%s\n", walBuffer2Sha1)), .comment = "check repo3 for WAL file");
531
532 HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001");
533
534 // -------------------------------------------------------------------------------------------------------------------------
535 TEST_TITLE("push WAL to one repo");
536
537 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
538 TEST_RESULT_LOG(
539 "P00 WARN: WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
540 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
541 "P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
542
543 TEST_STORAGE_LIST(
544 storageTest, "repo2/archive/test/11-1/0000000100000001",
545 strZ(strNewFmt("000000010000000100000002-%s\n", walBuffer2Sha1)), .comment = "check repo2 for WAL file");
546
547 // -------------------------------------------------------------------------------------------------------------------------
548 TEST_TITLE("WAL already exists in both repos");
549
550 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
551 TEST_RESULT_LOG(
552 "P00 WARN: WAL file '000000010000000100000002' already exists in the repo2 archive with the same checksum\n"
553 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
554 "P00 WARN: WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
555 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
556 "P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
557
558 // -------------------------------------------------------------------------------------------------------------------------
559 TEST_TITLE("push succeeds on one repo when other repo fails to load archive.info");
560
561 TEST_STORAGE_EXISTS(
562 storageTest, strZ(strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
563 .remove = true);
564 TEST_STORAGE_EXISTS(
565 storageTest, strZ(strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
566 .remove = true);
567 HRN_STORAGE_MODE(storageTest, "repo2", .mode = 0200);
568
569 TEST_ERROR(
570 cmdArchivePush(), CommandError,
571 "archive-push command encountered error(s):\n"
572 "repo2: [FileOpenError] unable to load info file '" TEST_PATH "/repo2/archive/test/archive.info' or"
573 " '" TEST_PATH "/repo2/archive/test/archive.info.copy':\n"
574 "FileOpenError: unable to open file '" TEST_PATH "/repo2/archive/test/archive.info' for read: [13] Permission denied\n"
575 "FileOpenError: unable to open file '" TEST_PATH "/repo2/archive/test/archive.info.copy' for read:"
576 " [13] Permission denied\n"
577 "HINT: archive.info cannot be opened but is required to push/get WAL segments.\n"
578 "HINT: is archive_command configured correctly in postgresql.conf?\n"
579 "HINT: has a stanza-create been performed?\n"
580 "HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.");
581
582 // Make sure WAL got pushed to repo3
583 TEST_STORAGE_EXISTS(
584 storageTest, strZ(strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
585 .remove = true);
586
587 HRN_STORAGE_MODE(storageTest, "repo2");
588
589 // -------------------------------------------------------------------------------------------------------------------------
590 TEST_TITLE("push succeeds on one repo when other repo fails to read path");
591
592 HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1", .mode = 0200);
593
594 TEST_ERROR(
595 cmdArchivePush(), CommandError,
596 "archive-push command encountered error(s):\n"
597 "repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test/11-1/0000000100000001':"
598 " [13] Permission denied");
599
600 // Make sure WAL got pushed to repo3
601 TEST_STORAGE_EXISTS(
602 storageTest, strZ(strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
603 .remove = true);
604
605 HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1");
606 }
607
608 // *****************************************************************************************************************************
609 if (testBegin("Asynchronous cmdArchivePush() and cmdArchivePushAsync()"))
610 {
611 harnessLogLevelSet(logLevelDetail);
612
613 // Install local command handler shim
614 static const ProtocolServerHandler testLocalHandlerList[] = {PROTOCOL_SERVER_HANDLER_ARCHIVE_PUSH_LIST};
615 hrnProtocolLocalShimInstall(testLocalHandlerList, PROTOCOL_SERVER_HANDLER_LIST_SIZE(testLocalHandlerList));
616
617 // -------------------------------------------------------------------------------------------------------------------------
618 TEST_TITLE("command must be run on the pg host");
619
620 StringList *argList = strLstNew();
621 hrnCfgArgRawZ(argList, cfgOptPgHost, "host");
622 hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
623 strLstAddZ(argList, "--" CFGOPT_SPOOL_PATH "=/spool");
624 strLstAddZ(argList, "--" CFGOPT_STANZA "=test2");
625 strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC);
626 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
627
628 TEST_ERROR(cmdArchivePush(), HostInvalidError, "archive-push command must be run on the PostgreSQL host");
629
630 // -------------------------------------------------------------------------------------------------------------------------
631 TEST_TITLE("pg1-path must be set when async");
632
633 argList = strLstNew();
634 hrnCfgArgRawZ(argList, cfgOptSpoolPath, "/spool");
635 hrnCfgArgRawZ(argList, cfgOptStanza, "test2");
636 hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
637 strLstAddZ(argList, "/000000010000000100000001");
638 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
639
640 TEST_ERROR(cmdArchivePush(), OptionRequiredError, "'archive-push' command in async mode requires option 'pg1-path'");
641
642 // -------------------------------------------------------------------------------------------------------------------------
643 TEST_TITLE("check timeout on async error");
644
645 // Call with a bogus exe name so the async process will error out and we can make sure timeouts work
646 argList = strLstNew();
647 hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
648 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
649 hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
650 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
651 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
652 hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, "1");
653 strLstAddZ(argList, "pg_wal/bogus");
654 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .exeBogus = true);
655
656 HRN_STORAGE_PATH_CREATE(storageTest, strZ(cfgOptionStr(cfgOptPgPath)));
657 THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
658
659 TEST_ERROR(
660 cmdArchivePush(), ArchiveTimeoutError,
661 "unable to push WAL file 'bogus' to the archive asynchronously after 1 second(s)");
662
663 // Create pg_control and archive.info for next set of tests
664 // -------------------------------------------------------------------------------------------------------------------------
665 argList = strLstNew();
666 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
667 hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
668 hrnCfgArgRawBool(argList, cfgOptCompress, false);
669 hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
670 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
671 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
672 hrnCfgArgRawBool(argList, cfgOptLogSubprocess, true);
673
674 HRN_STORAGE_PUT(storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
675 hrnPgControlToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}));
676
677 HRN_INFO_PUT(
678 storageTest, "repo/archive/test/archive.info",
679 "[db]\n"
680 "db-id=1\n"
681 "\n"
682 "[db:history]\n"
683 "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n");
684
685 // -------------------------------------------------------------------------------------------------------------------------
686 TEST_TITLE("async, ignore error file on first pass");
687
688 // Write out an error file that will be ignored on the first pass, then the async process will write a new one
689 StringList *argListTemp = strLstDup(argList);
690 strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
691 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
692
693 HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status");
694
695 HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error", "25\nbogus error");
696
697 TEST_ERROR(cmdArchivePush(), AssertError, "no WAL files to process");
698
699 TEST_STORAGE_EXISTS(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error", .remove = true);
700
701 // -------------------------------------------------------------------------------------------------------------------------
702 TEST_TITLE("with lock, prevent async from running");
703
704 // Acquire a lock so the async process will not be able to run -- this will result in a timeout
705 argListTemp = strLstDup(argList);
706 strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
707 hrnCfgArgRawZ(argListTemp, cfgOptArchiveTimeout, "1");
708 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
709
710 THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
711
712 HRN_FORK_BEGIN()
713 {
714 HRN_FORK_CHILD_BEGIN()
715 {
716 lockAcquire(
717 cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), STRDEF("555-fefefefe"), cfgLockType(), 30000, true);
718
719 // Notify parent that lock has been acquired
720 HRN_FORK_CHILD_NOTIFY_PUT();
721
722 // Wait for parent to allow release lock
723 HRN_FORK_CHILD_NOTIFY_GET();
724
725 lockRelease(true);
726 }
727 HRN_FORK_CHILD_END();
728
729 HRN_FORK_PARENT_BEGIN()
730 {
731 // Wait for child to acquire lock
732 HRN_FORK_PARENT_NOTIFY_GET(0);
733
734 TEST_ERROR(
735 cmdArchivePush(), ArchiveTimeoutError,
736 "unable to push WAL file '000000010000000100000001' to the archive asynchronously after 1 second(s)");
737
738 // Notify child to release lock
739 HRN_FORK_PARENT_NOTIFY_PUT(0);
740 }
741 HRN_FORK_PARENT_END();
742 }
743 HRN_FORK_END();
744
745 // -------------------------------------------------------------------------------------------------------------------------
746 TEST_TITLE("async WAL push");
747
748 // Actually push a WAL file
749 argListTemp = strLstDup(argList);
750 strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
751 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
752
753 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000001.ready");
754
755 Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
756 bufUsedSet(walBuffer1, bufSize(walBuffer1));
757 memset(bufPtr(walBuffer1), 0xFF, bufSize(walBuffer1));
758 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1);
759 const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
760
761 HRN_STORAGE_PUT(storagePgWrite(),"pg_xlog/000000010000000100000001", walBuffer1);
762
763 TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
764 TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive asynchronously");
765
766 TEST_STORAGE_EXISTS(
767 storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
768 .comment = "check repo for WAL file");
769
770 // -------------------------------------------------------------------------------------------------------------------------
771 TEST_TITLE("direct tests of the async function");
772
773 argList = strLstNew();
774 hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
775 hrnCfgArgRawZ(argList, cfgOptStanza, "test");
776 hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
777 hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
778 hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
779 hrnCfgArgRawBool(argList, cfgOptLogSubprocess, true);
780 hrnCfgArgRawZ(argList, cfgOptCompressType, "none");
781 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
782
783 TEST_ERROR(cmdArchivePushAsync(), ParamRequiredError, "WAL path to push required");
784
785 // -------------------------------------------------------------------------------------------------------------------------
786 TEST_TITLE("async, check that global.error is created");
787
788 // Remove data from prior tests
789 HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true);
790 HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT);
791
792 HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "pg_xlog/archive_status", .recurse = true);
793 HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status");
794
795 strLstAddZ(argList, TEST_PATH "/pg/pg_xlog");
796 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
797
798 TEST_ERROR(cmdArchivePushAsync(), AssertError, "no WAL files to process");
799
800 TEST_STORAGE_GET(
801 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error", "25\nno WAL files to process",
802 .comment = "check global.error");
803
804 TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "global.error\n", .comment = "check status files");
805
806 // -------------------------------------------------------------------------------------------------------------------------
807 TEST_TITLE("add repo, push already pushed WAL and new WAL");
808
809 // Add repo3
810 hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 3, TEST_PATH "/repo3");
811 HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
812
813 HRN_INFO_PUT(
814 storageTest, "repo3/archive/test/archive.info",
815 "[db]\n"
816 "db-id=1\n"
817 "\n"
818 "[db:history]\n"
819 "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n");
820
821 // Recreate ready file for WAL 1
822 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000001.ready");
823
824 TEST_STORAGE_EXISTS(
825 storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
826 .comment = "check repo1 for WAL 1 file");
827
828 // Create a ready file for WAL 2 but don't create the segment yet -- this will test the file error
829 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000002.ready");
830
831 TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
832 TEST_RESULT_LOG_FMT(
833 "P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000001...000000010000000100000002\n"
834 "P01 WARN: WAL file '000000010000000100000001' already exists in the repo1 archive with the same checksum\n"
835 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
836 "P01 DETAIL: pushed WAL file '000000010000000100000001' to the archive\n"
837 "P01 WARN: could not push WAL file '000000010000000100000002' to the archive (will be retried): "
838 "[55] raised from local-1 shim protocol: " STORAGE_ERROR_READ_MISSING,
839 TEST_PATH "/pg/pg_xlog/000000010000000100000002");
840
841 TEST_STORAGE_EXISTS(
842 storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
843 .comment = "check repo1 for WAL 1 file");
844
845 TEST_STORAGE_EXISTS(
846 storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
847 .comment = "check repo3 for WAL 1 file");
848
849 TEST_STORAGE_LIST(
850 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
851 "000000010000000100000001.ok\n"
852 "000000010000000100000002.error\n",
853 .comment = "check status files");
854
855 // -------------------------------------------------------------------------------------------------------------------------
856 TEST_TITLE("create and push previously missing WAL");
857
858 // Create WAL 2 segment
859 Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024);
860 bufUsedSet(walBuffer2, bufSize(walBuffer2));
861 memset(bufPtr(walBuffer2), 0x0C, bufSize(walBuffer2));
862 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2);
863 const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2)));
864
865 HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000002", walBuffer2);
866
867 argListTemp = strLstDup(argList);
868 hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "1gb");
869 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync);
870
871 TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
872 TEST_RESULT_LOG(
873 "P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n"
874 "P01 DETAIL: pushed WAL file '000000010000000100000002' to the archive");
875
876 TEST_STORAGE_EXISTS(
877 storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
878 .comment = "check repo1 for WAL 2 file");
879 TEST_STORAGE_EXISTS(
880 storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
881 .comment = "check repo3 for WAL 2 file");
882
883 TEST_STORAGE_LIST(
884 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
885 "000000010000000100000001.ok\n"
886 "000000010000000100000002.ok\n",
887 .comment = "check status files");
888
889 // -------------------------------------------------------------------------------------------------------------------------
890 TEST_TITLE("push wal 2 again to get warnings from both repos");
891
892 // Remove the OK file so the WAL gets pushed again
893 HRN_STORAGE_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok");
894
895 TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
896 TEST_RESULT_LOG(
897 "P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n"
898 "P01 WARN: WAL file '000000010000000100000002' already exists in the repo1 archive with the same checksum\n"
899 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
900 "P01 WARN: WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
901 " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
902 "P01 DETAIL: pushed WAL file '000000010000000100000002' to the archive");
903
904 // -------------------------------------------------------------------------------------------------------------------------
905 TEST_TITLE("create and push WAL 3 to both repos");
906
907 // Create WAL 3 segment
908 Buffer *walBuffer3 = bufNew((size_t)16 * 1024 * 1024);
909 bufUsedSet(walBuffer3, bufSize(walBuffer3));
910 memset(bufPtr(walBuffer3), 0x44, bufSize(walBuffer3));
911 hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer3);
912 const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer3)));
913
914 HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000003", walBuffer3);
915
916 // Create ready file
917 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000003.ready");
918
919 TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segment");
920 TEST_RESULT_LOG(
921 "P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000003\n"
922 "P01 DETAIL: pushed WAL file '000000010000000100000003' to the archive");
923
924 TEST_STORAGE_EXISTS(
925 storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
926 .comment = "check repo1 for WAL 3 file");
927 TEST_STORAGE_EXISTS(
928 storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
929 .comment = "check repo3 for WAL 3 file");
930
931 // Remove the ready file to prevent WAL 3 from being considered for the next test
932 HRN_STORAGE_REMOVE(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000003.ready", .errorOnMissing = true);
933
934 // Check that drop functionality works
935 // -------------------------------------------------------------------------------------------------------------------------
936 // Remove status files
937 HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true);
938 HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT);
939
940 argListTemp = strLstDup(argList);
941 hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "16m");
942 HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync);
943
944 TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
945 TEST_RESULT_LOG(
946 "P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000001...000000010000000100000002\n"
947 "P00 WARN: dropped WAL file '000000010000000100000001' because archive queue exceeded 16MB\n"
948 "P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
949
950 TEST_STORAGE_GET(
951 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok",
952 "0\ndropped WAL file '000000010000000100000001' because archive queue exceeded 16MB", .comment = "check WAL 1 warning");
953
954 TEST_STORAGE_GET(
955 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok",
956 "0\ndropped WAL file '000000010000000100000002' because archive queue exceeded 16MB", .comment = "check WAL 2 warning");
957
958 TEST_STORAGE_LIST(
959 storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
960 "000000010000000100000001.ok\n"
961 "000000010000000100000002.ok\n",
962 .comment = "check status files");
963
964 // Uninstall local command handler shim
965 hrnProtocolLocalShimUninstall();
966 }
967
968 FUNCTION_HARNESS_RETURN_VOID();
969 }
970