1 /***********************************************************************************************************************************
2 Backup Manifest Handler
3
4 The backup manifest stores a complete list of all files, links, and paths in a backup along with metadata such as checksums, sizes,
5 timestamps, etc. A list of databases is also included for selective restore.
6
7 The purpose of the manifest is to allow the restore command to confidently reconstruct the PostgreSQL data directory and ensure that
8 nothing is missing or corrupt. It is also useful for reporting, e.g. size of backup, backup time, etc.
9 ***********************************************************************************************************************************/
10 #ifndef INFO_MANIFEST_H
11 #define INFO_MANIFEST_H
12
13 /***********************************************************************************************************************************
14 Constants
15 ***********************************************************************************************************************************/
16 #define BACKUP_MANIFEST_FILE "backup.manifest"
17 STRING_DECLARE(BACKUP_MANIFEST_FILE_STR);
18
19 #define MANIFEST_TARGET_PGDATA "pg_data"
20 STRING_DECLARE(MANIFEST_TARGET_PGDATA_STR);
21 #define MANIFEST_TARGET_PGTBLSPC "pg_tblspc"
22 STRING_DECLARE(MANIFEST_TARGET_PGTBLSPC_STR);
23
24 /***********************************************************************************************************************************
25 Object type
26 ***********************************************************************************************************************************/
27 typedef struct Manifest Manifest;
28
29 #include "command/backup/common.h"
30 #include "common/compress/helper.h"
31 #include "common/crypto/common.h"
32 #include "common/crypto/hash.h"
33 #include "common/type/variantList.h"
34 #include "common/type/object.h"
35 #include "info/info.h"
36 #include "info/infoBackup.h"
37 #include "storage/storage.h"
38
39 /***********************************************************************************************************************************
40 Manifest data
41 ***********************************************************************************************************************************/
42 typedef struct ManifestData
43 {
44 const String *backrestVersion; // pgBackRest version
45
46 const String *backupLabel; // Backup label (unique identifier for the backup)
47 const String *backupLabelPrior; // Backup label for backup this diff/incr is based on
48 time_t backupTimestampCopyStart; // When did the file copy start?
49 time_t backupTimestampStart; // When did the backup start?
50 time_t backupTimestampStop; // When did the backup stop?
51 BackupType backupType; // Type of backup: full, diff, incr
52
53 // ??? Note that these fields are redundant and verbose since storing the start/stop lsn as a uint64 would be sufficient.
54 // However, we currently lack the functions to transform these values back and forth so this will do for now.
55 const String *archiveStart; // First WAL file in the backup
56 const String *archiveStop; // Last WAL file in the backup
57 const String *lsnStart; // Start LSN for the backup
58 const String *lsnStop; // Stop LSN for the backup
59
60 unsigned int pgId; // PostgreSQL id in backup.info
61 unsigned int pgVersion; // PostgreSQL version
62 uint64_t pgSystemId; // PostgreSQL system identifier
63 unsigned int pgCatalogVersion; // PostgreSQL catalog version
64
65 bool backupOptionArchiveCheck; // Will WAL segments be checked at the end of the backup?
66 bool backupOptionArchiveCopy; // Will WAL segments be copied to the backup?
67 const Variant *backupOptionStandby; // Will the backup be performed from a standby?
68 const Variant *backupOptionBufferSize; // Buffer size used for file/protocol operations
69 const Variant *backupOptionChecksumPage; // Will page checksums be verified?
70 CompressType backupOptionCompressType; // Compression type used for the backup
71 const Variant *backupOptionCompressLevel; // Level used for compression (if type not none)
72 const Variant *backupOptionCompressLevelNetwork; // Level used for network compression
73 const Variant *backupOptionDelta; // Will a checksum delta be performed?
74 bool backupOptionHardLink; // Will hardlinks be created in the backup?
75 bool backupOptionOnline; // Will an online backup be performed?
76 const Variant *backupOptionProcessMax; // How many processes will be used for backup?
77 } ManifestData;
78
79 /***********************************************************************************************************************************
80 Db type
81 ***********************************************************************************************************************************/
82 typedef struct ManifestDb
83 {
84 const String *name; // Db name (must be first member in struct)
85 unsigned int id; // Db oid
86 unsigned int lastSystemId; // Highest oid used by system objects in this database
87 } ManifestDb;
88
89 /***********************************************************************************************************************************
90 File type
91 ***********************************************************************************************************************************/
92 typedef struct ManifestFile
93 {
94 const String *name; // File name (must be first member in struct)
95 bool primary:1; // Should this file be copied from the primary?
96 bool checksumPage:1; // Does this file have page checksums?
97 bool checksumPageError:1; // Is there an error in the page checksum?
98 mode_t mode; // File mode
99 char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
100 const VariantList *checksumPageErrorList; // List of page checksum errors if there are any
101 const String *user; // User name
102 const String *group; // Group name
103 const String *reference; // Reference to a prior backup
104 uint64_t size; // Original size
105 uint64_t sizeRepo; // Size in repo
106 time_t timestamp; // Original timestamp
107 } ManifestFile;
108
109 /***********************************************************************************************************************************
110 Link type
111 ***********************************************************************************************************************************/
112 typedef struct ManifestLink
113 {
114 const String *name; // Link name (must be first member in struct)
115 const String *destination; // Link destination
116 const String *user; // User name
117 const String *group; // Group name
118 } ManifestLink;
119
120 /***********************************************************************************************************************************
121 Path type
122 ***********************************************************************************************************************************/
123 typedef struct ManifestPath
124 {
125 const String *name; // Path name (must be first member in struct)
126 mode_t mode; // Directory mode
127 const String *user; // User name
128 const String *group; // Group name
129 } ManifestPath;
130
131 /***********************************************************************************************************************************
132 Target type
133 ***********************************************************************************************************************************/
134 typedef enum
135 {
136 manifestTargetTypePath,
137 manifestTargetTypeLink,
138 } ManifestTargetType;
139
140 typedef struct ManifestTarget
141 {
142 const String *name; // Target name (must be first member in struct)
143 ManifestTargetType type; // Target type
144 const String *path; // Target path (if path or link)
145 const String *file; // Target file (if file link)
146 unsigned int tablespaceId; // Oid if this link is a tablespace
147 const String *tablespaceName; // Name of the tablespace
148 } ManifestTarget;
149
150 /***********************************************************************************************************************************
151 Constructors
152 ***********************************************************************************************************************************/
153 // Build a new manifest for a PostgreSQL data directory
154 Manifest *manifestNewBuild(
155 const Storage *storagePg, unsigned int pgVersion, unsigned int pgCatalogVersion, bool online, bool checksumPage,
156 const StringList *excludeList, const VariantList *tablespaceList);
157
158 // Load a manifest from IO
159 Manifest *manifestNewLoad(IoRead *read);
160
161 /***********************************************************************************************************************************
162 Getters/Setters
163 ***********************************************************************************************************************************/
164 typedef struct ManifestPub
165 {
166 MemContext *memContext; // Mem context
167 Info *info; // Base info object
168 ManifestData data; // Manifest data and options
169 List *dbList; // List of databases
170 List *fileList; // List of files
171 List *linkList; // List of links
172 List *pathList; // List of paths
173 List *targetList; // List of targets
174 } ManifestPub;
175
176 // Get/set the cipher subpassphrase
177 __attribute__((always_inline)) static inline const String *
manifestCipherSubPass(const Manifest * const this)178 manifestCipherSubPass(const Manifest *const this)
179 {
180 return infoCipherPass(THIS_PUB(Manifest)->info);
181 }
182
183 __attribute__((always_inline)) static inline void
manifestCipherSubPassSet(Manifest * const this,const String * const cipherSubPass)184 manifestCipherSubPassSet(Manifest *const this, const String *const cipherSubPass)
185 {
186 infoCipherPassSet(THIS_PUB(Manifest)->info, cipherSubPass);
187 }
188
189 // Get manifest configuration and options
190 __attribute__((always_inline)) static inline const ManifestData *
manifestData(const Manifest * const this)191 manifestData(const Manifest *const this)
192 {
193 return &(THIS_PUB(Manifest)->data);
194 }
195
196 // Set backup label
197 void manifestBackupLabelSet(Manifest *this, const String *backupLabel);
198
199 /***********************************************************************************************************************************
200 Build functions
201 ***********************************************************************************************************************************/
202 // Validate the timestamps in the manifest given a copy start time, i.e. all times should be <= the copy start time
203 void manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType compressType);
204
205 // Create a diff/incr backup by comparing to a previous backup manifest
206 void manifestBuildIncr(Manifest *this, const Manifest *prior, BackupType type, const String *archiveStart);
207
208 // Set remaining values before the final save
209 void manifestBuildComplete(
210 Manifest *this, time_t timestampStart, const String *lsnStart, const String *archiveStart, time_t timestampStop,
211 const String *lsnStop, const String *archiveStop, unsigned int pgId, uint64_t pgSystemId, const VariantList *dbList,
212 bool optionArchiveCheck, bool optionArchiveCopy, size_t optionBufferSize, unsigned int optionCompressLevel,
213 unsigned int optionCompressLevelNetwork, bool optionHardLink, unsigned int optionProcessMax, bool optionStandby);
214
215 /***********************************************************************************************************************************
216 Functions
217 ***********************************************************************************************************************************/
218 // Ensure that symlinks do not point to the same file, directory, or subdirectory of another link
219 void manifestLinkCheck(const Manifest *this);
220
221 // Move to a new parent mem context
222 __attribute__((always_inline)) static inline Manifest *
manifestMove(Manifest * const this,MemContext * const parentNew)223 manifestMove(Manifest *const this, MemContext *const parentNew)
224 {
225 return objMove(this, parentNew);
226 }
227
228 // Manifest save
229 void manifestSave(Manifest *this, IoWrite *write);
230
231 // Validate a completed manifest. Use strict mode only when saving the manifest after a backup.
232 void manifestValidate(Manifest *this, bool strict);
233
234 /***********************************************************************************************************************************
235 Db functions and getters/setters
236 ***********************************************************************************************************************************/
237 __attribute__((always_inline)) static inline const ManifestDb *
manifestDb(const Manifest * const this,const unsigned int dbIdx)238 manifestDb(const Manifest *const this, const unsigned int dbIdx)
239 {
240 return lstGet(THIS_PUB(Manifest)->dbList, dbIdx);
241 }
242
243 const ManifestDb *manifestDbFind(const Manifest *this, const String *name);
244
245 // If the database requested is not found in the list, return the default passed rather than throw an error
246 __attribute__((always_inline)) static inline const ManifestDb *
manifestDbFindDefault(const Manifest * const this,const String * const name,const ManifestDb * const dbDefault)247 manifestDbFindDefault(const Manifest *const this, const String *const name, const ManifestDb *const dbDefault)
248 {
249 ASSERT_INLINE(name != NULL);
250 return lstFindDefault(THIS_PUB(Manifest)->dbList, &name, (void *)dbDefault);
251 }
252
253 __attribute__((always_inline)) static inline unsigned int
manifestDbTotal(const Manifest * const this)254 manifestDbTotal(const Manifest *const this)
255 {
256 return lstSize(THIS_PUB(Manifest)->dbList);
257 }
258
259 /***********************************************************************************************************************************
260 File functions and getters/setters
261 ***********************************************************************************************************************************/
262 __attribute__((always_inline)) static inline const ManifestFile *
manifestFile(const Manifest * const this,const unsigned int fileIdx)263 manifestFile(const Manifest *const this, const unsigned int fileIdx)
264 {
265 return lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
266 }
267
268 void manifestFileAdd(Manifest *this, const ManifestFile *file);
269 const ManifestFile *manifestFileFind(const Manifest *this, const String *name);
270
271 // If the file requested is not found in the list, return the default passed rather than throw an error
272 __attribute__((always_inline)) static inline const ManifestFile *
manifestFileFindDefault(const Manifest * const this,const String * const name,const ManifestFile * const fileDefault)273 manifestFileFindDefault(const Manifest *const this, const String *const name, const ManifestFile *const fileDefault)
274 {
275 ASSERT_INLINE(name != NULL);
276 return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, (void *)fileDefault);
277 }
278
279 void manifestFileRemove(const Manifest *this, const String *name);
280
281 __attribute__((always_inline)) static inline unsigned int
manifestFileTotal(const Manifest * const this)282 manifestFileTotal(const Manifest *const this)
283 {
284 return lstSize(THIS_PUB(Manifest)->fileList);
285 }
286
287 // Update a file with new data
288 void manifestFileUpdate(
289 Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
290 bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList);
291
292 /***********************************************************************************************************************************
293 Link functions and getters/setters
294 ***********************************************************************************************************************************/
295 __attribute__((always_inline)) static inline const ManifestLink *
manifestLink(const Manifest * const this,const unsigned int linkIdx)296 manifestLink(const Manifest *const this, const unsigned int linkIdx)
297 {
298 return lstGet(THIS_PUB(Manifest)->linkList, linkIdx);
299 }
300
301 const ManifestLink *manifestLinkFind(const Manifest *this, const String *name);
302
303 // If the link requested is not found in the list, return the default passed rather than throw an error
304 __attribute__((always_inline)) static inline const ManifestLink *
manifestLinkFindDefault(const Manifest * const this,const String * const name,const ManifestLink * const linkDefault)305 manifestLinkFindDefault(const Manifest *const this, const String *const name, const ManifestLink *const linkDefault)
306 {
307 ASSERT_INLINE(name != NULL);
308 return lstFindDefault(THIS_PUB(Manifest)->linkList, &name, (void *)linkDefault);
309 }
310
311 void manifestLinkRemove(const Manifest *this, const String *name);
312
313 __attribute__((always_inline)) static inline unsigned int
manifestLinkTotal(const Manifest * const this)314 manifestLinkTotal(const Manifest *const this)
315 {
316 return lstSize(THIS_PUB(Manifest)->linkList);
317 }
318
319 void manifestLinkUpdate(const Manifest *this, const String *name, const String *path);
320
321 /***********************************************************************************************************************************
322 Path functions and getters/setters
323 ***********************************************************************************************************************************/
324 __attribute__((always_inline)) static inline const ManifestPath *
manifestPath(const Manifest * const this,const unsigned int pathIdx)325 manifestPath(const Manifest *const this, const unsigned int pathIdx)
326 {
327 return lstGet(THIS_PUB(Manifest)->pathList, pathIdx);
328 }
329
330 const ManifestPath *manifestPathFind(const Manifest *this, const String *name);
331
332 // If the path requested is not found in the list, return the default passed rather than throw an error
333 __attribute__((always_inline)) static inline const ManifestPath *
manifestPathFindDefault(const Manifest * const this,const String * const name,const ManifestPath * const pathDefault)334 manifestPathFindDefault(const Manifest *const this, const String *const name, const ManifestPath *const pathDefault)
335 {
336 ASSERT_INLINE(name != NULL);
337 return lstFindDefault(THIS_PUB(Manifest)->pathList, &name, (void *)pathDefault);
338 }
339
340 // Data directory relative path for any manifest file/link/path/target name
341 String *manifestPathPg(const String *manifestPath);
342
343 __attribute__((always_inline)) static inline unsigned int
manifestPathTotal(const Manifest * const this)344 manifestPathTotal(const Manifest *const this)
345 {
346 return lstSize(THIS_PUB(Manifest)->pathList);
347 }
348
349 /***********************************************************************************************************************************
350 Target functions and getters/setters
351 ***********************************************************************************************************************************/
352 __attribute__((always_inline)) static inline const ManifestTarget *
manifestTarget(const Manifest * const this,const unsigned int targetIdx)353 manifestTarget(const Manifest *const this, const unsigned int targetIdx)
354 {
355 return lstGet(THIS_PUB(Manifest)->targetList, targetIdx);
356 }
357
358 const ManifestTarget *manifestTargetFind(const Manifest *this, const String *name);
359
360 // Base target, i.e. the target that is the data directory
361 __attribute__((always_inline)) static inline const ManifestTarget *
manifestTargetBase(const Manifest * const this)362 manifestTargetBase(const Manifest *const this)
363 {
364 return manifestTargetFind(this, MANIFEST_TARGET_PGDATA_STR);
365 }
366
367 // Absolute path to the target
368 String *manifestTargetPath(const Manifest *this, const ManifestTarget *target);
369
370 void manifestTargetRemove(const Manifest *this, const String *name);
371
372 __attribute__((always_inline)) static inline unsigned int
manifestTargetTotal(const Manifest * const this)373 manifestTargetTotal(const Manifest *const this)
374 {
375 return lstSize(THIS_PUB(Manifest)->targetList);
376 }
377
378 void manifestTargetUpdate(const Manifest *this, const String *name, const String *path, const String *file);
379
380 /***********************************************************************************************************************************
381 Destructor
382 ***********************************************************************************************************************************/
383 __attribute__((always_inline)) static inline void
manifestFree(Manifest * const this)384 manifestFree(Manifest *const this)
385 {
386 objFree(this);
387 }
388
389 /***********************************************************************************************************************************
390 Helper functions
391 ***********************************************************************************************************************************/
392 // Load backup manifest
393 Manifest *manifestLoadFile(const Storage *storage, const String *fileName, CipherType cipherType, const String *cipherPass);
394
395 /***********************************************************************************************************************************
396 Macros for function logging
397 ***********************************************************************************************************************************/
398 #define FUNCTION_LOG_MANIFEST_TYPE \
399 Manifest *
400 #define FUNCTION_LOG_MANIFEST_FORMAT(value, buffer, bufferSize) \
401 objToLog(value, "Manifest", buffer, bufferSize)
402
403 #define FUNCTION_LOG_MANIFEST_DB_TYPE \
404 ManifestDb *
405 #define FUNCTION_LOG_MANIFEST_DB_FORMAT(value, buffer, bufferSize) \
406 objToLog(value, "ManifestDb", buffer, bufferSize)
407
408 #define FUNCTION_LOG_MANIFEST_FILE_TYPE \
409 ManifestFile *
410 #define FUNCTION_LOG_MANIFEST_FILE_FORMAT(value, buffer, bufferSize) \
411 objToLog(value, "ManifestFile", buffer, bufferSize)
412
413 #define FUNCTION_LOG_MANIFEST_LINK_TYPE \
414 ManifestLink *
415 #define FUNCTION_LOG_MANIFEST_LINK_FORMAT(value, buffer, bufferSize) \
416 objToLog(value, "ManifestLink", buffer, bufferSize)
417
418 #define FUNCTION_LOG_MANIFEST_PATH_TYPE \
419 ManifestPath *
420 #define FUNCTION_LOG_MANIFEST_PATH_FORMAT(value, buffer, bufferSize) \
421 objToLog(value, "ManifestPath", buffer, bufferSize)
422
423 #define FUNCTION_LOG_MANIFEST_TARGET_TYPE \
424 ManifestTarget *
425 #define FUNCTION_LOG_MANIFEST_TARGET_FORMAT(value, buffer, bufferSize) \
426 objToLog(value, "ManifestTarget", buffer, bufferSize)
427
428 #endif
429