1 /*------------------------------------------------------------------------- 2 * 3 * pg_backup_archiver.h 4 * 5 * Private interface to the pg_dump archiver routines. 6 * It is NOT intended that these routines be called by any 7 * dumper directly. 8 * 9 * See the headers to pg_restore for more details. 10 * 11 * Copyright (c) 2000, Philip Warner 12 * Rights are granted to use this software in any way so long 13 * as this notice is not removed. 14 * 15 * The author is not responsible for loss or damages that may 16 * result from its use. 17 * 18 * 19 * IDENTIFICATION 20 * src/bin/pg_dump/pg_backup_archiver.h 21 * 22 *------------------------------------------------------------------------- 23 */ 24 #ifndef __PG_BACKUP_ARCHIVE__ 25 #define __PG_BACKUP_ARCHIVE__ 26 27 #include <time.h> 28 29 #include "libpq-fe.h" 30 #include "pg_backup.h" 31 #include "pqexpbuffer.h" 32 33 #define LOBBUFSIZE 16384 34 35 /* 36 * Note: zlib.h must be included *after* libpq-fe.h, because the latter may 37 * include ssl.h, which has a naming conflict with zlib.h. 38 */ 39 #ifdef HAVE_LIBZ 40 #include <zlib.h> 41 #define GZCLOSE(fh) gzclose(fh) 42 #define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s)) 43 #define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s)) 44 #define GZEOF(fh) gzeof(fh) 45 #else 46 #define GZCLOSE(fh) fclose(fh) 47 #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) 48 #define GZREAD(p, s, n, fh) fread(p, s, n, fh) 49 #define GZEOF(fh) feof(fh) 50 /* this is just the redefinition of a libz constant */ 51 #define Z_DEFAULT_COMPRESSION (-1) 52 53 typedef struct _z_stream 54 { 55 void *next_in; 56 void *next_out; 57 size_t avail_in; 58 size_t avail_out; 59 } z_stream; 60 typedef z_stream *z_streamp; 61 #endif 62 63 /* Data block types */ 64 #define BLK_DATA 1 65 #define BLK_BLOBS 3 66 67 /* Encode version components into a convenient integer <maj><min><rev> */ 68 #define MAKE_ARCHIVE_VERSION(major, minor, rev) (((major) * 256 + (minor)) * 256 + (rev)) 69 70 #define ARCHIVE_MAJOR(version) (((version) >> 16) & 255) 71 #define ARCHIVE_MINOR(version) (((version) >> 8) & 255) 72 #define ARCHIVE_REV(version) (((version) ) & 255) 73 74 /* Historical version numbers (checked in code) */ 75 #define K_VERS_1_0 MAKE_ARCHIVE_VERSION(1, 0, 0) 76 #define K_VERS_1_2 MAKE_ARCHIVE_VERSION(1, 2, 0) /* Allow No ZLIB */ 77 #define K_VERS_1_3 MAKE_ARCHIVE_VERSION(1, 3, 0) /* BLOBs */ 78 #define K_VERS_1_4 MAKE_ARCHIVE_VERSION(1, 4, 0) /* Date & name in header */ 79 #define K_VERS_1_5 MAKE_ARCHIVE_VERSION(1, 5, 0) /* Handle dependencies */ 80 #define K_VERS_1_6 MAKE_ARCHIVE_VERSION(1, 6, 0) /* Schema field in TOCs */ 81 #define K_VERS_1_7 MAKE_ARCHIVE_VERSION(1, 7, 0) /* File Offset size in 82 * header */ 83 #define K_VERS_1_8 MAKE_ARCHIVE_VERSION(1, 8, 0) /* change interpretation 84 * of ID numbers and 85 * dependencies */ 86 #define K_VERS_1_9 MAKE_ARCHIVE_VERSION(1, 9, 0) /* add default_with_oids 87 * tracking */ 88 #define K_VERS_1_10 MAKE_ARCHIVE_VERSION(1, 10, 0) /* add tablespace */ 89 #define K_VERS_1_11 MAKE_ARCHIVE_VERSION(1, 11, 0) /* add toc section 90 * indicator */ 91 #define K_VERS_1_12 MAKE_ARCHIVE_VERSION(1, 12, 0) /* add separate BLOB 92 * entries */ 93 #define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path 94 * behavior */ 95 #define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */ 96 97 /* Current archive version number (the format we can output) */ 98 #define K_VERS_MAJOR 1 99 #define K_VERS_MINOR 14 100 #define K_VERS_REV 0 101 #define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV) 102 103 /* Newest format we can read */ 104 #define K_VERS_MAX MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, 255) 105 106 107 /* Flags to indicate disposition of offsets stored in files */ 108 #define K_OFFSET_POS_NOT_SET 1 109 #define K_OFFSET_POS_SET 2 110 #define K_OFFSET_NO_DATA 3 111 112 /* 113 * Special exit values from worker children. We reserve 0 for normal 114 * success; 1 and other small values should be interpreted as crashes. 115 */ 116 #define WORKER_OK 0 117 #define WORKER_CREATE_DONE 10 118 #define WORKER_INHIBIT_DATA 11 119 #define WORKER_IGNORED_ERRORS 12 120 121 typedef struct _archiveHandle ArchiveHandle; 122 typedef struct _tocEntry TocEntry; 123 struct ParallelState; 124 125 #define READ_ERROR_EXIT(fd) \ 126 do { \ 127 if (feof(fd)) \ 128 fatal("could not read from input file: end of file"); \ 129 else \ 130 fatal("could not read from input file: %m"); \ 131 } while (0) 132 133 #define WRITE_ERROR_EXIT \ 134 do { \ 135 fatal("could not write to output file: %m"); \ 136 } while (0) 137 138 typedef enum T_Action 139 { 140 ACT_DUMP, 141 ACT_RESTORE 142 } T_Action; 143 144 typedef void (*ClosePtrType) (ArchiveHandle *AH); 145 typedef void (*ReopenPtrType) (ArchiveHandle *AH); 146 typedef void (*ArchiveEntryPtrType) (ArchiveHandle *AH, TocEntry *te); 147 148 typedef void (*StartDataPtrType) (ArchiveHandle *AH, TocEntry *te); 149 typedef void (*WriteDataPtrType) (ArchiveHandle *AH, const void *data, size_t dLen); 150 typedef void (*EndDataPtrType) (ArchiveHandle *AH, TocEntry *te); 151 152 typedef void (*StartBlobsPtrType) (ArchiveHandle *AH, TocEntry *te); 153 typedef void (*StartBlobPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); 154 typedef void (*EndBlobPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); 155 typedef void (*EndBlobsPtrType) (ArchiveHandle *AH, TocEntry *te); 156 157 typedef int (*WriteBytePtrType) (ArchiveHandle *AH, const int i); 158 typedef int (*ReadBytePtrType) (ArchiveHandle *AH); 159 typedef void (*WriteBufPtrType) (ArchiveHandle *AH, const void *c, size_t len); 160 typedef void (*ReadBufPtrType) (ArchiveHandle *AH, void *buf, size_t len); 161 typedef void (*WriteExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 162 typedef void (*ReadExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 163 typedef void (*PrintExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 164 typedef void (*PrintTocDataPtrType) (ArchiveHandle *AH, TocEntry *te); 165 166 typedef void (*PrepParallelRestorePtrType) (ArchiveHandle *AH); 167 typedef void (*ClonePtrType) (ArchiveHandle *AH); 168 typedef void (*DeClonePtrType) (ArchiveHandle *AH); 169 170 typedef int (*WorkerJobDumpPtrType) (ArchiveHandle *AH, TocEntry *te); 171 typedef int (*WorkerJobRestorePtrType) (ArchiveHandle *AH, TocEntry *te); 172 173 typedef size_t (*CustomOutPtrType) (ArchiveHandle *AH, const void *buf, size_t len); 174 175 typedef enum 176 { 177 SQL_SCAN = 0, /* normal */ 178 SQL_IN_SINGLE_QUOTE, /* '...' literal */ 179 SQL_IN_DOUBLE_QUOTE /* "..." identifier */ 180 } sqlparseState; 181 182 typedef struct 183 { 184 sqlparseState state; /* see above */ 185 bool backSlash; /* next char is backslash quoted? */ 186 PQExpBuffer curCmd; /* incomplete line (NULL if not created) */ 187 } sqlparseInfo; 188 189 typedef enum 190 { 191 STAGE_NONE = 0, 192 STAGE_INITIALIZING, 193 STAGE_PROCESSING, 194 STAGE_FINALIZING 195 } ArchiverStage; 196 197 typedef enum 198 { 199 OUTPUT_SQLCMDS = 0, /* emitting general SQL commands */ 200 OUTPUT_COPYDATA, /* writing COPY data */ 201 OUTPUT_OTHERDATA /* writing data as INSERT commands */ 202 } ArchiverOutput; 203 204 /* 205 * For historical reasons, ACL items are interspersed with everything else in 206 * a dump file's TOC; typically they're right after the object they're for. 207 * However, we need to restore data before ACLs, as otherwise a read-only 208 * table (ie one where the owner has revoked her own INSERT privilege) causes 209 * data restore failures. On the other hand, matview REFRESH commands should 210 * come out after ACLs, as otherwise non-superuser-owned matviews might not 211 * be able to execute. (If the permissions at the time of dumping would not 212 * allow a REFRESH, too bad; we won't fix that for you.) We also want event 213 * triggers to be restored after ACLs, so that they can't mess those up. 214 * 215 * These considerations force us to make three passes over the TOC, 216 * restoring the appropriate subset of items in each pass. We assume that 217 * the dependency sort resulted in an appropriate ordering of items within 218 * each subset. 219 * 220 * XXX This mechanism should be superseded by tracking dependencies on ACLs 221 * properly; but we'll still need it for old dump files even after that. 222 */ 223 typedef enum 224 { 225 RESTORE_PASS_MAIN = 0, /* Main pass (most TOC item types) */ 226 RESTORE_PASS_ACL, /* ACL item types */ 227 RESTORE_PASS_POST_ACL /* Event trigger and matview refresh items */ 228 229 #define RESTORE_PASS_LAST RESTORE_PASS_POST_ACL 230 } RestorePass; 231 232 #define REQ_SCHEMA 0x01 /* want schema */ 233 #define REQ_DATA 0x02 /* want data */ 234 #define REQ_SPECIAL 0x04 /* for special TOC entries */ 235 236 struct _archiveHandle 237 { 238 Archive public; /* Public part of archive */ 239 int version; /* Version of file */ 240 241 char *archiveRemoteVersion; /* When reading an archive, the 242 * version of the dumped DB */ 243 char *archiveDumpVersion; /* When reading an archive, the version of 244 * the dumper */ 245 246 size_t intSize; /* Size of an integer in the archive */ 247 size_t offSize; /* Size of a file offset in the archive - 248 * Added V1.7 */ 249 ArchiveFormat format; /* Archive format */ 250 251 sqlparseInfo sqlparse; /* state for parsing INSERT data */ 252 253 time_t createDate; /* Date archive created */ 254 255 /* 256 * Fields used when discovering archive format. For tar format, we load 257 * the first block into the lookahead buffer, and verify that it looks 258 * like a tar header. The tar module must then consume bytes from the 259 * lookahead buffer before reading any more from the file. For custom 260 * format, we load only the "PGDMP" marker into the buffer, and then set 261 * readHeader after confirming it matches. The buffer is vestigial in 262 * this case, as the subsequent code just checks readHeader and doesn't 263 * examine the buffer. 264 */ 265 int readHeader; /* Set if we already read "PGDMP" marker */ 266 char *lookahead; /* Buffer used when reading header to discover 267 * format */ 268 size_t lookaheadSize; /* Allocated size of buffer */ 269 size_t lookaheadLen; /* Length of valid data in lookahead */ 270 size_t lookaheadPos; /* Current read position in lookahead buffer */ 271 272 ArchiveEntryPtrType ArchiveEntryPtr; /* Called for each metadata object */ 273 StartDataPtrType StartDataPtr; /* Called when table data is about to be 274 * dumped */ 275 WriteDataPtrType WriteDataPtr; /* Called to send some table data to the 276 * archive */ 277 EndDataPtrType EndDataPtr; /* Called when table data dump is finished */ 278 WriteBytePtrType WriteBytePtr; /* Write a byte to output */ 279 ReadBytePtrType ReadBytePtr; /* Read a byte from an archive */ 280 WriteBufPtrType WriteBufPtr; /* Write a buffer of output to the archive */ 281 ReadBufPtrType ReadBufPtr; /* Read a buffer of input from the archive */ 282 ClosePtrType ClosePtr; /* Close the archive */ 283 ReopenPtrType ReopenPtr; /* Reopen the archive */ 284 WriteExtraTocPtrType WriteExtraTocPtr; /* Write extra TOC entry data 285 * associated with the current 286 * archive format */ 287 ReadExtraTocPtrType ReadExtraTocPtr; /* Read extra info associated with 288 * archive format */ 289 PrintExtraTocPtrType PrintExtraTocPtr; /* Extra TOC info for format */ 290 PrintTocDataPtrType PrintTocDataPtr; 291 292 StartBlobsPtrType StartBlobsPtr; 293 EndBlobsPtrType EndBlobsPtr; 294 StartBlobPtrType StartBlobPtr; 295 EndBlobPtrType EndBlobPtr; 296 297 SetupWorkerPtrType SetupWorkerPtr; 298 WorkerJobDumpPtrType WorkerJobDumpPtr; 299 WorkerJobRestorePtrType WorkerJobRestorePtr; 300 301 PrepParallelRestorePtrType PrepParallelRestorePtr; 302 ClonePtrType ClonePtr; /* Clone format-specific fields */ 303 DeClonePtrType DeClonePtr; /* Clean up cloned fields */ 304 305 CustomOutPtrType CustomOutPtr; /* Alternative script output routine */ 306 307 /* Stuff for direct DB connection */ 308 char *archdbname; /* DB name *read* from archive */ 309 char *savedPassword; /* password for ropt->username, if known */ 310 char *use_role; 311 PGconn *connection; 312 /* If connCancel isn't NULL, SIGINT handler will send a cancel */ 313 PGcancel *volatile connCancel; 314 315 int connectToDB; /* Flag to indicate if direct DB connection is 316 * required */ 317 ArchiverOutput outputKind; /* Flag for what we're currently writing */ 318 bool pgCopyIn; /* Currently in libpq 'COPY IN' mode. */ 319 320 int loFd; /* BLOB fd */ 321 int writingBlob; /* Flag */ 322 int blobCount; /* # of blobs restored */ 323 324 char *fSpec; /* Archive File Spec */ 325 FILE *FH; /* General purpose file handle */ 326 void *OF; 327 int gzOut; /* Output file */ 328 329 struct _tocEntry *toc; /* Header of circular list of TOC entries */ 330 int tocCount; /* Number of TOC entries */ 331 DumpId maxDumpId; /* largest DumpId among all TOC entries */ 332 333 /* arrays created after the TOC list is complete: */ 334 struct _tocEntry **tocsByDumpId; /* TOCs indexed by dumpId */ 335 DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ 336 337 struct _tocEntry *currToc; /* Used when dumping data */ 338 int compression; /*--------- 339 * Compression requested on open(). 340 * Possible values for compression: 341 * -1 Z_DEFAULT_COMPRESSION 342 * 0 COMPRESSION_NONE 343 * 1-9 levels for gzip compression 344 *--------- 345 */ 346 bool dosync; /* data requested to be synced on sight */ 347 ArchiveMode mode; /* File mode - r or w */ 348 void *formatData; /* Header data specific to file format */ 349 350 /* these vars track state to avoid sending redundant SET commands */ 351 char *currUser; /* current username, or NULL if unknown */ 352 char *currSchema; /* current schema, or NULL */ 353 char *currTablespace; /* current tablespace, or NULL */ 354 char *currTableAm; /* current table access method, or NULL */ 355 356 void *lo_buf; 357 size_t lo_buf_used; 358 size_t lo_buf_size; 359 360 int noTocComments; 361 ArchiverStage stage; 362 ArchiverStage lastErrorStage; 363 RestorePass restorePass; /* used only during parallel restore */ 364 struct _tocEntry *currentTE; 365 struct _tocEntry *lastErrorTE; 366 }; 367 368 struct _tocEntry 369 { 370 struct _tocEntry *prev; 371 struct _tocEntry *next; 372 CatalogId catalogId; 373 DumpId dumpId; 374 teSection section; 375 bool hadDumper; /* Archiver was passed a dumper routine (used 376 * in restore) */ 377 char *tag; /* index tag */ 378 char *namespace; /* null or empty string if not in a schema */ 379 char *tablespace; /* null if not in a tablespace; empty string 380 * means use database default */ 381 char *tableam; /* table access method, only for TABLE tags */ 382 char *owner; 383 char *desc; 384 char *defn; 385 char *dropStmt; 386 char *copyStmt; 387 DumpId *dependencies; /* dumpIds of objects this one depends on */ 388 int nDeps; /* number of dependencies */ 389 390 DataDumperPtr dataDumper; /* Routine to dump data for object */ 391 const void *dataDumperArg; /* Arg for above routine */ 392 void *formatData; /* TOC Entry data specific to file format */ 393 394 /* working state while dumping/restoring */ 395 pgoff_t dataLength; /* item's data size; 0 if none or unknown */ 396 int reqs; /* do we need schema and/or data of object 397 * (REQ_* bit mask) */ 398 bool created; /* set for DATA member if TABLE was created */ 399 400 /* working state (needed only for parallel restore) */ 401 struct _tocEntry *pending_prev; /* list links for pending-items list; */ 402 struct _tocEntry *pending_next; /* NULL if not in that list */ 403 int depCount; /* number of dependencies not yet restored */ 404 DumpId *revDeps; /* dumpIds of objects depending on this one */ 405 int nRevDeps; /* number of such dependencies */ 406 DumpId *lockDeps; /* dumpIds of objects this one needs lock on */ 407 int nLockDeps; /* number of such dependencies */ 408 }; 409 410 extern int parallel_restore(ArchiveHandle *AH, TocEntry *te); 411 extern void on_exit_close_archive(Archive *AHX); 412 413 extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3); 414 415 /* Options for ArchiveEntry */ 416 typedef struct _archiveOpts 417 { 418 const char *tag; 419 const char *namespace; 420 const char *tablespace; 421 const char *tableam; 422 const char *owner; 423 const char *description; 424 teSection section; 425 const char *createStmt; 426 const char *dropStmt; 427 const char *copyStmt; 428 const DumpId *deps; 429 int nDeps; 430 DataDumperPtr dumpFn; 431 const void *dumpArg; 432 } ArchiveOpts; 433 #define ARCHIVE_OPTS(...) &(ArchiveOpts){__VA_ARGS__} 434 /* Called to add a TOC entry */ 435 extern TocEntry *ArchiveEntry(Archive *AHX, CatalogId catalogId, 436 DumpId dumpId, ArchiveOpts *opts); 437 438 extern void WriteHead(ArchiveHandle *AH); 439 extern void ReadHead(ArchiveHandle *AH); 440 extern void WriteToc(ArchiveHandle *AH); 441 extern void ReadToc(ArchiveHandle *AH); 442 extern void WriteDataChunks(ArchiveHandle *AH, struct ParallelState *pstate); 443 extern void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te); 444 extern ArchiveHandle *CloneArchive(ArchiveHandle *AH); 445 extern void DeCloneArchive(ArchiveHandle *AH); 446 447 extern int TocIDRequired(ArchiveHandle *AH, DumpId id); 448 TocEntry *getTocEntryByDumpId(ArchiveHandle *AH, DumpId id); 449 extern bool checkSeek(FILE *fp); 450 451 #define appendStringLiteralAHX(buf,str,AH) \ 452 appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings) 453 454 #define appendByteaLiteralAHX(buf,str,len,AH) \ 455 appendByteaLiteral(buf, str, len, (AH)->public.std_strings) 456 457 /* 458 * Mandatory routines for each supported format 459 */ 460 461 extern size_t WriteInt(ArchiveHandle *AH, int i); 462 extern int ReadInt(ArchiveHandle *AH); 463 extern char *ReadStr(ArchiveHandle *AH); 464 extern size_t WriteStr(ArchiveHandle *AH, const char *s); 465 466 int ReadOffset(ArchiveHandle *, pgoff_t *); 467 size_t WriteOffset(ArchiveHandle *, pgoff_t, int); 468 469 extern void StartRestoreBlobs(ArchiveHandle *AH); 470 extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop); 471 extern void EndRestoreBlob(ArchiveHandle *AH, Oid oid); 472 extern void EndRestoreBlobs(ArchiveHandle *AH); 473 474 extern void InitArchiveFmt_Custom(ArchiveHandle *AH); 475 extern void InitArchiveFmt_Null(ArchiveHandle *AH); 476 extern void InitArchiveFmt_Directory(ArchiveHandle *AH); 477 extern void InitArchiveFmt_Tar(ArchiveHandle *AH); 478 479 extern bool isValidTarHeader(char *header); 480 481 extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname); 482 extern void DropBlobIfExists(ArchiveHandle *AH, Oid oid); 483 484 void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH); 485 int ahprintf(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3); 486 487 #endif 488