1 /*------------------------------------------------------------------------- 2 * 3 * pg_backup_archiver.h 4 * 5 * Private interface to the pg_dump archiver routines. 6 * It is NOT intended that these routines be called by any 7 * dumper directly. 8 * 9 * See the headers to pg_restore for more details. 10 * 11 * Copyright (c) 2000, Philip Warner 12 * Rights are granted to use this software in any way so long 13 * as this notice is not removed. 14 * 15 * The author is not responsible for loss or damages that may 16 * result from it's use. 17 * 18 * 19 * IDENTIFICATION 20 * src/bin/pg_dump/pg_backup_archiver.h 21 * 22 *------------------------------------------------------------------------- 23 */ 24 #ifndef __PG_BACKUP_ARCHIVE__ 25 #define __PG_BACKUP_ARCHIVE__ 26 27 28 #include <time.h> 29 30 #include "pg_backup.h" 31 32 #include "libpq-fe.h" 33 #include "pqexpbuffer.h" 34 35 #define LOBBUFSIZE 16384 36 37 /* 38 * Note: zlib.h must be included *after* libpq-fe.h, because the latter may 39 * include ssl.h, which has a naming conflict with zlib.h. 40 */ 41 #ifdef HAVE_LIBZ 42 #include <zlib.h> 43 #define GZCLOSE(fh) gzclose(fh) 44 #define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s)) 45 #define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s)) 46 #define GZEOF(fh) gzeof(fh) 47 #else 48 #define GZCLOSE(fh) fclose(fh) 49 #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) 50 #define GZREAD(p, s, n, fh) fread(p, s, n, fh) 51 #define GZEOF(fh) feof(fh) 52 /* this is just the redefinition of a libz constant */ 53 #define Z_DEFAULT_COMPRESSION (-1) 54 55 typedef struct _z_stream 56 { 57 void *next_in; 58 void *next_out; 59 size_t avail_in; 60 size_t avail_out; 61 } z_stream; 62 typedef z_stream *z_streamp; 63 #endif 64 65 /* Data block types */ 66 #define BLK_DATA 1 67 #define BLK_BLOBS 3 68 69 /* Encode version components into a convenient integer <maj><min><rev> */ 70 #define MAKE_ARCHIVE_VERSION(major, minor, rev) (((major) * 256 + (minor)) * 256 + (rev)) 71 72 #define ARCHIVE_MAJOR(version) (((version) >> 16) & 255) 73 #define ARCHIVE_MINOR(version) (((version) >> 8) & 255) 74 #define ARCHIVE_REV(version) (((version) ) & 255) 75 76 /* Historical version numbers (checked in code) */ 77 #define K_VERS_1_0 MAKE_ARCHIVE_VERSION(1, 0, 0) 78 #define K_VERS_1_2 MAKE_ARCHIVE_VERSION(1, 2, 0) /* Allow No ZLIB */ 79 #define K_VERS_1_3 MAKE_ARCHIVE_VERSION(1, 3, 0) /* BLOBs */ 80 #define K_VERS_1_4 MAKE_ARCHIVE_VERSION(1, 4, 0) /* Date & name in header */ 81 #define K_VERS_1_5 MAKE_ARCHIVE_VERSION(1, 5, 0) /* Handle dependencies */ 82 #define K_VERS_1_6 MAKE_ARCHIVE_VERSION(1, 6, 0) /* Schema field in TOCs */ 83 #define K_VERS_1_7 MAKE_ARCHIVE_VERSION(1, 7, 0) /* File Offset size in 84 * header */ 85 #define K_VERS_1_8 MAKE_ARCHIVE_VERSION(1, 8, 0) /* change interpretation 86 * of ID numbers and 87 * dependencies */ 88 #define K_VERS_1_9 MAKE_ARCHIVE_VERSION(1, 9, 0) /* add default_with_oids 89 * tracking */ 90 #define K_VERS_1_10 MAKE_ARCHIVE_VERSION(1, 10, 0) /* add tablespace */ 91 #define K_VERS_1_11 MAKE_ARCHIVE_VERSION(1, 11, 0) /* add toc section 92 * indicator */ 93 #define K_VERS_1_12 MAKE_ARCHIVE_VERSION(1, 12, 0) /* add separate BLOB 94 * entries */ 95 #define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path 96 * behavior */ 97 98 /* Current archive version number (the format we can output) */ 99 #define K_VERS_MAJOR 1 100 #define K_VERS_MINOR 13 101 #define K_VERS_REV 0 102 #define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV) 103 104 /* Newest format we can read */ 105 #define K_VERS_MAX MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, 255) 106 107 108 /* Flags to indicate disposition of offsets stored in files */ 109 #define K_OFFSET_POS_NOT_SET 1 110 #define K_OFFSET_POS_SET 2 111 #define K_OFFSET_NO_DATA 3 112 113 /* 114 * Special exit values from worker children. We reserve 0 for normal 115 * success; 1 and other small values should be interpreted as crashes. 116 */ 117 #define WORKER_OK 0 118 #define WORKER_CREATE_DONE 10 119 #define WORKER_INHIBIT_DATA 11 120 #define WORKER_IGNORED_ERRORS 12 121 122 typedef struct _archiveHandle ArchiveHandle; 123 typedef struct _tocEntry TocEntry; 124 struct ParallelState; 125 126 #define READ_ERROR_EXIT(fd) \ 127 do { \ 128 if (feof(fd)) \ 129 exit_horribly(modulename, \ 130 "could not read from input file: end of file\n"); \ 131 else \ 132 exit_horribly(modulename, \ 133 "could not read from input file: %s\n", strerror(errno)); \ 134 } while (0) 135 136 #define WRITE_ERROR_EXIT \ 137 do { \ 138 exit_horribly(modulename, "could not write to output file: %s\n", \ 139 strerror(errno)); \ 140 } while (0) 141 142 typedef enum T_Action 143 { 144 ACT_DUMP, 145 ACT_RESTORE 146 } T_Action; 147 148 typedef void (*ClosePtrType) (ArchiveHandle *AH); 149 typedef void (*ReopenPtrType) (ArchiveHandle *AH); 150 typedef void (*ArchiveEntryPtrType) (ArchiveHandle *AH, TocEntry *te); 151 152 typedef void (*StartDataPtrType) (ArchiveHandle *AH, TocEntry *te); 153 typedef void (*WriteDataPtrType) (ArchiveHandle *AH, const void *data, size_t dLen); 154 typedef void (*EndDataPtrType) (ArchiveHandle *AH, TocEntry *te); 155 156 typedef void (*StartBlobsPtrType) (ArchiveHandle *AH, TocEntry *te); 157 typedef void (*StartBlobPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); 158 typedef void (*EndBlobPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); 159 typedef void (*EndBlobsPtrType) (ArchiveHandle *AH, TocEntry *te); 160 161 typedef int (*WriteBytePtrType) (ArchiveHandle *AH, const int i); 162 typedef int (*ReadBytePtrType) (ArchiveHandle *AH); 163 typedef void (*WriteBufPtrType) (ArchiveHandle *AH, const void *c, size_t len); 164 typedef void (*ReadBufPtrType) (ArchiveHandle *AH, void *buf, size_t len); 165 typedef void (*SaveArchivePtrType) (ArchiveHandle *AH); 166 typedef void (*WriteExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 167 typedef void (*ReadExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 168 typedef void (*PrintExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); 169 typedef void (*PrintTocDataPtrType) (ArchiveHandle *AH, TocEntry *te); 170 171 typedef void (*ClonePtrType) (ArchiveHandle *AH); 172 typedef void (*DeClonePtrType) (ArchiveHandle *AH); 173 174 typedef int (*WorkerJobDumpPtrType) (ArchiveHandle *AH, TocEntry *te); 175 typedef int (*WorkerJobRestorePtrType) (ArchiveHandle *AH, TocEntry *te); 176 177 typedef size_t (*CustomOutPtrType) (ArchiveHandle *AH, const void *buf, size_t len); 178 179 typedef enum 180 { 181 SQL_SCAN = 0, /* normal */ 182 SQL_IN_SINGLE_QUOTE, /* '...' literal */ 183 SQL_IN_DOUBLE_QUOTE /* "..." identifier */ 184 } sqlparseState; 185 186 typedef struct 187 { 188 sqlparseState state; /* see above */ 189 bool backSlash; /* next char is backslash quoted? */ 190 PQExpBuffer curCmd; /* incomplete line (NULL if not created) */ 191 } sqlparseInfo; 192 193 typedef enum 194 { 195 STAGE_NONE = 0, 196 STAGE_INITIALIZING, 197 STAGE_PROCESSING, 198 STAGE_FINALIZING 199 } ArchiverStage; 200 201 typedef enum 202 { 203 OUTPUT_SQLCMDS = 0, /* emitting general SQL commands */ 204 OUTPUT_COPYDATA, /* writing COPY data */ 205 OUTPUT_OTHERDATA /* writing data as INSERT commands */ 206 } ArchiverOutput; 207 208 /* 209 * For historical reasons, ACL items are interspersed with everything else in 210 * a dump file's TOC; typically they're right after the object they're for. 211 * However, we need to restore data before ACLs, as otherwise a read-only 212 * table (ie one where the owner has revoked her own INSERT privilege) causes 213 * data restore failures. On the other hand, matview REFRESH commands should 214 * come out after ACLs, as otherwise non-superuser-owned matviews might not 215 * be able to execute. (If the permissions at the time of dumping would not 216 * allow a REFRESH, too bad; we won't fix that for you.) We also want event 217 * triggers to be restored after ACLs, so that they can't mess those up. 218 * 219 * These considerations force us to make three passes over the TOC, 220 * restoring the appropriate subset of items in each pass. We assume that 221 * the dependency sort resulted in an appropriate ordering of items within 222 * each subset. 223 * 224 * XXX This mechanism should be superseded by tracking dependencies on ACLs 225 * properly; but we'll still need it for old dump files even after that. 226 */ 227 typedef enum 228 { 229 RESTORE_PASS_MAIN = 0, /* Main pass (most TOC item types) */ 230 RESTORE_PASS_ACL, /* ACL item types */ 231 RESTORE_PASS_POST_ACL /* Event trigger and matview refresh items */ 232 233 #define RESTORE_PASS_LAST RESTORE_PASS_POST_ACL 234 } RestorePass; 235 236 typedef enum 237 { 238 REQ_SCHEMA = 0x01, /* want schema */ 239 REQ_DATA = 0x02, /* want data */ 240 REQ_SPECIAL = 0x04 /* for special TOC entries */ 241 } teReqs; 242 243 struct _archiveHandle 244 { 245 Archive public; /* Public part of archive */ 246 int version; /* Version of file */ 247 248 char *archiveRemoteVersion; /* When reading an archive, the 249 * version of the dumped DB */ 250 char *archiveDumpVersion; /* When reading an archive, the version of 251 * the dumper */ 252 253 int debugLevel; /* Used for logging (currently only by 254 * --verbose) */ 255 size_t intSize; /* Size of an integer in the archive */ 256 size_t offSize; /* Size of a file offset in the archive - 257 * Added V1.7 */ 258 ArchiveFormat format; /* Archive format */ 259 260 sqlparseInfo sqlparse; /* state for parsing INSERT data */ 261 262 time_t createDate; /* Date archive created */ 263 264 /* 265 * Fields used when discovering archive format. For tar format, we load 266 * the first block into the lookahead buffer, and verify that it looks 267 * like a tar header. The tar module must then consume bytes from the 268 * lookahead buffer before reading any more from the file. For custom 269 * format, we load only the "PGDMP" marker into the buffer, and then set 270 * readHeader after confirming it matches. The buffer is vestigial in 271 * this case, as the subsequent code just checks readHeader and doesn't 272 * examine the buffer. 273 */ 274 int readHeader; /* Set if we already read "PGDMP" marker */ 275 char *lookahead; /* Buffer used when reading header to discover 276 * format */ 277 size_t lookaheadSize; /* Allocated size of buffer */ 278 size_t lookaheadLen; /* Length of valid data in lookahead */ 279 size_t lookaheadPos; /* Current read position in lookahead buffer */ 280 281 ArchiveEntryPtrType ArchiveEntryPtr; /* Called for each metadata object */ 282 StartDataPtrType StartDataPtr; /* Called when table data is about to be 283 * dumped */ 284 WriteDataPtrType WriteDataPtr; /* Called to send some table data to the 285 * archive */ 286 EndDataPtrType EndDataPtr; /* Called when table data dump is finished */ 287 WriteBytePtrType WriteBytePtr; /* Write a byte to output */ 288 ReadBytePtrType ReadBytePtr; /* Read a byte from an archive */ 289 WriteBufPtrType WriteBufPtr; /* Write a buffer of output to the archive */ 290 ReadBufPtrType ReadBufPtr; /* Read a buffer of input from the archive */ 291 ClosePtrType ClosePtr; /* Close the archive */ 292 ReopenPtrType ReopenPtr; /* Reopen the archive */ 293 WriteExtraTocPtrType WriteExtraTocPtr; /* Write extra TOC entry data 294 * associated with the current 295 * archive format */ 296 ReadExtraTocPtrType ReadExtraTocPtr; /* Read extra info associated with 297 * archive format */ 298 PrintExtraTocPtrType PrintExtraTocPtr; /* Extra TOC info for format */ 299 PrintTocDataPtrType PrintTocDataPtr; 300 301 StartBlobsPtrType StartBlobsPtr; 302 EndBlobsPtrType EndBlobsPtr; 303 StartBlobPtrType StartBlobPtr; 304 EndBlobPtrType EndBlobPtr; 305 306 SetupWorkerPtrType SetupWorkerPtr; 307 WorkerJobDumpPtrType WorkerJobDumpPtr; 308 WorkerJobRestorePtrType WorkerJobRestorePtr; 309 310 ClonePtrType ClonePtr; /* Clone format-specific fields */ 311 DeClonePtrType DeClonePtr; /* Clean up cloned fields */ 312 313 CustomOutPtrType CustomOutPtr; /* Alternative script output routine */ 314 315 /* Stuff for direct DB connection */ 316 char *archdbname; /* DB name *read* from archive */ 317 char *savedPassword; /* password for ropt->username, if known */ 318 char *use_role; 319 PGconn *connection; 320 /* If connCancel isn't NULL, SIGINT handler will send a cancel */ 321 PGcancel *volatile connCancel; 322 323 int connectToDB; /* Flag to indicate if direct DB connection is 324 * required */ 325 ArchiverOutput outputKind; /* Flag for what we're currently writing */ 326 bool pgCopyIn; /* Currently in libpq 'COPY IN' mode. */ 327 328 int loFd; /* BLOB fd */ 329 int writingBlob; /* Flag */ 330 int blobCount; /* # of blobs restored */ 331 332 char *fSpec; /* Archive File Spec */ 333 FILE *FH; /* General purpose file handle */ 334 void *OF; 335 int gzOut; /* Output file */ 336 337 struct _tocEntry *toc; /* Header of circular list of TOC entries */ 338 int tocCount; /* Number of TOC entries */ 339 DumpId maxDumpId; /* largest DumpId among all TOC entries */ 340 341 /* arrays created after the TOC list is complete: */ 342 struct _tocEntry **tocsByDumpId; /* TOCs indexed by dumpId */ 343 DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ 344 345 struct _tocEntry *currToc; /* Used when dumping data */ 346 int compression; /* Compression requested on open Possible 347 * values for compression: -1 348 * Z_DEFAULT_COMPRESSION 0 COMPRESSION_NONE 349 * 1-9 levels for gzip compression */ 350 bool dosync; /* data requested to be synced on sight */ 351 ArchiveMode mode; /* File mode - r or w */ 352 void *formatData; /* Header data specific to file format */ 353 354 /* these vars track state to avoid sending redundant SET commands */ 355 char *currUser; /* current username, or NULL if unknown */ 356 char *currSchema; /* current schema, or NULL */ 357 char *currTablespace; /* current tablespace, or NULL */ 358 bool currWithOids; /* current default_with_oids setting */ 359 360 void *lo_buf; 361 size_t lo_buf_used; 362 size_t lo_buf_size; 363 364 int noTocComments; 365 ArchiverStage stage; 366 ArchiverStage lastErrorStage; 367 RestorePass restorePass; /* used only during parallel restore */ 368 struct _tocEntry *currentTE; 369 struct _tocEntry *lastErrorTE; 370 }; 371 372 struct _tocEntry 373 { 374 struct _tocEntry *prev; 375 struct _tocEntry *next; 376 CatalogId catalogId; 377 DumpId dumpId; 378 teSection section; 379 bool hadDumper; /* Archiver was passed a dumper routine (used 380 * in restore) */ 381 char *tag; /* index tag */ 382 char *namespace; /* null or empty string if not in a schema */ 383 char *tablespace; /* null if not in a tablespace; empty string 384 * means use database default */ 385 char *owner; 386 bool withOids; /* Used only by "TABLE" tags */ 387 char *desc; 388 char *defn; 389 char *dropStmt; 390 char *copyStmt; 391 DumpId *dependencies; /* dumpIds of objects this one depends on */ 392 int nDeps; /* number of dependencies */ 393 394 DataDumperPtr dataDumper; /* Routine to dump data for object */ 395 void *dataDumperArg; /* Arg for above routine */ 396 void *formatData; /* TOC Entry data specific to file format */ 397 398 /* working state while dumping/restoring */ 399 teReqs reqs; /* do we need schema and/or data of object */ 400 bool created; /* set for DATA member if TABLE was created */ 401 402 /* working state (needed only for parallel restore) */ 403 struct _tocEntry *par_prev; /* list links for pending/ready items; */ 404 struct _tocEntry *par_next; /* these are NULL if not in either list */ 405 int depCount; /* number of dependencies not yet restored */ 406 DumpId *revDeps; /* dumpIds of objects depending on this one */ 407 int nRevDeps; /* number of such dependencies */ 408 DumpId *lockDeps; /* dumpIds of objects this one needs lock on */ 409 int nLockDeps; /* number of such dependencies */ 410 }; 411 412 extern int parallel_restore(ArchiveHandle *AH, TocEntry *te); 413 extern void on_exit_close_archive(Archive *AHX); 414 415 extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt,...) pg_attribute_printf(3, 4); 416 417 extern void WriteTOC(ArchiveHandle *AH); 418 extern void ReadTOC(ArchiveHandle *AH); 419 extern void WriteHead(ArchiveHandle *AH); 420 extern void ReadHead(ArchiveHandle *AH); 421 extern void WriteToc(ArchiveHandle *AH); 422 extern void ReadToc(ArchiveHandle *AH); 423 extern void WriteDataChunks(ArchiveHandle *AH, struct ParallelState *pstate); 424 extern void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te); 425 extern ArchiveHandle *CloneArchive(ArchiveHandle *AH); 426 extern void DeCloneArchive(ArchiveHandle *AH); 427 428 extern teReqs TocIDRequired(ArchiveHandle *AH, DumpId id); 429 TocEntry *getTocEntryByDumpId(ArchiveHandle *AH, DumpId id); 430 extern bool checkSeek(FILE *fp); 431 432 #define appendStringLiteralAHX(buf,str,AH) \ 433 appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings) 434 435 #define appendByteaLiteralAHX(buf,str,len,AH) \ 436 appendByteaLiteral(buf, str, len, (AH)->public.std_strings) 437 438 /* 439 * Mandatory routines for each supported format 440 */ 441 442 extern size_t WriteInt(ArchiveHandle *AH, int i); 443 extern int ReadInt(ArchiveHandle *AH); 444 extern char *ReadStr(ArchiveHandle *AH); 445 extern size_t WriteStr(ArchiveHandle *AH, const char *s); 446 447 int ReadOffset(ArchiveHandle *, pgoff_t *); 448 size_t WriteOffset(ArchiveHandle *, pgoff_t, int); 449 450 extern void StartRestoreBlobs(ArchiveHandle *AH); 451 extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop); 452 extern void EndRestoreBlob(ArchiveHandle *AH, Oid oid); 453 extern void EndRestoreBlobs(ArchiveHandle *AH); 454 455 extern void InitArchiveFmt_Custom(ArchiveHandle *AH); 456 extern void InitArchiveFmt_Null(ArchiveHandle *AH); 457 extern void InitArchiveFmt_Directory(ArchiveHandle *AH); 458 extern void InitArchiveFmt_Tar(ArchiveHandle *AH); 459 460 extern bool isValidTarHeader(char *header); 461 462 extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname); 463 extern void DropBlobIfExists(ArchiveHandle *AH, Oid oid); 464 465 void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH); 466 int ahprintf(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3); 467 468 void ahlog(ArchiveHandle *AH, int level, const char *fmt,...) pg_attribute_printf(3, 4); 469 470 #endif 471