1 /*-------------------------------------------------------------------------
2 *
3 * parsexlog.c
4 * Functions for reading Write-Ahead-Log
5 *
6 * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
8 *
9 *-------------------------------------------------------------------------
10 */
11
12 #include "postgres_fe.h"
13
14 #include <unistd.h>
15
16 #include "pg_rewind.h"
17 #include "filemap.h"
18
19 #include "access/rmgr.h"
20 #include "access/xlog_internal.h"
21 #include "access/xlogreader.h"
22 #include "catalog/pg_control.h"
23 #include "catalog/storage_xlog.h"
24 #include "commands/dbcommands_xlog.h"
25
26
27 /*
28 * RmgrNames is an array of resource manager names, to make error messages
29 * a bit nicer.
30 */
31 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
32 name,
33
34 static const char *RmgrNames[RM_MAX_ID + 1] = {
35 #include "access/rmgrlist.h"
36 };
37
38 static void extractPageInfo(XLogReaderState *record);
39
40 static int xlogreadfd = -1;
41 static XLogSegNo xlogreadsegno = -1;
42 static char xlogfpath[MAXPGPATH];
43
44 typedef struct XLogPageReadPrivate
45 {
46 const char *datadir;
47 int tliIndex;
48 } XLogPageReadPrivate;
49
50 static int SimpleXLogPageRead(XLogReaderState *xlogreader,
51 XLogRecPtr targetPagePtr,
52 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
53 TimeLineID *pageTLI);
54
55 /*
56 * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
57 * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
58 * the data blocks touched by the WAL records, and return them in a page map.
59 *
60 * 'endpoint' is the end of the last record to read. The record starting at
61 * 'endpoint' is the first one that is not read.
62 */
63 void
extractPageMap(const char * datadir,XLogRecPtr startpoint,int tliIndex,XLogRecPtr endpoint)64 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
65 XLogRecPtr endpoint)
66 {
67 XLogRecord *record;
68 XLogReaderState *xlogreader;
69 char *errormsg;
70 XLogPageReadPrivate private;
71
72 private.datadir = datadir;
73 private.tliIndex = tliIndex;
74 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
75 &private);
76 if (xlogreader == NULL)
77 pg_fatal("out of memory");
78
79 do
80 {
81 record = XLogReadRecord(xlogreader, startpoint, &errormsg);
82
83 if (record == NULL)
84 {
85 XLogRecPtr errptr;
86
87 errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
88
89 if (errormsg)
90 pg_fatal("could not read WAL record at %X/%X: %s",
91 (uint32) (errptr >> 32), (uint32) (errptr),
92 errormsg);
93 else
94 pg_fatal("could not read WAL record at %X/%X",
95 (uint32) (errptr >> 32), (uint32) (errptr));
96 }
97
98 extractPageInfo(xlogreader);
99
100 startpoint = InvalidXLogRecPtr; /* continue reading at next record */
101
102 } while (xlogreader->EndRecPtr < endpoint);
103
104 /*
105 * If 'endpoint' didn't point exactly at a record boundary, the caller
106 * messed up.
107 */
108 Assert(xlogreader->EndRecPtr == endpoint);
109
110 XLogReaderFree(xlogreader);
111 if (xlogreadfd != -1)
112 {
113 close(xlogreadfd);
114 xlogreadfd = -1;
115 }
116 }
117
118 /*
119 * Reads one WAL record. Returns the end position of the record, without
120 * doing anything with the record itself.
121 */
122 XLogRecPtr
readOneRecord(const char * datadir,XLogRecPtr ptr,int tliIndex)123 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
124 {
125 XLogRecord *record;
126 XLogReaderState *xlogreader;
127 char *errormsg;
128 XLogPageReadPrivate private;
129 XLogRecPtr endptr;
130
131 private.datadir = datadir;
132 private.tliIndex = tliIndex;
133 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
134 &private);
135 if (xlogreader == NULL)
136 pg_fatal("out of memory");
137
138 record = XLogReadRecord(xlogreader, ptr, &errormsg);
139 if (record == NULL)
140 {
141 if (errormsg)
142 pg_fatal("could not read WAL record at %X/%X: %s",
143 (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
144 else
145 pg_fatal("could not read WAL record at %X/%X",
146 (uint32) (ptr >> 32), (uint32) (ptr));
147 }
148 endptr = xlogreader->EndRecPtr;
149
150 XLogReaderFree(xlogreader);
151 if (xlogreadfd != -1)
152 {
153 close(xlogreadfd);
154 xlogreadfd = -1;
155 }
156
157 return endptr;
158 }
159
160 /*
161 * Find the previous checkpoint preceding given WAL location.
162 */
163 void
findLastCheckpoint(const char * datadir,XLogRecPtr forkptr,int tliIndex,XLogRecPtr * lastchkptrec,TimeLineID * lastchkpttli,XLogRecPtr * lastchkptredo)164 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
165 XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
166 XLogRecPtr *lastchkptredo)
167 {
168 /* Walk backwards, starting from the given record */
169 XLogRecord *record;
170 XLogRecPtr searchptr;
171 XLogReaderState *xlogreader;
172 char *errormsg;
173 XLogPageReadPrivate private;
174
175 /*
176 * The given fork pointer points to the end of the last common record,
177 * which is not necessarily the beginning of the next record, if the
178 * previous record happens to end at a page boundary. Skip over the page
179 * header in that case to find the next record.
180 */
181 if (forkptr % XLOG_BLCKSZ == 0)
182 {
183 if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
184 forkptr += SizeOfXLogLongPHD;
185 else
186 forkptr += SizeOfXLogShortPHD;
187 }
188
189 private.datadir = datadir;
190 private.tliIndex = tliIndex;
191 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
192 &private);
193 if (xlogreader == NULL)
194 pg_fatal("out of memory");
195
196 searchptr = forkptr;
197 for (;;)
198 {
199 uint8 info;
200
201 record = XLogReadRecord(xlogreader, searchptr, &errormsg);
202
203 if (record == NULL)
204 {
205 if (errormsg)
206 pg_fatal("could not find previous WAL record at %X/%X: %s",
207 (uint32) (searchptr >> 32), (uint32) (searchptr),
208 errormsg);
209 else
210 pg_fatal("could not find previous WAL record at %X/%X",
211 (uint32) (searchptr >> 32), (uint32) (searchptr));
212 }
213
214 /*
215 * Check if it is a checkpoint record. This checkpoint record needs to
216 * be the latest checkpoint before WAL forked and not the checkpoint
217 * where the master has been stopped to be rewinded.
218 */
219 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
220 if (searchptr < forkptr &&
221 XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
222 (info == XLOG_CHECKPOINT_SHUTDOWN ||
223 info == XLOG_CHECKPOINT_ONLINE))
224 {
225 CheckPoint checkPoint;
226
227 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
228 *lastchkptrec = searchptr;
229 *lastchkpttli = checkPoint.ThisTimeLineID;
230 *lastchkptredo = checkPoint.redo;
231 break;
232 }
233
234 /* Walk backwards to previous record. */
235 searchptr = record->xl_prev;
236 }
237
238 XLogReaderFree(xlogreader);
239 if (xlogreadfd != -1)
240 {
241 close(xlogreadfd);
242 xlogreadfd = -1;
243 }
244 }
245
246 /* XLogreader callback function, to read a WAL page */
247 static int
SimpleXLogPageRead(XLogReaderState * xlogreader,XLogRecPtr targetPagePtr,int reqLen,XLogRecPtr targetRecPtr,char * readBuf,TimeLineID * pageTLI)248 SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
249 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
250 TimeLineID *pageTLI)
251 {
252 XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
253 uint32 targetPageOff;
254 XLogRecPtr targetSegEnd;
255 XLogSegNo targetSegNo;
256 int r;
257
258 XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
259 XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
260 targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
261
262 /*
263 * See if we need to switch to a new segment because the requested record
264 * is not in the currently open one.
265 */
266 if (xlogreadfd >= 0 &&
267 !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
268 {
269 close(xlogreadfd);
270 xlogreadfd = -1;
271 }
272
273 XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
274
275 if (xlogreadfd < 0)
276 {
277 char xlogfname[MAXFNAMELEN];
278
279 /*
280 * Since incomplete segments are copied into next timelines, switch to
281 * the timeline holding the required segment. Assuming this scan can
282 * be done both forward and backward, consider also switching timeline
283 * accordingly.
284 */
285 while (private->tliIndex < targetNentries - 1 &&
286 targetHistory[private->tliIndex].end < targetSegEnd)
287 private->tliIndex++;
288 while (private->tliIndex > 0 &&
289 targetHistory[private->tliIndex].begin >= targetSegEnd)
290 private->tliIndex--;
291
292 XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
293 xlogreadsegno, WalSegSz);
294
295 snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
296
297 xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
298
299 if (xlogreadfd < 0)
300 {
301 pg_log_error("could not open file \"%s\": %m", xlogfpath);
302 return -1;
303 }
304 }
305
306 /*
307 * At this point, we have the right segment open.
308 */
309 Assert(xlogreadfd != -1);
310
311 /* Read the requested page */
312 if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
313 {
314 pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
315 return -1;
316 }
317
318
319 r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
320 if (r != XLOG_BLCKSZ)
321 {
322 if (r < 0)
323 pg_log_error("could not read file \"%s\": %m", xlogfpath);
324 else
325 pg_log_error("could not read file \"%s\": read %d of %zu",
326 xlogfpath, r, (Size) XLOG_BLCKSZ);
327
328 return -1;
329 }
330
331 Assert(targetSegNo == xlogreadsegno);
332
333 *pageTLI = targetHistory[private->tliIndex].tli;
334 return XLOG_BLCKSZ;
335 }
336
337 /*
338 * Extract information on which blocks the current record modifies.
339 */
340 static void
extractPageInfo(XLogReaderState * record)341 extractPageInfo(XLogReaderState *record)
342 {
343 int block_id;
344 RmgrId rmid = XLogRecGetRmid(record);
345 uint8 info = XLogRecGetInfo(record);
346 uint8 rminfo = info & ~XLR_INFO_MASK;
347
348 /* Is this a special record type that I recognize? */
349
350 if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
351 {
352 /*
353 * New databases can be safely ignored. It won't be present in the
354 * source system, so it will be deleted. There's one corner-case,
355 * though: if a new, different, database is also created in the source
356 * system, we'll see that the files already exist and not copy them.
357 * That's OK, though; WAL replay of creating the new database, from
358 * the source systems's WAL, will re-copy the new database,
359 * overwriting the database created in the target system.
360 */
361 }
362 else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
363 {
364 /*
365 * An existing database was dropped. We'll see that the files don't
366 * exist in the target data dir, and copy them in toto from the source
367 * system. No need to do anything special here.
368 */
369 }
370 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
371 {
372 /*
373 * We can safely ignore these. The file will be removed from the
374 * target, if it doesn't exist in source system. If a file with same
375 * name is created in source system, too, there will be WAL records
376 * for all the blocks in it.
377 */
378 }
379 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
380 {
381 /*
382 * We can safely ignore these. When we compare the sizes later on,
383 * we'll notice that they differ, and copy the missing tail from
384 * source system.
385 */
386 }
387 else if (info & XLR_SPECIAL_REL_UPDATE)
388 {
389 /*
390 * This record type modifies a relation file in some special way, but
391 * we don't recognize the type. That's bad - we don't know how to
392 * track that change.
393 */
394 pg_fatal("WAL record modifies a relation, but record type is not recognized: "
395 "lsn: %X/%X, rmgr: %s, info: %02X",
396 (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
397 RmgrNames[rmid], info);
398 }
399
400 for (block_id = 0; block_id <= record->max_block_id; block_id++)
401 {
402 RelFileNode rnode;
403 ForkNumber forknum;
404 BlockNumber blkno;
405
406 if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
407 continue;
408
409 /* We only care about the main fork; others are copied in toto */
410 if (forknum != MAIN_FORKNUM)
411 continue;
412
413 process_block_change(forknum, rnode, blkno);
414 }
415 }
416