1 /*-------------------------------------------------------------------------
2 *
3 * parsexlog.c
4 * Functions for reading Write-Ahead-Log
5 *
6 * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
8 *
9 *-------------------------------------------------------------------------
10 */
11
12 #include "postgres_fe.h"
13
14 #include <unistd.h>
15
16 #include "pg_rewind.h"
17 #include "filemap.h"
18 #include "logging.h"
19
20 #include "access/rmgr.h"
21 #include "access/xlog_internal.h"
22 #include "access/xlogreader.h"
23 #include "catalog/pg_control.h"
24 #include "catalog/storage_xlog.h"
25 #include "commands/dbcommands_xlog.h"
26
27
28 /*
29 * RmgrNames is an array of resource manager names, to make error messages
30 * a bit nicer.
31 */
32 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
33 name,
34
35 static const char *RmgrNames[RM_MAX_ID + 1] = {
36 #include "access/rmgrlist.h"
37 };
38
39 static void extractPageInfo(XLogReaderState *record);
40
41 static int xlogreadfd = -1;
42 static XLogSegNo xlogreadsegno = -1;
43 static char xlogfpath[MAXPGPATH];
44
45 typedef struct XLogPageReadPrivate
46 {
47 const char *datadir;
48 int tliIndex;
49 } XLogPageReadPrivate;
50
51 static int SimpleXLogPageRead(XLogReaderState *xlogreader,
52 XLogRecPtr targetPagePtr,
53 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
54 TimeLineID *pageTLI);
55
56 /*
57 * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
58 * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
59 * the data blocks touched by the WAL records, and return them in a page map.
60 *
61 * 'endpoint' is the end of the last record to read. The record starting at
62 * 'endpoint' is the first one that is not read.
63 */
64 void
extractPageMap(const char * datadir,XLogRecPtr startpoint,int tliIndex,XLogRecPtr endpoint)65 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
66 XLogRecPtr endpoint)
67 {
68 XLogRecord *record;
69 XLogReaderState *xlogreader;
70 char *errormsg;
71 XLogPageReadPrivate private;
72
73 private.datadir = datadir;
74 private.tliIndex = tliIndex;
75 xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
76 if (xlogreader == NULL)
77 pg_fatal("out of memory\n");
78
79 do
80 {
81 record = XLogReadRecord(xlogreader, startpoint, &errormsg);
82
83 if (record == NULL)
84 {
85 XLogRecPtr errptr;
86
87 errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
88
89 if (errormsg)
90 pg_fatal("could not read WAL record at %X/%X: %s\n",
91 (uint32) (errptr >> 32), (uint32) (errptr),
92 errormsg);
93 else
94 pg_fatal("could not read WAL record at %X/%X\n",
95 (uint32) (errptr >> 32), (uint32) (errptr));
96 }
97
98 extractPageInfo(xlogreader);
99
100 startpoint = InvalidXLogRecPtr; /* continue reading at next record */
101
102 } while (xlogreader->EndRecPtr < endpoint);
103
104 /*
105 * If 'endpoint' didn't point exactly at a record boundary, the caller
106 * messed up.
107 */
108 Assert(xlogreader->EndRecPtr == endpoint);
109
110 XLogReaderFree(xlogreader);
111 if (xlogreadfd != -1)
112 {
113 close(xlogreadfd);
114 xlogreadfd = -1;
115 }
116 }
117
118 /*
119 * Reads one WAL record. Returns the end position of the record, without
120 * doing anything with the record itself.
121 */
122 XLogRecPtr
readOneRecord(const char * datadir,XLogRecPtr ptr,int tliIndex)123 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
124 {
125 XLogRecord *record;
126 XLogReaderState *xlogreader;
127 char *errormsg;
128 XLogPageReadPrivate private;
129 XLogRecPtr endptr;
130
131 private.datadir = datadir;
132 private.tliIndex = tliIndex;
133 xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
134 if (xlogreader == NULL)
135 pg_fatal("out of memory\n");
136
137 record = XLogReadRecord(xlogreader, ptr, &errormsg);
138 if (record == NULL)
139 {
140 if (errormsg)
141 pg_fatal("could not read WAL record at %X/%X: %s\n",
142 (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
143 else
144 pg_fatal("could not read WAL record at %X/%X\n",
145 (uint32) (ptr >> 32), (uint32) (ptr));
146 }
147 endptr = xlogreader->EndRecPtr;
148
149 XLogReaderFree(xlogreader);
150 if (xlogreadfd != -1)
151 {
152 close(xlogreadfd);
153 xlogreadfd = -1;
154 }
155
156 return endptr;
157 }
158
159 /*
160 * Find the previous checkpoint preceding given WAL location.
161 */
162 void
findLastCheckpoint(const char * datadir,XLogRecPtr forkptr,int tliIndex,XLogRecPtr * lastchkptrec,TimeLineID * lastchkpttli,XLogRecPtr * lastchkptredo)163 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
164 XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
165 XLogRecPtr *lastchkptredo)
166 {
167 /* Walk backwards, starting from the given record */
168 XLogRecord *record;
169 XLogRecPtr searchptr;
170 XLogReaderState *xlogreader;
171 char *errormsg;
172 XLogPageReadPrivate private;
173
174 /*
175 * The given fork pointer points to the end of the last common record,
176 * which is not necessarily the beginning of the next record, if the
177 * previous record happens to end at a page boundary. Skip over the page
178 * header in that case to find the next record.
179 */
180 if (forkptr % XLOG_BLCKSZ == 0)
181 forkptr += (forkptr % XLogSegSize == 0) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD;
182
183 private.datadir = datadir;
184 private.tliIndex = tliIndex;
185 xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
186 if (xlogreader == NULL)
187 pg_fatal("out of memory\n");
188
189 searchptr = forkptr;
190 for (;;)
191 {
192 uint8 info;
193
194 record = XLogReadRecord(xlogreader, searchptr, &errormsg);
195
196 if (record == NULL)
197 {
198 if (errormsg)
199 pg_fatal("could not find previous WAL record at %X/%X: %s\n",
200 (uint32) (searchptr >> 32), (uint32) (searchptr),
201 errormsg);
202 else
203 pg_fatal("could not find previous WAL record at %X/%X\n",
204 (uint32) (searchptr >> 32), (uint32) (searchptr));
205 }
206
207 /*
208 * Check if it is a checkpoint record. This checkpoint record needs to
209 * be the latest checkpoint before WAL forked and not the checkpoint
210 * where the master has been stopped to be rewinded.
211 */
212 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
213 if (searchptr < forkptr &&
214 XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
215 (info == XLOG_CHECKPOINT_SHUTDOWN ||
216 info == XLOG_CHECKPOINT_ONLINE))
217 {
218 CheckPoint checkPoint;
219
220 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
221 *lastchkptrec = searchptr;
222 *lastchkpttli = checkPoint.ThisTimeLineID;
223 *lastchkptredo = checkPoint.redo;
224 break;
225 }
226
227 /* Walk backwards to previous record. */
228 searchptr = record->xl_prev;
229 }
230
231 XLogReaderFree(xlogreader);
232 if (xlogreadfd != -1)
233 {
234 close(xlogreadfd);
235 xlogreadfd = -1;
236 }
237 }
238
239 /* XLogreader callback function, to read a WAL page */
240 static int
SimpleXLogPageRead(XLogReaderState * xlogreader,XLogRecPtr targetPagePtr,int reqLen,XLogRecPtr targetRecPtr,char * readBuf,TimeLineID * pageTLI)241 SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
242 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
243 TimeLineID *pageTLI)
244 {
245 XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
246 uint32 targetPageOff;
247 XLogRecPtr targetSegEnd;
248 XLogSegNo targetSegNo;
249
250 XLByteToSeg(targetPagePtr, targetSegNo);
251 XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, targetSegEnd);
252 targetPageOff = targetPagePtr % XLogSegSize;
253
254 /*
255 * See if we need to switch to a new segment because the requested record
256 * is not in the currently open one.
257 */
258 if (xlogreadfd >= 0 && !XLByteInSeg(targetPagePtr, xlogreadsegno))
259 {
260 close(xlogreadfd);
261 xlogreadfd = -1;
262 }
263
264 XLByteToSeg(targetPagePtr, xlogreadsegno);
265
266 if (xlogreadfd < 0)
267 {
268 char xlogfname[MAXFNAMELEN];
269
270 /*
271 * Since incomplete segments are copied into next timelines, switch to
272 * the timeline holding the required segment. Assuming this scan can
273 * be done both forward and backward, consider also switching timeline
274 * accordingly.
275 */
276 while (private->tliIndex < targetNentries - 1 &&
277 targetHistory[private->tliIndex].end < targetSegEnd)
278 private->tliIndex++;
279 while (private->tliIndex > 0 &&
280 targetHistory[private->tliIndex].begin >= targetSegEnd)
281 private->tliIndex--;
282
283 XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, xlogreadsegno);
284
285 snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
286
287 xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
288
289 if (xlogreadfd < 0)
290 {
291 printf(_("could not open file \"%s\": %s\n"), xlogfpath,
292 strerror(errno));
293 return -1;
294 }
295 }
296
297 /*
298 * At this point, we have the right segment open.
299 */
300 Assert(xlogreadfd != -1);
301
302 /* Read the requested page */
303 if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
304 {
305 printf(_("could not seek in file \"%s\": %s\n"), xlogfpath,
306 strerror(errno));
307 return -1;
308 }
309
310 if (read(xlogreadfd, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
311 {
312 printf(_("could not read from file \"%s\": %s\n"), xlogfpath,
313 strerror(errno));
314 return -1;
315 }
316
317 Assert(targetSegNo == xlogreadsegno);
318
319 *pageTLI = targetHistory[private->tliIndex].tli;
320 return XLOG_BLCKSZ;
321 }
322
323 /*
324 * Extract information on which blocks the current record modifies.
325 */
326 static void
extractPageInfo(XLogReaderState * record)327 extractPageInfo(XLogReaderState *record)
328 {
329 int block_id;
330 RmgrId rmid = XLogRecGetRmid(record);
331 uint8 info = XLogRecGetInfo(record);
332 uint8 rminfo = info & ~XLR_INFO_MASK;
333
334 /* Is this a special record type that I recognize? */
335
336 if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
337 {
338 /*
339 * New databases can be safely ignored. It won't be present in the
340 * source system, so it will be deleted. There's one corner-case,
341 * though: if a new, different, database is also created in the source
342 * system, we'll see that the files already exist and not copy them.
343 * That's OK, though; WAL replay of creating the new database, from
344 * the source systems's WAL, will re-copy the new database,
345 * overwriting the database created in the target system.
346 */
347 }
348 else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
349 {
350 /*
351 * An existing database was dropped. We'll see that the files don't
352 * exist in the target data dir, and copy them in toto from the source
353 * system. No need to do anything special here.
354 */
355 }
356 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
357 {
358 /*
359 * We can safely ignore these. The file will be removed from the
360 * target, if it doesn't exist in source system. If a file with same
361 * name is created in source system, too, there will be WAL records
362 * for all the blocks in it.
363 */
364 }
365 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
366 {
367 /*
368 * We can safely ignore these. When we compare the sizes later on,
369 * we'll notice that they differ, and copy the missing tail from
370 * source system.
371 */
372 }
373 else if (info & XLR_SPECIAL_REL_UPDATE)
374 {
375 /*
376 * This record type modifies a relation file in some special way, but
377 * we don't recognize the type. That's bad - we don't know how to
378 * track that change.
379 */
380 pg_fatal("WAL record modifies a relation, but record type is not recognized\n"
381 "lsn: %X/%X, rmgr: %s, info: %02X\n",
382 (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
383 RmgrNames[rmid], info);
384 }
385
386 for (block_id = 0; block_id <= record->max_block_id; block_id++)
387 {
388 RelFileNode rnode;
389 ForkNumber forknum;
390 BlockNumber blkno;
391
392 if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
393 continue;
394
395 /* We only care about the main fork; others are copied in toto */
396 if (forknum != MAIN_FORKNUM)
397 continue;
398
399 process_block_change(forknum, rnode, blkno);
400 }
401 }
402