1 /*-------------------------------------------------------------------------
2 *
3 * copy_fetch.c
4 * Functions for using a data directory as the source.
5 *
6 * Portions Copyright (c) 2013-2017, PostgreSQL Global Development Group
7 *
8 *-------------------------------------------------------------------------
9 */
10 #include "postgres_fe.h"
11
12 #include <sys/stat.h>
13 #include <dirent.h>
14 #include <fcntl.h>
15 #include <unistd.h>
16
17 #include "datapagemap.h"
18 #include "fetch.h"
19 #include "file_ops.h"
20 #include "filemap.h"
21 #include "logging.h"
22 #include "pg_rewind.h"
23
24 #include "catalog/catalog.h"
25
26 static void recurse_dir(const char *datadir, const char *path,
27 process_file_callback_t callback);
28
29 static void execute_pagemap(datapagemap_t *pagemap, const char *path);
30
31 /*
32 * Traverse through all files in a data directory, calling 'callback'
33 * for each file.
34 */
35 void
traverse_datadir(const char * datadir,process_file_callback_t callback)36 traverse_datadir(const char *datadir, process_file_callback_t callback)
37 {
38 recurse_dir(datadir, NULL, callback);
39 }
40
41 /*
42 * recursive part of traverse_datadir
43 *
44 * parentpath is the current subdirectory's path relative to datadir,
45 * or NULL at the top level.
46 */
47 static void
recurse_dir(const char * datadir,const char * parentpath,process_file_callback_t callback)48 recurse_dir(const char *datadir, const char *parentpath,
49 process_file_callback_t callback)
50 {
51 DIR *xldir;
52 struct dirent *xlde;
53 char fullparentpath[MAXPGPATH];
54
55 if (parentpath)
56 snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
57 else
58 snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
59
60 xldir = opendir(fullparentpath);
61 if (xldir == NULL)
62 pg_fatal("could not open directory \"%s\": %s\n",
63 fullparentpath, strerror(errno));
64
65 while (errno = 0, (xlde = readdir(xldir)) != NULL)
66 {
67 struct stat fst;
68 char fullpath[MAXPGPATH * 2];
69 char path[MAXPGPATH * 2];
70
71 if (strcmp(xlde->d_name, ".") == 0 ||
72 strcmp(xlde->d_name, "..") == 0)
73 continue;
74
75 snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
76
77 if (lstat(fullpath, &fst) < 0)
78 {
79 if (errno == ENOENT)
80 {
81 /*
82 * File doesn't exist anymore. This is ok, if the new master
83 * is running and the file was just removed. If it was a data
84 * file, there should be a WAL record of the removal. If it
85 * was something else, it couldn't have been anyway.
86 *
87 * TODO: But complain if we're processing the target dir!
88 */
89 }
90 else
91 pg_fatal("could not stat file \"%s\": %s\n",
92 fullpath, strerror(errno));
93 }
94
95 if (parentpath)
96 snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
97 else
98 snprintf(path, sizeof(path), "%s", xlde->d_name);
99
100 if (S_ISREG(fst.st_mode))
101 callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
102 else if (S_ISDIR(fst.st_mode))
103 {
104 callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
105 /* recurse to handle subdirectories */
106 recurse_dir(datadir, path, callback);
107 }
108 #ifndef WIN32
109 else if (S_ISLNK(fst.st_mode))
110 #else
111 else if (pgwin32_is_junction(fullpath))
112 #endif
113 {
114 #if defined(HAVE_READLINK) || defined(WIN32)
115 char link_target[MAXPGPATH];
116 int len;
117
118 len = readlink(fullpath, link_target, sizeof(link_target));
119 if (len < 0)
120 pg_fatal("could not read symbolic link \"%s\": %s\n",
121 fullpath, strerror(errno));
122 if (len >= sizeof(link_target))
123 pg_fatal("symbolic link \"%s\" target is too long\n",
124 fullpath);
125 link_target[len] = '\0';
126
127 callback(path, FILE_TYPE_SYMLINK, 0, link_target);
128
129 /*
130 * If it's a symlink within pg_tblspc, we need to recurse into it,
131 * to process all the tablespaces. We also follow a symlink if
132 * it's for pg_wal. Symlinks elsewhere are ignored.
133 */
134 if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
135 strcmp(path, "pg_wal") == 0)
136 recurse_dir(datadir, path, callback);
137 #else
138 pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
139 fullpath);
140 #endif /* HAVE_READLINK */
141 }
142 }
143
144 if (errno)
145 pg_fatal("could not read directory \"%s\": %s\n",
146 fullparentpath, strerror(errno));
147
148 if (closedir(xldir))
149 pg_fatal("could not close directory \"%s\": %s\n",
150 fullparentpath, strerror(errno));
151 }
152
153 /*
154 * Copy a file from source to target, between 'begin' and 'end' offsets.
155 *
156 * If 'trunc' is true, any existing file with the same name is truncated.
157 */
158 static void
rewind_copy_file_range(const char * path,off_t begin,off_t end,bool trunc)159 rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
160 {
161 PGAlignedBlock buf;
162 char srcpath[MAXPGPATH];
163 int srcfd;
164
165 snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
166
167 srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
168 if (srcfd < 0)
169 pg_fatal("could not open source file \"%s\": %s\n",
170 srcpath, strerror(errno));
171
172 if (lseek(srcfd, begin, SEEK_SET) == -1)
173 pg_fatal("could not seek in source file: %s\n", strerror(errno));
174
175 open_target_file(path, trunc);
176
177 while (end - begin > 0)
178 {
179 int readlen;
180 int len;
181
182 if (end - begin > sizeof(buf))
183 len = sizeof(buf);
184 else
185 len = end - begin;
186
187 readlen = read(srcfd, buf.data, len);
188
189 if (readlen < 0)
190 pg_fatal("could not read file \"%s\": %s\n",
191 srcpath, strerror(errno));
192 else if (readlen == 0)
193 pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath);
194
195 write_target_range(buf.data, begin, readlen);
196 begin += readlen;
197 }
198
199 if (close(srcfd) != 0)
200 pg_fatal("could not close file \"%s\": %s\n", srcpath, strerror(errno));
201 }
202
203 /*
204 * Copy all relation data files from datadir_source to datadir_target, which
205 * are marked in the given data page map.
206 */
207 void
copy_executeFileMap(filemap_t * map)208 copy_executeFileMap(filemap_t *map)
209 {
210 file_entry_t *entry;
211 int i;
212
213 for (i = 0; i < map->narray; i++)
214 {
215 entry = map->array[i];
216 execute_pagemap(&entry->pagemap, entry->path);
217
218 switch (entry->action)
219 {
220 case FILE_ACTION_NONE:
221 /* ok, do nothing.. */
222 break;
223
224 case FILE_ACTION_COPY:
225 rewind_copy_file_range(entry->path, 0, entry->newsize, true);
226 break;
227
228 case FILE_ACTION_TRUNCATE:
229 truncate_target_file(entry->path, entry->newsize);
230 break;
231
232 case FILE_ACTION_COPY_TAIL:
233 rewind_copy_file_range(entry->path, entry->oldsize,
234 entry->newsize, false);
235 break;
236
237 case FILE_ACTION_CREATE:
238 create_target(entry);
239 break;
240
241 case FILE_ACTION_REMOVE:
242 remove_target(entry);
243 break;
244 }
245 }
246
247 close_target_file();
248 }
249
250 static void
execute_pagemap(datapagemap_t * pagemap,const char * path)251 execute_pagemap(datapagemap_t *pagemap, const char *path)
252 {
253 datapagemap_iterator_t *iter;
254 BlockNumber blkno;
255 off_t offset;
256
257 iter = datapagemap_iterate(pagemap);
258 while (datapagemap_next(iter, &blkno))
259 {
260 offset = blkno * BLCKSZ;
261 rewind_copy_file_range(path, offset, offset + BLCKSZ, false);
262 /* Ok, this block has now been copied from new data dir to old */
263 }
264 pg_free(iter);
265 }
266