1 #include "../../../burp.h"
2 #include "../../../alloc.h"
3 #include "../../../fsops.h"
4 #include "../../../hexmap.h"
5 #include "../../../lock.h"
6 #include "../../../log.h"
7 #include "../../../prepend.h"
8 #include "../../../protocol2/blk.h"
9 #include "../../../sbuf.h"
10 #include "../../../strlist.h"
11 #include "../../sdirs.h"
12 #include "../backup_phase4.h"
13 #include "dindex.h"
14
backup_in_progress(const char * fullpath)15 static int backup_in_progress(const char *fullpath)
16 {
17 int ret=-1;
18 struct stat statp;
19 char *working=NULL;
20 char *finishing=NULL;
21 char *dfiles_regenerating=NULL;
22
23 if(!(working=prepend_s(fullpath, "working"))
24 || !(finishing=prepend_s(fullpath, "finishing"))
25 || !(dfiles_regenerating=prepend_s(fullpath, "dfiles.regenerating")))
26 goto end;
27
28 if(!lstat(working, &statp)
29 || !lstat(finishing, &statp))
30 {
31 logp("%s looks like it has a backup in progress.\n",
32 fullpath);
33 ret=1;
34 goto end;
35 }
36 if(!lstat(dfiles_regenerating, &statp))
37 {
38 logp("%s looks like it was interrupted whilst "
39 "regenerating its dfiles.\n", fullpath);
40 ret=1;
41 goto end;
42 }
43 ret=0;
44 end:
45 if(ret==1)
46 logp("Give up clean up attempt.\n");
47 free_w(&working);
48 free_w(&finishing);
49 free_w(&dfiles_regenerating);
50 return ret;
51 }
52
53 // Returns 0 on OK, -1 on error, 1 if there were backups already in progress.
get_dfiles_to_merge(struct sdirs * sdirs,struct strlist ** s)54 static int get_dfiles_to_merge(struct sdirs *sdirs, struct strlist **s)
55 {
56 int i=0;
57 int n=0;
58 int ret=-1;
59 struct stat statp;
60 char *fullpath=NULL;
61 char *dfiles=NULL;
62 struct dirent **dir=NULL;
63
64 if((n=scandir(sdirs->clients, &dir, filter_dot, NULL))<0)
65 {
66 logp("scandir failed for %s in %s: %s\n",
67 sdirs->clients, __func__, strerror(errno));
68 goto end;
69 }
70 for(i=0; i<n; i++)
71 {
72 free_w(&fullpath);
73 if(!(fullpath=prepend_s(sdirs->clients, dir[i]->d_name)))
74 goto end;
75 switch(is_dir(fullpath, dir[i]))
76 {
77 case 0: continue;
78 case 1: break;
79 default: logp("is_dir(%s): %s\n",
80 fullpath, strerror(errno));
81 goto end;
82 }
83
84 if(strcmp(sdirs->client, fullpath))
85 {
86 switch(backup_in_progress(fullpath))
87 {
88 case 0: break;
89 case 1: ret=1;
90 default: goto end;
91 }
92 }
93
94 free_w(&dfiles);
95 if(!(dfiles=prepend_s(fullpath, "dfiles"))
96 || lstat(dfiles, &statp))
97 continue;
98
99 // Have a good entry. Add it to the list.
100 if(strlist_add(s, dfiles, 0))
101 goto end;
102 }
103
104 ret=0;
105 end:
106 free_w(&fullpath);
107 free_w(&dfiles);
108 if(dir)
109 {
110 for(i=0; i<n; i++)
111 free(dir[i]);
112 free(dir);
113 }
114 return ret;
115 }
116
do_unlink(struct blk * oblk,const char * datadir)117 static int do_unlink(struct blk *oblk, const char *datadir)
118 {
119 int ret=-1;
120 char *fullpath=NULL;
121 char *savepath=uint64_to_savepathstr(oblk->savepath);
122 if(!(fullpath=prepend_s(datadir, savepath)))
123 goto end;
124 errno=0;
125 if(unlink(fullpath) && errno!=ENOENT)
126 {
127 logp("Could not unlink %s: %s\n", fullpath, strerror(errno));
128 goto end;
129 }
130 logp("Deleted %s\n", savepath);
131 ret=0;
132 end:
133 free_w(&fullpath);
134 return ret;
135 }
136
137 #ifndef UTEST
138 static
139 #endif
compare_dindexes_and_unlink_datafiles(const char * dindex_old,const char * dindex_new,const char * datadir)140 int compare_dindexes_and_unlink_datafiles(const char *dindex_old,
141 const char *dindex_new, const char *datadir)
142 {
143 int ret=-1;
144 struct fzp *nzp=NULL;
145 struct fzp *ozp=NULL;
146 struct iobuf nbuf;
147 struct iobuf obuf;
148 struct blk nblk;
149 struct blk oblk;
150
151 iobuf_init(&nbuf);
152 iobuf_init(&obuf);
153 memset(&nblk, 0, sizeof(struct blk));
154 memset(&oblk, 0, sizeof(struct blk));
155
156 if(!(nzp=fzp_gzopen(dindex_new, "rb"))
157 || !(ozp=fzp_gzopen(dindex_old, "rb")))
158 goto end;
159
160 while(nzp || ozp)
161 {
162 if(nzp
163 && !nbuf.buf)
164 {
165 switch(iobuf_fill_from_fzp(&nbuf, nzp))
166 {
167 case 1: fzp_close(&nzp);
168 break;
169 case 0: if(nbuf.cmd!=CMD_SAVE_PATH)
170 {
171 logp("unknown cmd in %s: %s\n",
172 __func__,
173 iobuf_to_printable(&nbuf));
174 goto end;
175 }
176 if(blk_set_from_iobuf_savepath(&nblk,
177 &nbuf)) goto end;
178 break;
179 default: goto end; // Error;
180 }
181 }
182
183 if(ozp
184 && !obuf.buf)
185 {
186 switch(iobuf_fill_from_fzp(&obuf, ozp))
187 {
188 case 1: fzp_close(&ozp);
189 break;
190 case 0: if(obuf.cmd!=CMD_SAVE_PATH)
191 {
192 logp("unknown cmd in %s: %c\n",
193 __func__, obuf.cmd);
194 goto end;
195 }
196 if(blk_set_from_iobuf_savepath(&oblk,
197 &obuf)) goto end;
198 break;
199 default: goto end; // Error;
200 }
201 }
202
203 if(nbuf.buf && !obuf.buf)
204 {
205 // No more from the old file. Time to stop.
206 break;
207 }
208 else if(!nbuf.buf && obuf.buf)
209 {
210 // No more in the new file. Delete old entry.
211 if(do_unlink(&oblk, datadir))
212 goto end;
213 iobuf_free_content(&obuf);
214 }
215 else if(!nbuf.buf && !obuf.buf)
216 {
217 continue;
218 }
219 else if(nblk.savepath==oblk.savepath)
220 {
221 // Same, free both and continue;
222 iobuf_free_content(&nbuf);
223 iobuf_free_content(&obuf);
224 }
225 else if(nblk.savepath<oblk.savepath)
226 {
227 // Only in the new file.
228 iobuf_free_content(&nbuf);
229 }
230 else
231 {
232 // Only in the old file.
233 if(do_unlink(&oblk, datadir))
234 goto end;
235 iobuf_free_content(&obuf);
236 }
237 }
238
239
240 ret=0;
241 end:
242 iobuf_free_content(&nbuf);
243 iobuf_free_content(&obuf);
244 fzp_close(&nzp);
245 fzp_close(&ozp);
246 return ret;
247 }
248
delete_unused_data_files(struct sdirs * sdirs,int resume)249 int delete_unused_data_files(struct sdirs *sdirs, int resume)
250 {
251 int ret=-1;
252 uint64_t fcount=0;
253 char hfile[32];
254 char *hlinks=NULL;
255 char *fullpath=NULL;
256 char *cindex_tmp=NULL;
257 char *cindex_new=NULL;
258 char *dindex_tmp=NULL;
259 char *dindex_new=NULL;
260 char *dindex_old=NULL;
261 struct strlist *s=NULL;
262 struct strlist *slist=NULL;
263 struct stat statp;
264 struct lock *lock=NULL;
265
266 if(!sdirs)
267 {
268 logp("No sdirs passed to %s\n", __func__);
269 goto end;
270 }
271
272 if(resume)
273 {
274 // Cannot do it on a resume, or it will delete files that are
275 // referenced in the backup we are resuming.
276 logp("Not attempting to clean up unused data files\n");
277 logp("because %s is resuming\n", sdirs->clients);
278 ret=0;
279 goto end;
280 }
281
282 if(!(lock=lock_alloc_and_init(sdirs->champ_dindex_lock)))
283 goto end;
284 lock_get(lock);
285 switch(lock->status)
286 {
287 case GET_LOCK_GOT:
288 break;
289 default:
290 logp("Could not get %s\n", sdirs->champ_dindex_lock);
291 logp("This should not happen.\n");
292 goto end;
293 }
294
295 logp("Attempting to clean up unused data files %s\n", sdirs->clients);
296
297 // Get all lists of files in all backups.
298 switch(get_dfiles_to_merge(sdirs, &slist))
299 {
300 case 0:
301 break; // OK.
302 case 1:
303 // Backups are in progress, cannot continue.
304 // But do not return an error.
305 ret=0;
306 default:
307 goto end; // Error.
308 }
309
310 if(!(dindex_tmp=prepend_s(sdirs->data, "dindex.tmp"))
311 || !(dindex_old=prepend_s(sdirs->data, "dindex")))
312 goto end;
313
314 // Get a list of the files that have been created since last time.
315 // (this enables us to clean up data files that were created for
316 // interrupted backups).
317 if(!(cindex_tmp=prepend_s(sdirs->cfiles, "cindex.tmp"))
318 || recursive_delete(cindex_tmp))
319 goto end;
320 if(!lstat(sdirs->cfiles, &statp))
321 {
322 if(mkdir(cindex_tmp, 0777)
323 || !(cindex_new=prepend_s(cindex_tmp, "cindex"))
324 || merge_files_in_dir_no_fcount(cindex_new,
325 sdirs->cfiles, merge_dindexes))
326 goto end;
327 if(!lstat(cindex_new, &statp))
328 {
329 if(lstat(dindex_old, &statp))
330 {
331 // The dindex file does not exist.
332 // Just rename cindex_new.
333 if(do_rename(cindex_new, dindex_old))
334 goto end;
335 }
336 else
337 {
338 // Merge it into the previous list of files
339 // from all backups.
340 if(merge_dindexes(dindex_tmp,
341 dindex_old, cindex_new)
342 || do_rename(dindex_tmp, dindex_old))
343 goto end;
344 }
345 }
346 }
347
348 // Create a directory of hardlinks to each list of files.
349 if(!(hlinks=prepend_s(dindex_tmp, "hlinks"))
350 || recursive_delete(dindex_tmp)
351 || mkdir(dindex_tmp, 0777)
352 || mkdir(hlinks, 0777))
353 goto end;
354 for(s=slist; s; s=s->next)
355 {
356 snprintf(hfile, sizeof(hfile), "%08" PRIX64, fcount++);
357 free_w(&fullpath);
358 if(!(fullpath=prepend_s(hlinks, hfile)))
359 goto end;
360 if(link(s->path, fullpath))
361 {
362 logp("Could not hardlink %s to %s: %s\n",
363 fullpath, s->path, strerror(errno));
364 goto end;
365 }
366 }
367
368 // Create a single list of files in all backups.
369 if(!(dindex_new=prepend_s(dindex_tmp, "dindex")))
370 goto end;
371 if(merge_files_in_dir(dindex_new,
372 dindex_tmp, "hlinks", fcount, merge_dindexes))
373 goto end;
374
375 if(!lstat(dindex_new, &statp))
376 {
377 if(!lstat(dindex_old, &statp)
378 && compare_dindexes_and_unlink_datafiles(dindex_old,
379 dindex_new, sdirs->data))
380 goto end;
381 if(do_rename(dindex_new, dindex_old))
382 goto end;
383
384 // No longer need the current cfiles directory.
385 if(recursive_delete(sdirs->cfiles))
386 goto end;
387 }
388
389 ret=0;
390 end:
391 strlists_free(&slist);
392 if(cindex_tmp) recursive_delete(cindex_tmp);
393 if(dindex_tmp) recursive_delete(dindex_tmp);
394 lock_release(lock);
395 lock_free(&lock);
396 free_w(&fullpath);
397 free_w(&hlinks);
398 free_w(&cindex_tmp);
399 free_w(&cindex_new);
400 free_w(&dindex_tmp);
401 free_w(&dindex_new);
402 free_w(&dindex_old);
403 return ret;
404 }
405