xref: /dragonfly/sbin/hammer/cmd_softprune.c (revision 25a2db75) 
1 /*
2  * Copyright (c) 2008 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * $DragonFly: src/sbin/hammer/cmd_softprune.c,v 1.7 2008/08/21 23:28:43 thomas Exp $
35  */
36 
37 #include "hammer.h"
38 
39 struct softprune {
40 	struct softprune *next;
41 	struct statfs fs;
42 	char *filesystem;
43 	struct hammer_ioc_prune prune;
44 	int maxelms;
45 	int prune_min;
46 };
47 
48 static void softprune_usage(int code);
49 static void hammer_softprune_scandir(struct softprune **basep,
50 			struct hammer_ioc_prune *template,
51 			const char *dirname);
52 static int hammer_softprune_scanmeta(int fd, struct softprune *scan,
53 			int delete_all);
54 static void hammer_meta_flushdelete(int fd, struct hammer_ioc_snapshot *dsnap);
55 static struct softprune *hammer_softprune_addentry(struct softprune **basep,
56 			struct hammer_ioc_prune *template,
57 			const char *dirpath, const char *denname,
58 			struct stat *st,
59 			const char *linkbuf, const char *tidptr);
60 static void hammer_softprune_addelm(struct softprune *scan, hammer_tid_t tid,
61 			time_t ct, time_t mt);
62 static void hammer_softprune_finalize(struct softprune *scan);
63 
64 /*
65  * prune <softlink-dir>
66  * prune-everything <filesystem>
67  */
68 void
69 hammer_cmd_softprune(char **av, int ac, int everything_opt)
70 {
71 	struct hammer_ioc_prune template;
72 	struct hammer_ioc_pseudofs_rw pfs;
73 	struct softprune *base, *scan;
74 	int fd;
75 	int rcode;
76 
77 	base = NULL;
78 	rcode = 0;
79 	if (TimeoutOpt > 0)
80 		alarm(TimeoutOpt);
81 
82 	bzero(&pfs, sizeof(pfs));
83 	pfs.bytes = sizeof(*pfs.ondisk);
84 	pfs.ondisk = malloc(pfs.bytes);
85 	bzero(pfs.ondisk, pfs.bytes);
86 	pfs.pfs_id = -1;
87 
88 	/*
89 	 * NOTE: To restrict to a single file XXX we have to set
90 	 * the localization the same (not yet implemented).  Typically
91 	 * two passes would be needed, one using HAMMER_LOCALIZE_MISC
92 	 * and one using HAMMER_LOCALIZE_INODE.
93 	 */
94 
95 	bzero(&template, sizeof(template));
96 	template.key_beg.localization = HAMMER_MIN_LOCALIZATION;
97 	template.key_beg.obj_id = HAMMER_MIN_OBJID;
98 	template.key_end.localization = HAMMER_MAX_LOCALIZATION;
99 	template.key_end.obj_id = HAMMER_MAX_OBJID;
100 	hammer_get_cycle(&template.key_end, NULL);
101 	template.stat_oldest_tid = HAMMER_MAX_TID;
102 
103 	/*
104 	 * For now just allow one directory
105 	 */
106 	if (ac == 0 || ac > 1)
107 		softprune_usage(1);
108 
109 	/*
110 	 * Scan the softlink directory.
111 	 */
112 	if (everything_opt) {
113 		const char *dummylink = "";
114 		scan = hammer_softprune_addentry(&base, &template,
115 						 *av, NULL, NULL,
116 						 dummylink, dummylink);
117 		if (scan == NULL)
118 			softprune_usage(1);
119 		scan->prune.nelms = 0;
120 		scan->prune.head.flags |= HAMMER_IOC_PRUNE_ALL;
121 	} else {
122 		hammer_softprune_scandir(&base, &template, *av);
123 		if (base == NULL) {
124 			const char *dummylink = "";
125 			scan = hammer_softprune_addentry(&base, &template,
126 							 *av, NULL, NULL,
127 							 dummylink, dummylink);
128 			if (scan == NULL)
129 				softprune_usage(1);
130 			scan->prune.nelms = 0;
131 		}
132 		++av;
133 		--ac;
134 	}
135 
136 	/*
137 	 * XXX future (need to store separate cycles for each filesystem)
138 	 */
139 	if (base->next) {
140 		fprintf(stderr, "Currently only one HAMMER filesystem may "
141 				"be specified in the softlink scan\n");
142 		exit(1);
143 	}
144 
145 	/*
146 	 * Issue the prunes
147 	 */
148 	for (scan = base; scan; scan = scan->next) {
149 		/*
150 		 * Open the filesystem for ioctl calls and extract the
151 		 * PFS.
152 		 */
153 		fd = open(scan->filesystem, O_RDONLY);
154 		if (fd < 0) {
155 			warn("Unable to open %s", scan->filesystem);
156 			rcode = 1;
157 			continue;
158 		}
159 
160 		if (ioctl(fd, HAMMERIOC_GET_PSEUDOFS, &pfs) < 0) {
161 			warn("Filesystem %s is not HAMMER", scan->filesystem);
162 			rcode = 1;
163 			close(fd);
164 			continue;
165 		}
166 		scan->prune_min = pfs.ondisk->prune_min;
167 
168 		/*
169 		 * Incorporate meta-data snapshots into the pruning regimen.
170 		 * If pruning everything we delete the meta-data snapshots.
171 		 */
172 		if (hammer_softprune_scanmeta(fd, scan, everything_opt) < 0) {
173 			warn("Filesystem %s could not scan meta-data snaps",
174 			     scan->filesystem);
175 			rcode = 1;
176 			close(fd);
177 			continue;
178 		}
179 
180 		/*
181 		 * Finalize operations
182 		 */
183 		hammer_softprune_finalize(scan);
184 		if (everything_opt) {
185 			printf("Prune %s: EVERYTHING\n",
186 			       scan->filesystem);
187 		} else {
188 			printf("Prune %s: %d snapshots\n",
189 			       scan->filesystem, scan->prune.nelms);
190 		}
191 		if (scan->prune.nelms == 0 &&
192 		    (scan->prune.head.flags & HAMMER_IOC_PRUNE_ALL) == 0) {
193 			fprintf(stderr, "No snapshots found\n");
194 			continue;
195 		}
196 
197 		printf("Prune %s: objspace %016jx:%04x %016jx:%04x "
198 		       "pfs_id %d\n",
199 		       scan->filesystem,
200 		       (uintmax_t)scan->prune.key_beg.obj_id,
201 		       scan->prune.key_beg.localization,
202 		       (uintmax_t)scan->prune.key_end.obj_id,
203 		       scan->prune.key_end.localization,
204 		       pfs.pfs_id);
205 		printf("Prune %s: prune_min is %dd/%02d:%02d:%02d\n",
206 		       scan->filesystem,
207 			pfs.ondisk->prune_min / (24 * 60 * 60),
208 			pfs.ondisk->prune_min / 60 / 60 % 24,
209 			pfs.ondisk->prune_min / 60 % 60,
210 			pfs.ondisk->prune_min % 60);
211 
212 		RunningIoctl = 1;
213 		if (ioctl(fd, HAMMERIOC_PRUNE, &scan->prune) < 0) {
214 			printf("Prune %s failed: %s\n",
215 			       scan->filesystem, strerror(errno));
216 			rcode = 2;
217 		} else if (scan->prune.head.flags & HAMMER_IOC_HEAD_INTR) {
218 			printf("Prune %s interrupted by timer at "
219 			       "%016jx %04x\n",
220 			       scan->filesystem,
221 			       (uintmax_t)scan->prune.key_cur.obj_id,
222 			       scan->prune.key_cur.localization);
223 			if (CyclePath)
224 				hammer_set_cycle(&scan->prune.key_cur, 0);
225 			rcode = 0;
226 		} else {
227 			if (CyclePath)
228 				hammer_reset_cycle();
229 			printf("Prune %s succeeded\n", scan->filesystem);
230 		}
231 		printf("Pruned %jd/%jd records (%jd directory entries) "
232 		       "and %jd bytes\n",
233 			(uintmax_t)scan->prune.stat_rawrecords,
234 			(uintmax_t)scan->prune.stat_scanrecords,
235 			(uintmax_t)scan->prune.stat_dirrecords,
236 			(uintmax_t)scan->prune.stat_bytes
237 		);
238 		RunningIoctl = 0;
239 		close(fd);
240 	}
241 	if (rcode)
242 		exit(rcode);
243 }
244 
245 /*
246  * Scan a directory for softlinks representing snapshots and build
247  * associated softprune structures.
248  *
249  * NOTE: Once a filesystem is completely converted to the meta-data
250  *	 snapshot mechanic we don't have to scan softlinks any more
251  *	 and can just use the meta-data.  But for now we do both.
252  */
253 static void
254 hammer_softprune_scandir(struct softprune **basep,
255 			 struct hammer_ioc_prune *template,
256 			 const char *dirname)
257 {
258 	struct stat st;
259 	struct dirent *den;
260 	DIR *dir;
261 	char *path;
262 	int len;
263 	char *linkbuf;
264 	char *ptr;
265 
266 	path = NULL;
267 	linkbuf = malloc(MAXPATHLEN);
268 
269 	if ((dir = opendir(dirname)) == NULL)
270 		err(1, "Cannot open directory %s", dirname);
271 	while ((den = readdir(dir)) != NULL) {
272 		if (strcmp(den->d_name, ".") == 0)
273 			continue;
274 		if (strcmp(den->d_name, "..") == 0)
275 			continue;
276 		if (path)
277 			free(path);
278 		asprintf(&path, "%s/%s", dirname, den->d_name);
279 		if (lstat(path, &st) < 0)
280 			continue;
281 		if (!S_ISLNK(st.st_mode))
282 			continue;
283 		if ((len = readlink(path, linkbuf, MAXPATHLEN - 1)) < 0)
284 			continue;
285 		linkbuf[len] = 0;
286 		if ((ptr = strrchr(linkbuf, '@')) &&
287 		    ptr > linkbuf && ptr[-1] == '@') {
288 			hammer_softprune_addentry(basep, template,
289 						  dirname, den->d_name, &st,
290 						  linkbuf, ptr - 1);
291 		}
292 	}
293 	free(linkbuf);
294 	if (path)
295 		free(path);
296 }
297 
298 /*
299  * Scan the metadata snapshots for the filesystem and either delete them
300  * or add them to the pruning list.
301  */
302 static
303 int
304 hammer_softprune_scanmeta(int fd, struct softprune *scan, int delete_all)
305 {
306 	struct hammer_ioc_version	version;
307 	struct hammer_ioc_snapshot	snapshot;
308 	struct hammer_ioc_snapshot	dsnapshot;
309 	struct hammer_snapshot_data	*snap;
310 	time_t ct;
311 
312 	/*
313 	 * Stop if we can't get the version.  Meta-data snapshots only
314 	 * exist for HAMMER version 3 or greater.
315 	 */
316 	bzero(&version, sizeof(version));
317 	if (ioctl(fd, HAMMERIOC_GET_VERSION, &version) < 0)
318 		return(-1);
319 	if (version.cur_version < 3)
320 		return(0);
321 
322 	bzero(&snapshot, sizeof(snapshot));
323 	bzero(&dsnapshot, sizeof(dsnapshot));
324 
325 	/*
326 	 * Scan meta-data snapshots, either add them to the prune list or
327 	 * delete them.  When deleting, just skip any entries which cannot
328 	 * be deleted.
329 	 */
330 	for (;;) {
331 		if (ioctl(fd, HAMMERIOC_GET_SNAPSHOT, &snapshot) < 0) {
332 			printf("hammer prune: Unable to access "
333 			       "meta-data snaps: %s\n", strerror(errno));
334 			return(-1);
335 		}
336 		while (snapshot.index < snapshot.count) {
337 			snap = &snapshot.snaps[snapshot.index];
338 			if (delete_all) {
339 				dsnapshot.snaps[dsnapshot.count++] = *snap;
340 				if (dsnapshot.count == HAMMER_SNAPS_PER_IOCTL)
341 					hammer_meta_flushdelete(fd, &dsnapshot);
342 			} else {
343 				ct = snap->ts / 1000000ULL;
344 				hammer_softprune_addelm(scan, snap->tid,
345 							ct, ct);
346 			}
347 			++snapshot.index;
348 		}
349 		if (snapshot.head.flags & HAMMER_IOC_SNAPSHOT_EOF)
350 			break;
351 		snapshot.index = 0;
352 	}
353 	if (delete_all)
354 		hammer_meta_flushdelete(fd, &dsnapshot);
355 	return(0);
356 }
357 
358 /*
359  * Flush any entries built up in the deletion snapshot ioctl structure.
360  * Used during a prune-everything.
361  */
362 static void
363 hammer_meta_flushdelete(int fd, struct hammer_ioc_snapshot *dsnap)
364 {
365 	while (dsnap->index < dsnap->count) {
366 		if (ioctl(fd, HAMMERIOC_DEL_SNAPSHOT, dsnap) < 0)
367 			break;
368 		if (dsnap->head.error == 0)
369 			break;
370 		++dsnap->index;
371 	}
372 	dsnap->index = 0;
373 	dsnap->count = 0;
374 }
375 
376 /*
377  * Add the softlink to the appropriate softprune structure, creating a new
378  * one if necessary.
379  */
380 static
381 struct softprune *
382 hammer_softprune_addentry(struct softprune **basep,
383 			 struct hammer_ioc_prune *template,
384 			 const char *dirpath, const char *denname __unused,
385 			 struct stat *st,
386 			 const char *linkbuf, const char *tidptr)
387 {
388 	struct softprune *scan;
389 	struct statfs fs;
390 	char *fspath;
391 
392 	/*
393 	 * Calculate filesystem path.
394 	 */
395 	if (linkbuf[0] == '/') {
396 		asprintf(&fspath, "%*.*s",
397 			 (int)(tidptr - linkbuf), (int)(tidptr - linkbuf),
398 			 linkbuf);
399 	} else {
400 		asprintf(&fspath, "%s/%*.*s", dirpath,
401 			 (int)(tidptr - linkbuf), (int)(tidptr - linkbuf),
402 			 linkbuf);
403 	}
404 	if (statfs(fspath, &fs) < 0) {
405 		free(fspath);
406 		return(NULL);
407 	}
408 
409 	/*
410 	 * Locate the filesystem in an existing softprune structure
411 	 */
412 	for (scan = *basep; scan; scan = scan->next) {
413 		if (bcmp(&fs.f_fsid, &scan->fs.f_fsid, sizeof(fs.f_fsid)) != 0)
414 			continue;
415 		if (strcmp(fs.f_mntonname, scan->fs.f_mntonname) != 0)
416 			continue;
417 		break;
418 	}
419 
420 	/*
421 	 * Create a new softprune structure if necessasry
422 	 */
423 	if (scan == NULL) {
424 		scan = malloc(sizeof(*scan));
425 		bzero(scan, sizeof(*scan));
426 
427 		scan->fs = fs;
428 		scan->filesystem = fspath;
429 		scan->prune = *template;
430 		scan->maxelms = 32;
431 		scan->prune.elms = malloc(sizeof(struct hammer_ioc_prune_elm) *
432 					  scan->maxelms);
433 		scan->next = *basep;
434 		*basep = scan;
435 	} else {
436 		free(fspath);
437 	}
438 	hammer_softprune_addelm(scan,
439 				(hammer_tid_t)strtoull(tidptr + 2, NULL, 0),
440 				(st ? st->st_ctime : 0),
441 				(st ? st->st_mtime : 0));
442 	return(scan);
443 }
444 
445 /*
446  * Add the entry (unsorted).  Just set the beg_tid, we will sort
447  * and set the remaining entries later.
448  *
449  * Always leave one entry free for our terminator.
450  */
451 static void
452 hammer_softprune_addelm(struct softprune *scan, hammer_tid_t tid,
453 			time_t ct, time_t mt)
454 {
455 	struct hammer_ioc_prune_elm *elm;
456 
457 	if (scan->prune.nelms >= scan->maxelms - 1) {
458 		scan->maxelms = (scan->maxelms * 3 / 2);
459 		scan->prune.elms = realloc(scan->prune.elms,
460 					   sizeof(*elm) * scan->maxelms);
461 	}
462 
463 	/*
464 	 * NOTE: Temporarily store the snapshot timestamp in mod_tid.
465 	 *	 This will be cleaned up in the finalization phase.
466 	 */
467 	elm = &scan->prune.elms[scan->prune.nelms];
468 	elm->beg_tid = tid;
469 	elm->end_tid = 0;
470 	elm->mod_tid = 0;
471 	if (ct < mt)
472 		elm->mod_tid = ct;
473 	else
474 		elm->mod_tid = mt;
475 	++scan->prune.nelms;
476 }
477 
478 /*
479  * Finalize a softprune structure after scanning in its softlinks.
480  * Sort the elements, remove duplicates, and then fill in end_tid and
481  * mod_tid.
482  *
483  * The array must end up in descending order.
484  */
485 static int
486 hammer_softprune_qsort_cmp(const void *arg1, const void *arg2)
487 {
488 	const struct hammer_ioc_prune_elm *elm1 = arg1;
489 	const struct hammer_ioc_prune_elm *elm2 = arg2;
490 
491 	if (elm1->beg_tid < elm2->beg_tid)
492 		return(1);
493 	if (elm1->beg_tid > elm2->beg_tid)
494 		return(-1);
495 	return(0);
496 }
497 
498 static void
499 hammer_softprune_finalize(struct softprune *scan)
500 {
501 	struct hammer_ioc_prune_elm *elm;
502 	time_t t;
503 	long delta;
504 	int i;
505 
506 	/*
507 	 * Don't do anything if there are no elements.
508 	 */
509 	if (scan->prune.nelms == 0)
510 		return;
511 
512 	/*
513 	 * Sort the elements in descending order, remove duplicates, and
514 	 * fill in any missing bits.
515 	 */
516 	qsort(scan->prune.elms, scan->prune.nelms, sizeof(*elm),
517 	      hammer_softprune_qsort_cmp);
518 
519 	for (i = 0; i < scan->prune.nelms; ++i) {
520 		elm = &scan->prune.elms[i];
521 		if (i == 0) {
522 			/*
523 			 * First (highest TID) (also last if only one element)
524 			 */
525 			elm->end_tid = HAMMER_MAX_TID;
526 		} else if (elm[0].beg_tid == elm[-1].beg_tid) {
527 			/*
528 			 * Remove duplicate
529 			 */
530 			--scan->prune.nelms;
531 			if (i != scan->prune.nelms) {
532 				bcopy(elm + 1, elm,
533 				      (scan->prune.nelms - i) * sizeof(*elm));
534 			}
535 			--i;
536 			continue;
537 		} else {
538 			/*
539 			 * Middle or last.
540 			 */
541 			elm->end_tid = elm[-1].beg_tid;
542 		}
543 	}
544 
545 	/*
546 	 * If a minimum retention time (in seconds) is configured for the
547 	 * PFS, remove any snapshots from the pruning list that are within
548 	 * the period.
549 	 */
550 	if (scan->prune_min) {
551 		t = time(NULL);
552 		for (i = scan->prune.nelms - 1; i >= 0; --i) {
553 			elm = &scan->prune.elms[i];
554 			if (elm->mod_tid == 0)
555 				continue;
556 			delta = (long)(t - (time_t)elm->mod_tid);
557 			if (delta < scan->prune_min)
558 				break;
559 		}
560 		++i;
561 		if (i) {
562 			printf("Prune %s: prune_min: Will not clean between "
563 			       "the teeth of the first %d snapshots\n",
564 			       scan->filesystem, i);
565 			bcopy(&scan->prune.elms[i], &scan->prune.elms[0],
566 			      (scan->prune.nelms - i) * sizeof(scan->prune.elms[0]));
567 			scan->prune.elms[0].end_tid = HAMMER_MAX_TID;
568 			scan->prune.nelms -= i;
569 		}
570 	}
571 
572 	/*
573 	 * Remove the first entry.  This entry represents the prune from
574 	 * the most recent snapshot to current.  We wish to retain the
575 	 * fine-grained history for this region.
576 	 */
577 	if (scan->prune.nelms) {
578 		bcopy(&scan->prune.elms[1], &scan->prune.elms[0],
579 		      (scan->prune.nelms - 1) * sizeof(scan->prune.elms[0]));
580 		--scan->prune.nelms;
581 	}
582 
583 	/*
584 	 * Add a final element to prune everything from transaction id
585 	 * 0 to the lowest transaction id (aka last so far).
586 	 */
587 	if (scan->prune.nelms) {
588 		assert(scan->prune.nelms < scan->maxelms);
589 		elm = &scan->prune.elms[scan->prune.nelms];
590 		elm->beg_tid = 1;
591 		elm->end_tid = elm[-1].beg_tid;
592 		++scan->prune.nelms;
593 	}
594 
595 	/*
596 	 * Adjust mod_tid to what the ioctl() expects.
597 	 */
598 	for (i = 0; i < scan->prune.nelms; ++i) {
599 		elm = &scan->prune.elms[i];
600 		elm->mod_tid = elm->end_tid - elm->beg_tid;
601 		printf("TID %016jx - %016jx\n",
602 		       (uintmax_t)elm->beg_tid, (uintmax_t)elm->end_tid);
603 	}
604 }
605 
606 static
607 void
608 softprune_usage(int code)
609 {
610 	fprintf(stderr, "Badly formed prune command, use:\n");
611 	fprintf(stderr, "hammer prune <softlink-dir>\n");
612 	fprintf(stderr, "hammer prune-everything <filesystem>\n");
613 	exit(code);
614 }
615 
616 
617