xref: /dragonfly/usr.bin/ktrdump/ktrdump.c (revision 9f3fc534)
1 /*-
2  * Copyright (c) 2002 Jake Burkholder
3  * Copyright (c) 2004 Robert Watson
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  * $FreeBSD: src/usr.bin/ktrdump/ktrdump.c,v 1.10 2005/05/21 09:55:06 ru Exp $
28  * $DragonFly: src/usr.bin/ktrdump/ktrdump.c,v 1.13 2008/11/10 02:05:31 swildner Exp $
29  */
30 
31 #include <sys/cdefs.h>
32 
33 #include <sys/types.h>
34 #include <sys/ktr.h>
35 #include <sys/mman.h>
36 #include <sys/stat.h>
37 #include <sys/queue.h>
38 
39 #include <err.h>
40 #include <fcntl.h>
41 #include <kvm.h>
42 #include <limits.h>
43 #include <nlist.h>
44 #include <stdint.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48 #include <unistd.h>
49 
50 #define	SBUFLEN	128
51 
52 struct ktr_buffer {
53 	struct ktr_entry *ents;
54 	int modified;
55 	int reset;
56 	int beg_idx;		/* Beginning index */
57 	int end_idx;		/* Ending index */
58 };
59 
60 static struct nlist nl1[] = {
61 	{ .n_name = "_ktr_version" },
62 	{ .n_name = "_ktr_entries" },
63 	{ .n_name = "_ncpus" },
64 	{ .n_name = NULL }
65 };
66 
67 static struct nlist nl2[] = {
68 	{ .n_name = "_tsc_frequency" },
69 	{ .n_name = NULL }
70 };
71 
72 static struct nlist nl_version_ktr_idx[] = {
73 	{ .n_name = "_ktr_idx" },
74 	{ .n_name = "_ktr_buf" },
75 	{ .n_name = NULL }
76 };
77 
78 static struct nlist nl_version_ktr_cpu[] = {
79 	{ .n_name = "_ktr_cpu" },
80 	{ .n_name = NULL }
81 };
82 
83 static int cflag;
84 static int fflag;
85 static int iflag;
86 static int lflag;
87 static int nflag;
88 static int qflag;
89 static int rflag;
90 static int sflag;
91 static int tflag;
92 static int xflag;
93 static int pflag;
94 static int Mflag;
95 static int Nflag;
96 static double tsc_frequency;
97 static double correction_factor = 0.0;
98 
99 static char corefile[PATH_MAX];
100 static char execfile[PATH_MAX];
101 
102 static char errbuf[_POSIX2_LINE_MAX];
103 static int ncpus;
104 static kvm_t *kd;
105 static int entries_per_buf;
106 static int fifo_mask;
107 static int ktr_version;
108 
109 static void usage(void);
110 static int earliest_ts(struct ktr_buffer *);
111 static void print_header(FILE *, int);
112 static void print_entry(FILE *, int, int, struct ktr_entry *, u_int64_t *);
113 static struct ktr_info *kvm_ktrinfo(void *);
114 static const char *kvm_string(const char *);
115 static const char *trunc_path(const char *, int);
116 static void read_symbols(const char *);
117 static const char *address_to_symbol(void *);
118 static struct ktr_buffer *ktr_bufs_init(void);
119 static void get_indices(struct ktr_entry **, int *);
120 static void load_bufs(struct ktr_buffer *, struct ktr_entry **, int *);
121 static void print_buf(FILE *, struct ktr_buffer *, int, u_int64_t *);
122 static void print_bufs_timesorted(FILE *, struct ktr_buffer *, u_int64_t *);
123 
124 
125 /*
126  * Reads the ktr trace buffer from kernel memory and prints the trace entries.
127  */
128 int
129 main(int ac, char **av)
130 {
131 	struct ktr_buffer *ktr_bufs;
132 	struct ktr_entry **ktr_kbuf;
133 	int *ktr_idx;
134 	FILE *fo;
135 	int64_t tts;
136 	int *ktr_start_index;
137 	int c;
138 	int n;
139 
140 	/*
141 	 * Parse commandline arguments.
142 	 */
143 	fo = stdout;
144 	while ((c = getopt(ac, av, "acfinqrtxpslA:N:M:o:")) != -1) {
145 		switch (c) {
146 		case 'a':
147 			cflag = 1;
148 			iflag = 1;
149 			rflag = 1;
150 			xflag = 1;
151 			pflag = 1;
152 			rflag = 1;
153 			sflag = 1;
154 			break;
155 		case 'c':
156 			cflag = 1;
157 			break;
158 		case 'N':
159 			if (strlcpy(execfile, optarg, sizeof(execfile))
160 			    >= sizeof(execfile))
161 				errx(1, "%s: File name too long", optarg);
162 			Nflag = 1;
163 			break;
164 		case 'f':
165 			fflag = 1;
166 			break;
167 		case 'l':
168 			lflag = 1;
169 			break;
170 		case 'i':
171 			iflag = 1;
172 			break;
173 		case 'A':
174 			correction_factor = strtod(optarg, NULL);
175 			break;
176 		case 'M':
177 			if (strlcpy(corefile, optarg, sizeof(corefile))
178 			    >= sizeof(corefile))
179 				errx(1, "%s: File name too long", optarg);
180 			Mflag = 1;
181 			break;
182 		case 'n':
183 			nflag = 1;
184 			break;
185 		case 'o':
186 			if ((fo = fopen(optarg, "w")) == NULL)
187 				err(1, "%s", optarg);
188 			break;
189 		case 'p':
190 			pflag++;
191 			break;
192 		case 'q':
193 			qflag++;
194 			break;
195 		case 'r':
196 			rflag = 1;
197 			break;
198 		case 's':
199 			sflag = 1;	/* sort across the cpus */
200 			break;
201 		case 't':
202 			tflag = 1;
203 			break;
204 		case 'x':
205 			xflag = 1;
206 			break;
207 		case '?':
208 		default:
209 			usage();
210 		}
211 	}
212 	if (cflag + iflag + tflag + xflag + fflag + pflag == 0) {
213 		cflag = 1;
214 		iflag = 1;
215 		tflag = 1;
216 		pflag = 1;
217 	}
218 	if (correction_factor != 0.0 && (rflag == 0 || nflag)) {
219 		fprintf(stderr, "Correction factor can only be applied with -r and without -n\n");
220 		exit(1);
221 	}
222 	ac -= optind;
223 	av += optind;
224 	if (ac != 0)
225 		usage();
226 
227 	/*
228 	 * Open our execfile and corefile, resolve needed symbols and read in
229 	 * the trace buffer.
230 	 */
231 	if ((kd = kvm_openfiles(Nflag ? execfile : NULL,
232 	    Mflag ? corefile : NULL, NULL, O_RDONLY, errbuf)) == NULL)
233 		errx(1, "%s", errbuf);
234 	if (kvm_nlist(kd, nl1) != 0)
235 		errx(1, "%s", kvm_geterr(kd));
236 	if (kvm_read(kd, nl1[0].n_value, &ktr_version, sizeof(ktr_version)) == -1)
237 		errx(1, "%s", kvm_geterr(kd));
238 	if (kvm_read(kd, nl1[2].n_value, &ncpus, sizeof(ncpus)) == -1)
239 		errx(1, "%s", kvm_geterr(kd));
240 	ktr_start_index = malloc(sizeof(*ktr_start_index) * ncpus);
241 	if (ktr_version >= KTR_VERSION_WITH_FREQ && kvm_nlist(kd, nl2) == 0) {
242 		if (kvm_read(kd, nl2[0].n_value, &tts, sizeof(tts)) == -1)
243 			errx(1, "%s", kvm_geterr(kd));
244 		tsc_frequency = (double)tts;
245 	}
246 	if (ktr_version > KTR_VERSION)
247 		errx(1, "ktr version too high for us to handle");
248 	if (kvm_read(kd, nl1[1].n_value, &entries_per_buf,
249 				sizeof(entries_per_buf)) == -1)
250 		errx(1, "%s", kvm_geterr(kd));
251 	fifo_mask = entries_per_buf - 1;
252 
253 	printf("TSC frequency is %6.3f MHz\n", tsc_frequency / 1000000.0);
254 
255 	ktr_kbuf = calloc(ncpus, sizeof(*ktr_kbuf));
256 	ktr_idx = calloc(ncpus, sizeof(*ktr_idx));
257 
258 	if (nflag == 0)
259 		read_symbols(Nflag ? execfile : NULL);
260 
261 	if (ktr_version < KTR_VERSION_KTR_CPU) {
262 		if (kvm_nlist(kd, nl_version_ktr_idx))
263 			errx(1, "%s", kvm_geterr(kd));
264 	} else {
265 		if (kvm_nlist(kd, nl_version_ktr_cpu))
266 			errx(1, "%s", kvm_geterr(kd));
267 	}
268 
269 	get_indices(ktr_kbuf, ktr_idx);
270 
271 	ktr_bufs = ktr_bufs_init();
272 
273 	if (sflag) {
274 		u_int64_t last_timestamp = 0;
275 		do {
276 			load_bufs(ktr_bufs, ktr_kbuf, ktr_idx);
277 			print_bufs_timesorted(fo, ktr_bufs, &last_timestamp);
278 			if (lflag)
279 				usleep(1000000 / 10);
280 		} while (lflag);
281 	} else {
282 		u_int64_t *last_timestamp = calloc(sizeof(u_int64_t), ncpus);
283 		do {
284 			load_bufs(ktr_bufs, ktr_kbuf, ktr_idx);
285 			for (n = 0; n < ncpus; ++n)
286 				print_buf(fo, ktr_bufs, n, &last_timestamp[n]);
287 			if (lflag)
288 				usleep(1000000 / 10);
289 		} while (lflag);
290 	}
291 	return (0);
292 }
293 
294 static void
295 print_header(FILE *fo, int row)
296 {
297 	if (qflag == 0 && (u_int32_t)row % 20 == 0) {
298 		fprintf(fo, "%-6s ", "index");
299 		if (cflag)
300 			fprintf(fo, "%-3s ", "cpu");
301 		if (tflag || rflag)
302 			fprintf(fo, "%-16s ", "timestamp");
303 		if (xflag) {
304 			if (nflag)
305 			    fprintf(fo, "%-10s %-10s", "caller2", "caller1");
306 			else
307 			    fprintf(fo, "%-20s %-20s", "caller2", "caller1");
308 		}
309 		if (iflag)
310 			fprintf(fo, "%-20s ", "ID");
311 		if (fflag)
312 			fprintf(fo, "%10s%-30s ", "", "file and line");
313 		if (pflag)
314 			fprintf(fo, "%s", "trace");
315 		fprintf(fo, "\n");
316 	}
317 }
318 
319 static void
320 print_entry(FILE *fo, int n, int row, struct ktr_entry *entry,
321 	    u_int64_t *last_timestamp)
322 {
323 	struct ktr_info *info = NULL;
324 
325 	fprintf(fo, " %06x ", row & 0x00FFFFFF);
326 	if (cflag)
327 		fprintf(fo, "%-3d ", n);
328 	if (tflag || rflag) {
329 		if (rflag && !nflag && tsc_frequency != 0.0) {
330 			fprintf(fo, "%13.3f uS ",
331 				(double)(entry->ktr_timestamp - *last_timestamp) * 1000000.0 / tsc_frequency - correction_factor);
332 		} else if (rflag) {
333 			fprintf(fo, "%-16ju ",
334 			    (uintmax_t)(entry->ktr_timestamp - *last_timestamp));
335 		} else {
336 			fprintf(fo, "%-16ju ",
337 			    (uintmax_t)entry->ktr_timestamp);
338 		}
339 	}
340 	if (xflag) {
341 		if (nflag) {
342 		    fprintf(fo, "%p %p ",
343 			    entry->ktr_caller2, entry->ktr_caller1);
344 		} else {
345 		    fprintf(fo, "%-20s ",
346 			    address_to_symbol(entry->ktr_caller2));
347 		    fprintf(fo, "%-20s ",
348 			    address_to_symbol(entry->ktr_caller1));
349 		}
350 	}
351 	if (iflag) {
352 		info = kvm_ktrinfo(entry->ktr_info);
353 		if (info)
354 			fprintf(fo, "%-20s ", kvm_string(info->kf_name));
355 		else
356 			fprintf(fo, "%-20s ", "<empty>");
357 	}
358 	if (fflag)
359 		fprintf(fo, "%34s:%-4d ", trunc_path(kvm_string(entry->ktr_file), 34), entry->ktr_line);
360 	if (pflag) {
361 		if (info == NULL)
362 			info = kvm_ktrinfo(entry->ktr_info);
363 		if (info)
364 			vfprintf(fo, kvm_string(info->kf_format), (void *)&entry->ktr_data);
365 	}
366 	fprintf(fo, "\n");
367 	*last_timestamp = entry->ktr_timestamp;
368 }
369 
370 static
371 struct ktr_info *
372 kvm_ktrinfo(void *kptr)
373 {
374 	static struct ktr_info save_info;
375 	static void *save_kptr;
376 
377 	if (kptr == NULL)
378 		return(NULL);
379 	if (save_kptr != kptr) {
380 		if (kvm_read(kd, (uintptr_t)kptr, &save_info, sizeof(save_info)) == -1) {
381 			bzero(&save_info, sizeof(save_info));
382 		} else {
383 			save_kptr = kptr;
384 		}
385 	}
386 	return(&save_info);
387 }
388 
389 static
390 const char *
391 kvm_string(const char *kptr)
392 {
393 	static char save_str[128];
394 	static const char *save_kptr;
395 	u_int l;
396 	u_int n;
397 
398 	if (kptr == NULL)
399 		return("?");
400 	if (save_kptr != kptr) {
401 		save_kptr = kptr;
402 		l = 0;
403 		while (l < sizeof(save_str) - 1) {
404 			n = 256 - ((intptr_t)(kptr + l) & 255);
405 			if (n > sizeof(save_str) - l - 1)
406 				n = sizeof(save_str) - l - 1;
407 			if (kvm_read(kd, (uintptr_t)(kptr + l), save_str + l, n) < 0)
408 				break;
409 			while (l < sizeof(save_str) && n) {
410 			    if (save_str[l] == 0)
411 				    break;
412 			    --n;
413 			    ++l;
414 			}
415 			if (n)
416 			    break;
417 		}
418 		save_str[l] = 0;
419 	}
420 	return(save_str);
421 }
422 
423 static
424 const char *
425 trunc_path(const char *str, int maxlen)
426 {
427 	int len = strlen(str);
428 
429 	if (len > maxlen)
430 		return(str + len - maxlen);
431 	else
432 		return(str);
433 }
434 
435 struct symdata {
436 	TAILQ_ENTRY(symdata) link;
437 	const char *symname;
438 	char *symaddr;
439 	char symtype;
440 };
441 
442 static TAILQ_HEAD(symlist, symdata) symlist;
443 static struct symdata *symcache;
444 static char *symbegin;
445 static char *symend;
446 
447 static
448 void
449 read_symbols(const char *file)
450 {
451 	char buf[256];
452 	char cmd[256];
453 	size_t buflen = sizeof(buf);
454 	FILE *fp;
455 	struct symdata *sym;
456 	char *s1;
457 	char *s2;
458 	char *s3;
459 
460 	TAILQ_INIT(&symlist);
461 
462 	if (file == NULL) {
463 		if (sysctlbyname("kern.bootfile", buf, &buflen, NULL, 0) < 0)
464 			file = "/boot/kernel";
465 		else
466 			file = buf;
467 	}
468 	snprintf(cmd, sizeof(cmd), "nm -n %s", file);
469 	if ((fp = popen(cmd, "r")) != NULL) {
470 		while (fgets(buf, sizeof(buf), fp) != NULL) {
471 		    s1 = strtok(buf, " \t\n");
472 		    s2 = strtok(NULL, " \t\n");
473 		    s3 = strtok(NULL, " \t\n");
474 		    if (s1 && s2 && s3) {
475 			sym = malloc(sizeof(struct symdata));
476 			sym->symaddr = (char *)strtoul(s1, NULL, 16);
477 			sym->symtype = s2[0];
478 			sym->symname = strdup(s3);
479 			if (strcmp(s3, "kernbase") == 0)
480 				symbegin = sym->symaddr;
481 			if (strcmp(s3, "end") == 0)
482 				symend = sym->symaddr;
483 			TAILQ_INSERT_TAIL(&symlist, sym, link);
484 		    }
485 		}
486 		pclose(fp);
487 	}
488 	symcache = TAILQ_FIRST(&symlist);
489 }
490 
491 static
492 const char *
493 address_to_symbol(void *kptr)
494 {
495 	static char buf[64];
496 
497 	if (symcache == NULL ||
498 	   (char *)kptr < symbegin || (char *)kptr >= symend
499 	) {
500 		snprintf(buf, sizeof(buf), "%p", kptr);
501 		return(buf);
502 	}
503 	while ((char *)symcache->symaddr < (char *)kptr) {
504 		if (TAILQ_NEXT(symcache, link) == NULL)
505 			break;
506 		symcache = TAILQ_NEXT(symcache, link);
507 	}
508 	while ((char *)symcache->symaddr > (char *)kptr) {
509 		if (symcache != TAILQ_FIRST(&symlist))
510 			symcache = TAILQ_PREV(symcache, symlist, link);
511 	}
512 	snprintf(buf, sizeof(buf), "%s+%d", symcache->symname,
513 		(int)((char *)kptr - symcache->symaddr));
514 	return(buf);
515 }
516 
517 static
518 struct ktr_buffer *
519 ktr_bufs_init(void)
520 {
521 	struct ktr_buffer *ktr_bufs, *it;
522 	int i;
523 
524 	ktr_bufs = malloc(sizeof(*ktr_bufs) * ncpus);
525 	if (!ktr_bufs)
526 		err(1, "can't allocate data structures\n");
527 	for (i = 0; i < ncpus; ++i) {
528 		it = ktr_bufs + i;
529 		it->ents = malloc(sizeof(struct ktr_entry) * entries_per_buf);
530 		if (it->ents == NULL)
531 			err(1, "can't allocate data structures\n");
532 		it->reset = 1;
533 		it->beg_idx = -1;
534 		it->end_idx = -1;
535 	}
536 	return ktr_bufs;
537 }
538 
539 static
540 void
541 get_indices(struct ktr_entry **ktr_kbuf, int *ktr_idx)
542 {
543 	static struct ktr_cpu *ktr_cpus;
544 	int i;
545 
546 	if (ktr_cpus == NULL)
547 		ktr_cpus = malloc(sizeof(*ktr_cpus) * ncpus);
548 
549 	if (ktr_version < KTR_VERSION_KTR_CPU) {
550 		if (kvm_read(kd, nl_version_ktr_idx[0].n_value, ktr_idx,
551 		    sizeof(*ktr_idx) * ncpus) == -1) {
552 			errx(1, "%s", kvm_geterr(kd));
553 		}
554 		if (ktr_kbuf[0] == NULL) {
555 			if (kvm_read(kd, nl_version_ktr_idx[1].n_value,
556 			    ktr_kbuf, sizeof(*ktr_kbuf) * ncpus) == -1) {
557 				errx(1, "%s", kvm_geterr(kd));
558 			}
559 		}
560 	} else {
561 		if (kvm_read(kd, nl_version_ktr_cpu[0].n_value,
562 			     ktr_cpus, sizeof(*ktr_cpus) * ncpus) == -1) {
563 				errx(1, "%s", kvm_geterr(kd));
564 		}
565 		for (i = 0; i < ncpus; ++i) {
566 			ktr_idx[i] = ktr_cpus[i].core.ktr_idx;
567 			ktr_kbuf[i] = ktr_cpus[i].core.ktr_buf;
568 		}
569 	}
570 }
571 
572 /*
573  * Get the trace buffer data from the kernel
574  */
575 static
576 void
577 load_bufs(struct ktr_buffer *ktr_bufs, struct ktr_entry **kbufs, int *ktr_idx)
578 {
579 	struct ktr_buffer *kbuf;
580 	int i;
581 
582 	get_indices(kbufs, ktr_idx);
583 	for (i = 0; i < ncpus; ++i) {
584 		kbuf = &ktr_bufs[i];
585 		if (ktr_idx[i] == kbuf->end_idx)
586 			continue;
587 		kbuf->end_idx = ktr_idx[i];
588 
589 		/*
590 		 * If we do not have a notion of the beginning index, assume
591 		 * it is entries_per_buf before the ending index.  Don't
592 		 * worry about underflows/negative numbers, the indices will
593 		 * be masked.
594 		 */
595 		if (kbuf->reset) {
596 			kbuf->beg_idx = kbuf->end_idx - entries_per_buf + 1;
597 			kbuf->reset = 0;
598 		}
599 		if (kvm_read(kd, (uintptr_t)kbufs[i], ktr_bufs[i].ents,
600 				sizeof(struct ktr_entry) * entries_per_buf)
601 									== -1)
602 			errx(1, "%s", kvm_geterr(kd));
603 		kbuf->modified = 1;
604 		kbuf->beg_idx = earliest_ts(kbuf);
605 	}
606 
607 }
608 
609 /*
610  * Locate the earliest timestamp iterating backwards from end_idx, but
611  * not going further back then beg_idx.  We have to do this because
612  * the kernel uses a circulating buffer.
613  */
614 static
615 int
616 earliest_ts(struct ktr_buffer *buf)
617 {
618 	struct ktr_entry *save;
619 	int count, scan, i, earliest;
620 
621 	count = 0;
622 	earliest = buf->end_idx - 1;
623 	save = &buf->ents[earliest & fifo_mask];
624 	for (scan = buf->end_idx - 1; scan != buf->beg_idx -1; --scan) {
625 		i = scan & fifo_mask;
626 		if (buf->ents[i].ktr_timestamp <= save->ktr_timestamp &&
627 		    buf->ents[i].ktr_timestamp > 0)
628 			earliest = scan;
629 		/*
630 		 * We may have gotten so far behind that beg_idx wrapped
631 		 * more then once around the buffer.  Just stop
632 		 */
633 		if (++count == entries_per_buf)
634 			break;
635 	}
636 	return earliest;
637 }
638 
639 static
640 void
641 print_buf(FILE *fo, struct ktr_buffer *ktr_bufs, int cpu,
642 	  u_int64_t *last_timestamp)
643 {
644 	struct ktr_buffer *buf = ktr_bufs + cpu;
645 
646 	if (buf->modified == 0)
647 		return;
648 	if (*last_timestamp == 0) {
649 		*last_timestamp =
650 			buf->ents[buf->beg_idx & fifo_mask].ktr_timestamp;
651 	}
652 	while (buf->beg_idx != buf->end_idx) {
653 		print_header(fo, buf->beg_idx);
654 		print_entry(fo, cpu, buf->beg_idx,
655 			    &buf->ents[buf->beg_idx & fifo_mask],
656 			    last_timestamp);
657 		++buf->beg_idx;
658 	}
659 	buf->modified = 0;
660 }
661 
662 static
663 void
664 print_bufs_timesorted(FILE *fo, struct ktr_buffer *ktr_bufs,
665 		      u_int64_t *last_timestamp)
666 {
667 	struct ktr_entry *ent;
668 	struct ktr_buffer *buf;
669 	int n, bestn;
670 	u_int64_t ts;
671 	static int row = 0;
672 
673 	for (;;) {
674 		ts = 0;
675 		bestn = -1;
676 		for (n = 0; n < ncpus; ++n) {
677 			buf = ktr_bufs + n;
678 			if (buf->beg_idx == buf->end_idx)
679 				continue;
680 			ent = &buf->ents[buf->beg_idx & fifo_mask];
681 			if (ts == 0 || (ts >= ent->ktr_timestamp)) {
682 				ts = ent->ktr_timestamp;
683 				bestn = n;
684 			}
685 		}
686 		if ((bestn < 0) || (ts < *last_timestamp))
687 			break;
688 		buf = ktr_bufs + bestn;
689 		print_header(fo, row);
690 		print_entry(fo, bestn, row,
691 			    &buf->ents[buf->beg_idx & fifo_mask],
692 			    last_timestamp);
693 		++buf->beg_idx;
694 		*last_timestamp = ts;
695 		++row;
696 	}
697 }
698 
699 static void
700 usage(void)
701 {
702 	fprintf(stderr, "usage: ktrdump [-acfilnpqrstx] [-A factor] [-N execfile] "
703 			"[-M corefile] [-o outfile]\n");
704 	exit(1);
705 }
706