1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
22 *
23 */
24
25 /*
26 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/conf.h>
33 #include <sys/cpuvar.h>
34 #include <sys/endian.h>
35 #include <sys/fcntl.h>
36 #include <sys/filio.h>
37 #include <sys/kdb.h>
38 #include <sys/kernel.h>
39 #include <sys/kmem.h>
40 #include <sys/kthread.h>
41 #include <sys/limits.h>
42 #include <sys/linker.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/mutex.h>
47 #include <sys/pcpu.h>
48 #include <sys/poll.h>
49 #include <sys/proc.h>
50 #include <sys/selinfo.h>
51 #include <sys/smp.h>
52 #include <sys/syscall.h>
53 #include <sys/sysent.h>
54 #include <sys/sysproto.h>
55 #include <sys/uio.h>
56 #include <sys/unistd.h>
57 #include <machine/stdarg.h>
58
59 #include <sys/dtrace.h>
60 #include <sys/dtrace_bsd.h>
61
62 #include "fbt.h"
63
64 MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
65
66 dtrace_provider_id_t fbt_id;
67 fbt_probe_t **fbt_probetab;
68 int fbt_probetab_mask;
69
70 static int fbt_unload(void);
71 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
72 static void fbt_provide_module(void *, modctl_t *);
73 static void fbt_destroy(void *, dtrace_id_t, void *);
74 static void fbt_enable(void *, dtrace_id_t, void *);
75 static void fbt_disable(void *, dtrace_id_t, void *);
76 static void fbt_load(void *);
77 static void fbt_suspend(void *, dtrace_id_t, void *);
78 static void fbt_resume(void *, dtrace_id_t, void *);
79
80 static dtrace_pattr_t fbt_attr = {
81 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
82 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
83 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
84 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
85 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
86 };
87
88 static dtrace_pops_t fbt_pops = {
89 .dtps_provide = NULL,
90 .dtps_provide_module = fbt_provide_module,
91 .dtps_enable = fbt_enable,
92 .dtps_disable = fbt_disable,
93 .dtps_suspend = fbt_suspend,
94 .dtps_resume = fbt_resume,
95 .dtps_getargdesc = fbt_getargdesc,
96 .dtps_getargval = NULL,
97 .dtps_usermode = NULL,
98 .dtps_destroy = fbt_destroy
99 };
100
101 static int fbt_probetab_size;
102 static int fbt_verbose = 0;
103
104 int
fbt_excluded(const char * name)105 fbt_excluded(const char *name)
106 {
107
108 if (strncmp(name, "dtrace_", 7) == 0 &&
109 strncmp(name, "dtrace_safe_", 12) != 0) {
110 /*
111 * Anything beginning with "dtrace_" may be called
112 * from probe context unless it explicitly indicates
113 * that it won't be called from probe context by
114 * using the prefix "dtrace_safe_".
115 */
116 return (1);
117 }
118
119 /*
120 * Omit instrumentation of functions that are probably in DDB. It
121 * makes it too hard to debug broken FBT.
122 *
123 * NB: kdb_enter() can be excluded, but its call to printf() can't be.
124 * This is generally OK since we're not yet in debugging context.
125 */
126 if (strncmp(name, "db_", 3) == 0 ||
127 strncmp(name, "kdb_", 4) == 0)
128 return (1);
129
130 /*
131 * Lock owner methods may be called from probe context.
132 */
133 if (strcmp(name, "owner_mtx") == 0 ||
134 strcmp(name, "owner_rm") == 0 ||
135 strcmp(name, "owner_rw") == 0 ||
136 strcmp(name, "owner_sx") == 0)
137 return (1);
138
139 /*
140 * Stack unwinders may be called from probe context on some
141 * platforms.
142 */
143 #if defined(__aarch64__) || defined(__riscv)
144 if (strcmp(name, "unwind_frame") == 0)
145 return (1);
146 #endif
147
148 /*
149 * When DTrace is built into the kernel we need to exclude
150 * the FBT functions from instrumentation.
151 */
152 #ifndef _KLD_MODULE
153 if (strncmp(name, "fbt_", 4) == 0)
154 return (1);
155 #endif
156
157 return (0);
158 }
159
160 static void
fbt_doubletrap(void)161 fbt_doubletrap(void)
162 {
163 fbt_probe_t *fbt;
164 int i;
165
166 for (i = 0; i < fbt_probetab_size; i++) {
167 fbt = fbt_probetab[i];
168
169 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
170 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
171 }
172 }
173
174 static void
fbt_provide_module(void * arg,modctl_t * lf)175 fbt_provide_module(void *arg, modctl_t *lf)
176 {
177 char modname[MAXPATHLEN];
178 int i;
179 size_t len;
180
181 strlcpy(modname, lf->filename, sizeof(modname));
182 len = strlen(modname);
183 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0)
184 modname[len - 3] = '\0';
185
186 /*
187 * Employees of dtrace and their families are ineligible. Void
188 * where prohibited.
189 */
190 if (strcmp(modname, "dtrace") == 0)
191 return;
192
193 /*
194 * To register with DTrace, a module must list 'dtrace' as a
195 * dependency in order for the kernel linker to resolve
196 * symbols like dtrace_register(). All modules with such a
197 * dependency are ineligible for FBT tracing.
198 */
199 for (i = 0; i < lf->ndeps; i++)
200 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0)
201 return;
202
203 if (lf->fbt_nentries) {
204 /*
205 * This module has some FBT entries allocated; we're afraid
206 * to screw with it.
207 */
208 return;
209 }
210
211 /*
212 * List the functions in the module and the symbol values.
213 */
214 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname);
215 }
216
217 static void
fbt_destroy_one(fbt_probe_t * fbt)218 fbt_destroy_one(fbt_probe_t *fbt)
219 {
220 fbt_probe_t *hash, *hashprev, *next;
221 int ndx;
222
223 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
224 for (hash = fbt_probetab[ndx], hashprev = NULL; hash != NULL;
225 hashprev = hash, hash = hash->fbtp_hashnext) {
226 if (hash == fbt) {
227 if ((next = fbt->fbtp_tracenext) != NULL)
228 next->fbtp_hashnext = hash->fbtp_hashnext;
229 else
230 next = hash->fbtp_hashnext;
231 if (hashprev != NULL)
232 hashprev->fbtp_hashnext = next;
233 else
234 fbt_probetab[ndx] = next;
235 goto free;
236 } else if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
237 for (next = hash; next->fbtp_tracenext != NULL;
238 next = next->fbtp_tracenext) {
239 if (fbt == next->fbtp_tracenext) {
240 next->fbtp_tracenext =
241 fbt->fbtp_tracenext;
242 goto free;
243 }
244 }
245 }
246 }
247 panic("probe %p not found in hash table", fbt);
248 free:
249 free(fbt, M_FBT);
250 }
251
252 static void
fbt_destroy(void * arg,dtrace_id_t id,void * parg)253 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
254 {
255 fbt_probe_t *fbt = parg, *next;
256 modctl_t *ctl;
257
258 do {
259 ctl = fbt->fbtp_ctl;
260 ctl->fbt_nentries--;
261
262 next = fbt->fbtp_probenext;
263 fbt_destroy_one(fbt);
264 fbt = next;
265 } while (fbt != NULL);
266 }
267
268 static void
fbt_enable(void * arg,dtrace_id_t id,void * parg)269 fbt_enable(void *arg, dtrace_id_t id, void *parg)
270 {
271 fbt_probe_t *fbt = parg;
272 modctl_t *ctl = fbt->fbtp_ctl;
273
274 ctl->nenabled++;
275
276 /*
277 * Now check that our modctl has the expected load count. If it
278 * doesn't, this module must have been unloaded and reloaded -- and
279 * we're not going to touch it.
280 */
281 if (ctl->loadcnt != fbt->fbtp_loadcnt) {
282 if (fbt_verbose) {
283 printf("fbt is failing for probe %s "
284 "(module %s reloaded)",
285 fbt->fbtp_name, ctl->filename);
286 }
287
288 return;
289 }
290
291 for (; fbt != NULL; fbt = fbt->fbtp_probenext) {
292 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval);
293 fbt->fbtp_enabled++;
294 }
295 }
296
297 static void
fbt_disable(void * arg,dtrace_id_t id,void * parg)298 fbt_disable(void *arg, dtrace_id_t id, void *parg)
299 {
300 fbt_probe_t *fbt = parg, *hash;
301 modctl_t *ctl = fbt->fbtp_ctl;
302
303 ASSERT(ctl->nenabled > 0);
304 ctl->nenabled--;
305
306 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
307 return;
308
309 for (; fbt != NULL; fbt = fbt->fbtp_probenext) {
310 fbt->fbtp_enabled--;
311
312 for (hash = fbt_probetab[FBT_ADDR2NDX(fbt->fbtp_patchpoint)];
313 hash != NULL; hash = hash->fbtp_hashnext) {
314 if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
315 for (; hash != NULL; hash = hash->fbtp_tracenext)
316 if (hash->fbtp_enabled > 0)
317 break;
318 break;
319 }
320 }
321 if (hash == NULL)
322 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
323 }
324 }
325
326 static void
fbt_suspend(void * arg,dtrace_id_t id,void * parg)327 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
328 {
329 fbt_probe_t *fbt = parg;
330 modctl_t *ctl = fbt->fbtp_ctl;
331
332 ASSERT(ctl->nenabled > 0);
333
334 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
335 return;
336
337 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
338 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
339 }
340
341 static void
fbt_resume(void * arg,dtrace_id_t id,void * parg)342 fbt_resume(void *arg, dtrace_id_t id, void *parg)
343 {
344 fbt_probe_t *fbt = parg;
345 modctl_t *ctl = fbt->fbtp_ctl;
346
347 ASSERT(ctl->nenabled > 0);
348
349 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
350 return;
351
352 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
353 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval);
354 }
355
356 static int
fbt_ctfoff_init(modctl_t * lf,linker_ctf_t * lc)357 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
358 {
359 const Elf_Sym *symp = lc->symtab;
360 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
361 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
362 size_t idwidth;
363 int i;
364 uint32_t *ctfoff;
365 uint32_t objtoff = hp->cth_objtoff;
366 uint32_t funcoff = hp->cth_funcoff;
367 uint_t kind, info, vlen;
368
369 /* Sanity check. */
370 if (hp->cth_magic != CTF_MAGIC) {
371 printf("Bad magic value in CTF data of '%s'\n",lf->pathname);
372 return (EINVAL);
373 }
374
375 if (lc->symtab == NULL) {
376 printf("No symbol table in '%s'\n",lf->pathname);
377 return (EINVAL);
378 }
379
380 ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK);
381 *lc->ctfoffp = ctfoff;
382
383 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4;
384
385 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) {
386 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) {
387 *ctfoff = 0xffffffff;
388 continue;
389 }
390
391 switch (ELF_ST_TYPE(symp->st_info)) {
392 case STT_OBJECT:
393 if (objtoff >= hp->cth_funcoff ||
394 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) {
395 *ctfoff = 0xffffffff;
396 break;
397 }
398
399 *ctfoff = objtoff;
400 objtoff += idwidth;
401 break;
402
403 case STT_FUNC:
404 if (funcoff >= hp->cth_typeoff) {
405 *ctfoff = 0xffffffff;
406 break;
407 }
408
409 *ctfoff = funcoff;
410
411 info = 0;
412 memcpy(&info, ctfdata + funcoff, idwidth);
413 if (hp->cth_version == CTF_VERSION_2) {
414 kind = CTF_V2_INFO_KIND(info);
415 vlen = CTF_V2_INFO_VLEN(info);
416 } else {
417 kind = CTF_V3_INFO_KIND(info);
418 vlen = CTF_V3_INFO_VLEN(info);
419 }
420
421 /*
422 * If we encounter a zero pad at the end, just skip it.
423 * Otherwise skip over the function and its return type
424 * (+2) and the argument list (vlen).
425 */
426 if (kind == CTF_K_UNKNOWN && vlen == 0)
427 funcoff += idwidth;
428 else
429 funcoff += idwidth * (vlen + 2);
430 break;
431
432 default:
433 *ctfoff = 0xffffffff;
434 break;
435 }
436 }
437
438 return (0);
439 }
440
441 static void
fbt_get_ctt_index(uint8_t version,const void * v,uint_t * indexp,uint_t * typep,int * ischildp)442 fbt_get_ctt_index(uint8_t version, const void *v, uint_t *indexp,
443 uint_t *typep, int *ischildp)
444 {
445 uint_t index, type;
446 int ischild;
447
448 if (version == CTF_VERSION_2) {
449 const struct ctf_type_v2 *ctt = v;
450
451 type = ctt->ctt_type;
452 index = CTF_V2_TYPE_TO_INDEX(ctt->ctt_type);
453 ischild = CTF_V2_TYPE_ISCHILD(ctt->ctt_type);
454 } else {
455 const struct ctf_type_v3 *ctt = v;
456
457 type = ctt->ctt_type;
458 index = CTF_V3_TYPE_TO_INDEX(ctt->ctt_type);
459 ischild = CTF_V3_TYPE_ISCHILD(ctt->ctt_type);
460 }
461
462 if (indexp != NULL)
463 *indexp = index;
464 if (typep != NULL)
465 *typep = type;
466 if (ischildp != NULL)
467 *ischildp = ischild;
468 }
469
470 static ssize_t
fbt_get_ctt_size(uint8_t version,const void * tp,ssize_t * sizep,ssize_t * incrementp)471 fbt_get_ctt_size(uint8_t version, const void *tp, ssize_t *sizep,
472 ssize_t *incrementp)
473 {
474 ssize_t size, increment;
475
476 if (version == CTF_VERSION_2) {
477 const struct ctf_type_v2 *ctt = tp;
478
479 if (ctt->ctt_size == CTF_V2_LSIZE_SENT) {
480 size = CTF_TYPE_LSIZE(ctt);
481 increment = sizeof (struct ctf_type_v2);
482 } else {
483 size = ctt->ctt_size;
484 increment = sizeof (struct ctf_stype_v2);
485 }
486 } else {
487 const struct ctf_type_v3 *ctt = tp;
488
489 if (ctt->ctt_size == CTF_V3_LSIZE_SENT) {
490 size = CTF_TYPE_LSIZE(ctt);
491 increment = sizeof (struct ctf_type_v3);
492 } else {
493 size = ctt->ctt_size;
494 increment = sizeof (struct ctf_stype_v3);
495 }
496 }
497
498 if (sizep)
499 *sizep = size;
500 if (incrementp)
501 *incrementp = increment;
502
503 return (size);
504 }
505
506 static void
fbt_get_ctt_info(uint8_t version,const void * tp,uint_t * kindp,uint_t * vlenp,int * isrootp)507 fbt_get_ctt_info(uint8_t version, const void *tp, uint_t *kindp, uint_t *vlenp,
508 int *isrootp)
509 {
510 uint_t kind, vlen;
511 int isroot;
512
513 if (version == CTF_VERSION_2) {
514 const struct ctf_type_v2 *ctt = tp;
515
516 kind = CTF_V2_INFO_KIND(ctt->ctt_info);
517 vlen = CTF_V2_INFO_VLEN(ctt->ctt_info);
518 isroot = CTF_V2_INFO_ISROOT(ctt->ctt_info);
519 } else {
520 const struct ctf_type_v3 *ctt = tp;
521
522 kind = CTF_V3_INFO_KIND(ctt->ctt_info);
523 vlen = CTF_V3_INFO_VLEN(ctt->ctt_info);
524 isroot = CTF_V3_INFO_ISROOT(ctt->ctt_info);
525 }
526
527 if (kindp != NULL)
528 *kindp = kind;
529 if (vlenp != NULL)
530 *vlenp = vlen;
531 if (isrootp != NULL)
532 *isrootp = isroot;
533 }
534
535 static int
fbt_typoff_init(linker_ctf_t * lc)536 fbt_typoff_init(linker_ctf_t *lc)
537 {
538 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
539 const void *tbuf, *tend, *tp;
540 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
541 size_t idwidth;
542 int ctf_typemax = 0;
543 uint32_t *xp;
544 ulong_t pop[CTF_K_MAX + 1] = { 0 };
545 uint8_t version;
546
547 /* Sanity check. */
548 if (hp->cth_magic != CTF_MAGIC)
549 return (EINVAL);
550
551 version = hp->cth_version;
552 idwidth = version == CTF_VERSION_2 ? 2 : 4;
553
554 tbuf = (const void *) (ctfdata + hp->cth_typeoff);
555 tend = (const void *) (ctfdata + hp->cth_stroff);
556
557 /*
558 * We make two passes through the entire type section. In this first
559 * pass, we count the number of each type and the total number of types.
560 */
561 for (tp = tbuf; tp < tend; ctf_typemax++) {
562 uint_t kind, type, vlen;
563 ssize_t size, increment;
564 size_t vbytes;
565
566 (void) fbt_get_ctt_size(version, tp, &size, &increment);
567 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL);
568 fbt_get_ctt_index(version, tp, NULL, &type, NULL);
569
570 switch (kind) {
571 case CTF_K_INTEGER:
572 case CTF_K_FLOAT:
573 vbytes = sizeof (uint_t);
574 break;
575 case CTF_K_ARRAY:
576 if (version == CTF_VERSION_2)
577 vbytes = sizeof (struct ctf_array_v2);
578 else
579 vbytes = sizeof (struct ctf_array_v3);
580 break;
581 case CTF_K_FUNCTION:
582 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t));
583 break;
584 case CTF_K_STRUCT:
585 case CTF_K_UNION:
586 if (version == CTF_VERSION_2) {
587 if (size < CTF_V2_LSTRUCT_THRESH)
588 vbytes =
589 sizeof (struct ctf_member_v2) * vlen;
590 else
591 vbytes =
592 sizeof (struct ctf_lmember_v2) * vlen;
593 } else {
594 if (size < CTF_V3_LSTRUCT_THRESH)
595 vbytes =
596 sizeof (struct ctf_member_v3) * vlen;
597 else
598 vbytes =
599 sizeof (struct ctf_lmember_v3) * vlen;
600 }
601 break;
602 case CTF_K_ENUM:
603 vbytes = sizeof (ctf_enum_t) * vlen;
604 break;
605 case CTF_K_FORWARD:
606 /*
607 * For forward declarations, ctt_type is the CTF_K_*
608 * kind for the tag, so bump that population count too.
609 * If ctt_type is unknown, treat the tag as a struct.
610 */
611 if (type == CTF_K_UNKNOWN || type >= CTF_K_MAX)
612 pop[CTF_K_STRUCT]++;
613 else
614 pop[type]++;
615 /*FALLTHRU*/
616 case CTF_K_UNKNOWN:
617 vbytes = 0;
618 break;
619 case CTF_K_POINTER:
620 case CTF_K_TYPEDEF:
621 case CTF_K_VOLATILE:
622 case CTF_K_CONST:
623 case CTF_K_RESTRICT:
624 vbytes = 0;
625 break;
626 default:
627 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
628 return (EIO);
629 }
630 tp = (const void *)((uintptr_t)tp + increment + vbytes);
631 pop[kind]++;
632 }
633
634 /* account for a sentinel value below */
635 ctf_typemax++;
636 *lc->typlenp = ctf_typemax;
637
638 xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER,
639 M_ZERO | M_WAITOK);
640
641 *lc->typoffp = xp;
642
643 /* type id 0 is used as a sentinel value */
644 *xp++ = 0;
645
646 /*
647 * In the second pass, fill in the type offset.
648 */
649 for (tp = tbuf; tp < tend; xp++) {
650 ssize_t size, increment;
651 uint_t kind, vlen;
652
653 size_t vbytes;
654
655 (void) fbt_get_ctt_size(version, tp, &size, &increment);
656 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL);
657
658 switch (kind) {
659 case CTF_K_INTEGER:
660 case CTF_K_FLOAT:
661 vbytes = sizeof (uint_t);
662 break;
663 case CTF_K_ARRAY:
664 if (version == CTF_VERSION_2)
665 vbytes = sizeof (struct ctf_array_v2);
666 else
667 vbytes = sizeof (struct ctf_array_v3);
668 break;
669 case CTF_K_FUNCTION:
670 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t));
671 break;
672 case CTF_K_STRUCT:
673 case CTF_K_UNION:
674 if (version == CTF_VERSION_2) {
675 if (size < CTF_V2_LSTRUCT_THRESH)
676 vbytes =
677 sizeof (struct ctf_member_v2) * vlen;
678 else
679 vbytes =
680 sizeof (struct ctf_lmember_v2) * vlen;
681 } else {
682 if (size < CTF_V3_LSTRUCT_THRESH)
683 vbytes =
684 sizeof (struct ctf_member_v3) * vlen;
685 else
686 vbytes =
687 sizeof (struct ctf_lmember_v3) * vlen;
688 }
689 break;
690 case CTF_K_ENUM:
691 vbytes = sizeof (ctf_enum_t) * vlen;
692 break;
693 case CTF_K_FORWARD:
694 case CTF_K_UNKNOWN:
695 vbytes = 0;
696 break;
697 case CTF_K_POINTER:
698 case CTF_K_TYPEDEF:
699 case CTF_K_VOLATILE:
700 case CTF_K_CONST:
701 case CTF_K_RESTRICT:
702 vbytes = 0;
703 break;
704 default:
705 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
706 return (EIO);
707 }
708 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata);
709 tp = (const void *)((uintptr_t)tp + increment + vbytes);
710 }
711
712 return (0);
713 }
714
715 /*
716 * CTF Declaration Stack
717 *
718 * In order to implement ctf_type_name(), we must convert a type graph back
719 * into a C type declaration. Unfortunately, a type graph represents a storage
720 * class ordering of the type whereas a type declaration must obey the C rules
721 * for operator precedence, and the two orderings are frequently in conflict.
722 * For example, consider these CTF type graphs and their C declarations:
723 *
724 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)()
725 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[]
726 *
727 * In each case, parentheses are used to raise operator * to higher lexical
728 * precedence, so the string form of the C declaration cannot be constructed by
729 * walking the type graph links and forming the string from left to right.
730 *
731 * The functions in this file build a set of stacks from the type graph nodes
732 * corresponding to the C operator precedence levels in the appropriate order.
733 * The code in ctf_type_name() can then iterate over the levels and nodes in
734 * lexical precedence order and construct the final C declaration string.
735 */
736 typedef struct ctf_list {
737 struct ctf_list *l_prev; /* previous pointer or tail pointer */
738 struct ctf_list *l_next; /* next pointer or head pointer */
739 } ctf_list_t;
740
741 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev))
742 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next))
743
744 typedef enum {
745 CTF_PREC_BASE,
746 CTF_PREC_POINTER,
747 CTF_PREC_ARRAY,
748 CTF_PREC_FUNCTION,
749 CTF_PREC_MAX
750 } ctf_decl_prec_t;
751
752 typedef struct ctf_decl_node {
753 ctf_list_t cd_list; /* linked list pointers */
754 ctf_id_t cd_type; /* type identifier */
755 uint_t cd_kind; /* type kind */
756 uint_t cd_n; /* type dimension if array */
757 } ctf_decl_node_t;
758
759 typedef struct ctf_decl {
760 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */
761 int cd_order[CTF_PREC_MAX]; /* storage order of decls */
762 ctf_decl_prec_t cd_qualp; /* qualifier precision */
763 ctf_decl_prec_t cd_ordp; /* ordered precision */
764 char *cd_buf; /* buffer for output */
765 char *cd_ptr; /* buffer location */
766 char *cd_end; /* buffer limit */
767 size_t cd_len; /* buffer space required */
768 int cd_err; /* saved error value */
769 } ctf_decl_t;
770
771 /*
772 * Simple doubly-linked list append routine. This implementation assumes that
773 * each list element contains an embedded ctf_list_t as the first member.
774 * An additional ctf_list_t is used to store the head (l_next) and tail
775 * (l_prev) pointers. The current head and tail list elements have their
776 * previous and next pointers set to NULL, respectively.
777 */
778 static void
ctf_list_append(ctf_list_t * lp,void * new)779 ctf_list_append(ctf_list_t *lp, void *new)
780 {
781 ctf_list_t *p = lp->l_prev; /* p = tail list element */
782 ctf_list_t *q = new; /* q = new list element */
783
784 lp->l_prev = q;
785 q->l_prev = p;
786 q->l_next = NULL;
787
788 if (p != NULL)
789 p->l_next = q;
790 else
791 lp->l_next = q;
792 }
793
794 /*
795 * Prepend the specified existing element to the given ctf_list_t. The
796 * existing pointer should be pointing at a struct with embedded ctf_list_t.
797 */
798 static void
ctf_list_prepend(ctf_list_t * lp,void * new)799 ctf_list_prepend(ctf_list_t *lp, void *new)
800 {
801 ctf_list_t *p = new; /* p = new list element */
802 ctf_list_t *q = lp->l_next; /* q = head list element */
803
804 lp->l_next = p;
805 p->l_prev = NULL;
806 p->l_next = q;
807
808 if (q != NULL)
809 q->l_prev = p;
810 else
811 lp->l_prev = p;
812 }
813
814 static void
ctf_decl_init(ctf_decl_t * cd,char * buf,size_t len)815 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len)
816 {
817 int i;
818
819 bzero(cd, sizeof (ctf_decl_t));
820
821 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++)
822 cd->cd_order[i] = CTF_PREC_BASE - 1;
823
824 cd->cd_qualp = CTF_PREC_BASE;
825 cd->cd_ordp = CTF_PREC_BASE;
826
827 cd->cd_buf = buf;
828 cd->cd_ptr = buf;
829 cd->cd_end = buf + len;
830 }
831
832 static void
ctf_decl_fini(ctf_decl_t * cd)833 ctf_decl_fini(ctf_decl_t *cd)
834 {
835 ctf_decl_node_t *cdp, *ndp;
836 int i;
837
838 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) {
839 for (cdp = ctf_list_next(&cd->cd_nodes[i]);
840 cdp != NULL; cdp = ndp) {
841 ndp = ctf_list_next(cdp);
842 free(cdp, M_FBT);
843 }
844 }
845 }
846
847 static const void *
ctf_lookup_by_id(linker_ctf_t * lc,ctf_id_t type)848 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type)
849 {
850 const void *tp;
851 uint32_t offset;
852 uint32_t *typoff = *lc->typoffp;
853
854 if (type >= *lc->typlenp) {
855 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp);
856 return(NULL);
857 }
858
859 /* Check if the type isn't cross-referenced. */
860 if ((offset = typoff[type]) == 0) {
861 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type);
862 return(NULL);
863 }
864
865 tp = (const void *) (lc->ctftab + offset + sizeof(ctf_header_t));
866
867 return (tp);
868 }
869
870 static void
fbt_array_info(linker_ctf_t * lc,ctf_id_t type,ctf_arinfo_t * arp)871 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp)
872 {
873 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
874 const void *tp;
875 ssize_t increment;
876 uint_t kind;
877
878 bzero(arp, sizeof(*arp));
879
880 if ((tp = ctf_lookup_by_id(lc, type)) == NULL)
881 return;
882
883 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL);
884 if (kind != CTF_K_ARRAY)
885 return;
886
887 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment);
888
889 if (hp->cth_version == CTF_VERSION_2) {
890 const struct ctf_array_v2 *ap;
891
892 ap = (const struct ctf_array_v2 *)((uintptr_t)tp + increment);
893 arp->ctr_contents = ap->cta_contents;
894 arp->ctr_index = ap->cta_index;
895 arp->ctr_nelems = ap->cta_nelems;
896 } else {
897 const struct ctf_array_v3 *ap;
898
899 ap = (const struct ctf_array_v3 *)((uintptr_t)tp + increment);
900 arp->ctr_contents = ap->cta_contents;
901 arp->ctr_index = ap->cta_index;
902 arp->ctr_nelems = ap->cta_nelems;
903 }
904 }
905
906 static const char *
ctf_strptr(linker_ctf_t * lc,int name)907 ctf_strptr(linker_ctf_t *lc, int name)
908 {
909 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
910 const char *strp = "";
911
912 if (name < 0 || name >= hp->cth_strlen)
913 return(strp);
914
915 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t));
916
917 return (strp);
918 }
919
920 static const char *
ctf_type_rname(linker_ctf_t * lc,const void * v)921 ctf_type_rname(linker_ctf_t *lc, const void *v)
922 {
923 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
924 uint_t name;
925
926 if (hp->cth_version == CTF_VERSION_2) {
927 const struct ctf_type_v2 *ctt = v;
928
929 name = ctt->ctt_name;
930 } else {
931 const struct ctf_type_v3 *ctt = v;
932
933 name = ctt->ctt_name;
934 }
935
936 return (ctf_strptr(lc, name));
937 }
938
939 static void
ctf_decl_push(ctf_decl_t * cd,linker_ctf_t * lc,ctf_id_t type)940 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type)
941 {
942 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
943 ctf_decl_node_t *cdp;
944 ctf_decl_prec_t prec;
945 uint_t kind, n = 1, t;
946 int is_qual = 0;
947
948 const void *tp;
949 ctf_arinfo_t ar;
950
951 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) {
952 cd->cd_err = ENOENT;
953 return;
954 }
955
956 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL);
957 fbt_get_ctt_index(hp->cth_version, tp, NULL, &t, NULL);
958
959 switch (kind) {
960 case CTF_K_ARRAY:
961 fbt_array_info(lc, type, &ar);
962 ctf_decl_push(cd, lc, ar.ctr_contents);
963 n = ar.ctr_nelems;
964 prec = CTF_PREC_ARRAY;
965 break;
966
967 case CTF_K_TYPEDEF:
968 if (ctf_type_rname(lc, tp)[0] == '\0') {
969 ctf_decl_push(cd, lc, t);
970 return;
971 }
972 prec = CTF_PREC_BASE;
973 break;
974
975 case CTF_K_FUNCTION:
976 ctf_decl_push(cd, lc, t);
977 prec = CTF_PREC_FUNCTION;
978 break;
979
980 case CTF_K_POINTER:
981 ctf_decl_push(cd, lc, t);
982 prec = CTF_PREC_POINTER;
983 break;
984
985 case CTF_K_VOLATILE:
986 case CTF_K_CONST:
987 case CTF_K_RESTRICT:
988 ctf_decl_push(cd, lc, t);
989 prec = cd->cd_qualp;
990 is_qual++;
991 break;
992
993 default:
994 prec = CTF_PREC_BASE;
995 }
996
997 cdp = malloc(sizeof(*cdp), M_FBT, M_WAITOK);
998 cdp->cd_type = type;
999 cdp->cd_kind = kind;
1000 cdp->cd_n = n;
1001
1002 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL)
1003 cd->cd_order[prec] = cd->cd_ordp++;
1004
1005 /*
1006 * Reset cd_qualp to the highest precedence level that we've seen so
1007 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER).
1008 */
1009 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY)
1010 cd->cd_qualp = prec;
1011
1012 /*
1013 * C array declarators are ordered inside out so prepend them. Also by
1014 * convention qualifiers of base types precede the type specifier (e.g.
1015 * const int vs. int const) even though the two forms are equivalent.
1016 */
1017 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE))
1018 ctf_list_prepend(&cd->cd_nodes[prec], cdp);
1019 else
1020 ctf_list_append(&cd->cd_nodes[prec], cdp);
1021 }
1022
1023 static void
ctf_decl_sprintf(ctf_decl_t * cd,const char * format,...)1024 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...)
1025 {
1026 size_t len = (size_t)(cd->cd_end - cd->cd_ptr);
1027 va_list ap;
1028 size_t n;
1029
1030 va_start(ap, format);
1031 n = vsnprintf(cd->cd_ptr, len, format, ap);
1032 va_end(ap);
1033
1034 cd->cd_ptr += MIN(n, len);
1035 cd->cd_len += n;
1036 }
1037
1038 static ssize_t
fbt_type_name(linker_ctf_t * lc,ctf_id_t type,char * buf,size_t len)1039 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len)
1040 {
1041 ctf_decl_t cd;
1042 ctf_decl_node_t *cdp;
1043 ctf_decl_prec_t prec, lp, rp;
1044 int ptr, arr;
1045 uint_t k;
1046
1047 if (lc == NULL && type == CTF_ERR)
1048 return (-1); /* simplify caller code by permitting CTF_ERR */
1049
1050 ctf_decl_init(&cd, buf, len);
1051 ctf_decl_push(&cd, lc, type);
1052
1053 if (cd.cd_err != 0) {
1054 ctf_decl_fini(&cd);
1055 return (-1);
1056 }
1057
1058 /*
1059 * If the type graph's order conflicts with lexical precedence order
1060 * for pointers or arrays, then we need to surround the declarations at
1061 * the corresponding lexical precedence with parentheses. This can
1062 * result in either a parenthesized pointer (*) as in int (*)() or
1063 * int (*)[], or in a parenthesized pointer and array as in int (*[])().
1064 */
1065 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER;
1066 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY;
1067
1068 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1;
1069 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1;
1070
1071 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */
1072
1073 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) {
1074 for (cdp = ctf_list_next(&cd.cd_nodes[prec]);
1075 cdp != NULL; cdp = ctf_list_next(cdp)) {
1076
1077 const void *tp = ctf_lookup_by_id(lc, cdp->cd_type);
1078 const char *name = ctf_type_rname(lc, tp);
1079
1080 if (k != CTF_K_POINTER && k != CTF_K_ARRAY)
1081 ctf_decl_sprintf(&cd, " ");
1082
1083 if (lp == prec) {
1084 ctf_decl_sprintf(&cd, "(");
1085 lp = -1;
1086 }
1087
1088 switch (cdp->cd_kind) {
1089 case CTF_K_INTEGER:
1090 case CTF_K_FLOAT:
1091 case CTF_K_TYPEDEF:
1092 ctf_decl_sprintf(&cd, "%s", name);
1093 break;
1094 case CTF_K_POINTER:
1095 ctf_decl_sprintf(&cd, "*");
1096 break;
1097 case CTF_K_ARRAY:
1098 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n);
1099 break;
1100 case CTF_K_FUNCTION:
1101 ctf_decl_sprintf(&cd, "()");
1102 break;
1103 case CTF_K_STRUCT:
1104 case CTF_K_FORWARD:
1105 ctf_decl_sprintf(&cd, "struct %s", name);
1106 break;
1107 case CTF_K_UNION:
1108 ctf_decl_sprintf(&cd, "union %s", name);
1109 break;
1110 case CTF_K_ENUM:
1111 ctf_decl_sprintf(&cd, "enum %s", name);
1112 break;
1113 case CTF_K_VOLATILE:
1114 ctf_decl_sprintf(&cd, "volatile");
1115 break;
1116 case CTF_K_CONST:
1117 ctf_decl_sprintf(&cd, "const");
1118 break;
1119 case CTF_K_RESTRICT:
1120 ctf_decl_sprintf(&cd, "restrict");
1121 break;
1122 }
1123
1124 k = cdp->cd_kind;
1125 }
1126
1127 if (rp == prec)
1128 ctf_decl_sprintf(&cd, ")");
1129 }
1130
1131 ctf_decl_fini(&cd);
1132 return (cd.cd_len);
1133 }
1134
1135 static void
fbt_getargdesc(void * arg __unused,dtrace_id_t id __unused,void * parg,dtrace_argdesc_t * desc)1136 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc)
1137 {
1138 const ctf_header_t *hp;
1139 const char *dp;
1140 fbt_probe_t *fbt = parg;
1141 linker_ctf_t lc;
1142 modctl_t *ctl = fbt->fbtp_ctl;
1143 size_t idwidth;
1144 int ndx = desc->dtargd_ndx;
1145 int symindx = fbt->fbtp_symindx;
1146 uint32_t *ctfoff;
1147 uint32_t offset, type;
1148 uint_t info, n;
1149 ushort_t kind;
1150
1151 if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) {
1152 (void) strcpy(desc->dtargd_native, "int");
1153 return;
1154 }
1155
1156 desc->dtargd_ndx = DTRACE_ARGNONE;
1157
1158 /* Get a pointer to the CTF data and it's length. */
1159 if (linker_ctf_get(ctl, &lc) != 0)
1160 /* No CTF data? Something wrong? *shrug* */
1161 return;
1162
1163 /* Check if this module hasn't been initialised yet. */
1164 if (*lc.ctfoffp == NULL) {
1165 /*
1166 * Initialise the CTF object and function symindx to
1167 * byte offset array.
1168 */
1169 if (fbt_ctfoff_init(ctl, &lc) != 0)
1170 return;
1171
1172 /* Initialise the CTF type to byte offset array. */
1173 if (fbt_typoff_init(&lc) != 0)
1174 return;
1175 }
1176
1177 ctfoff = *lc.ctfoffp;
1178
1179 if (ctfoff == NULL || *lc.typoffp == NULL)
1180 return;
1181
1182 /* Check if the symbol index is out of range. */
1183 if (symindx >= lc.nsym)
1184 return;
1185
1186 /* Check if the symbol isn't cross-referenced. */
1187 if ((offset = ctfoff[symindx]) == 0xffffffff)
1188 return;
1189
1190 hp = (const ctf_header_t *) lc.ctftab;
1191 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4;
1192 dp = (const char *)(lc.ctftab + offset + sizeof(ctf_header_t));
1193
1194 info = 0;
1195 memcpy(&info, dp, idwidth);
1196 dp += idwidth;
1197 if (hp->cth_version == CTF_VERSION_2) {
1198 kind = CTF_V2_INFO_KIND(info);
1199 n = CTF_V2_INFO_VLEN(info);
1200 } else {
1201 kind = CTF_V3_INFO_KIND(info);
1202 n = CTF_V3_INFO_VLEN(info);
1203 }
1204
1205 if (kind == CTF_K_UNKNOWN && n == 0) {
1206 printf("%s(%d): Unknown function!\n",__func__,__LINE__);
1207 return;
1208 }
1209
1210 if (kind != CTF_K_FUNCTION) {
1211 printf("%s(%d): Expected a function!\n",__func__,__LINE__);
1212 return;
1213 }
1214
1215 if (fbt->fbtp_roffset != 0) {
1216 /* Only return type is available for args[1] in return probe. */
1217 if (ndx > 1)
1218 return;
1219 ASSERT(ndx == 1);
1220 } else {
1221 /* Check if the requested argument doesn't exist. */
1222 if (ndx >= n)
1223 return;
1224
1225 /* Skip the return type and arguments up to the one requested. */
1226 dp += idwidth * (ndx + 1);
1227 }
1228
1229 type = 0;
1230 memcpy(&type, dp, idwidth);
1231 if (fbt_type_name(&lc, type, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0)
1232 desc->dtargd_ndx = ndx;
1233 }
1234
1235 static int
fbt_linker_file_cb(linker_file_t lf,void * arg)1236 fbt_linker_file_cb(linker_file_t lf, void *arg)
1237 {
1238
1239 fbt_provide_module(arg, lf);
1240
1241 return (0);
1242 }
1243
1244 static void
fbt_load(void * dummy)1245 fbt_load(void *dummy)
1246 {
1247 /* Default the probe table size if not specified. */
1248 if (fbt_probetab_size == 0)
1249 fbt_probetab_size = FBT_PROBETAB_SIZE;
1250
1251 /* Choose the hash mask for the probe table. */
1252 fbt_probetab_mask = fbt_probetab_size - 1;
1253
1254 /* Allocate memory for the probe table. */
1255 fbt_probetab =
1256 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO);
1257
1258 dtrace_doubletrap_func = fbt_doubletrap;
1259 dtrace_invop_add(fbt_invop);
1260
1261 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
1262 NULL, &fbt_pops, NULL, &fbt_id) != 0)
1263 return;
1264
1265 /* Create probes for the kernel and already-loaded modules. */
1266 linker_file_foreach(fbt_linker_file_cb, NULL);
1267 }
1268
1269 static int
fbt_unload(void)1270 fbt_unload(void)
1271 {
1272 int error = 0;
1273
1274 /* De-register the invalid opcode handler. */
1275 dtrace_invop_remove(fbt_invop);
1276
1277 dtrace_doubletrap_func = NULL;
1278
1279 /* De-register this DTrace provider. */
1280 if ((error = dtrace_unregister(fbt_id)) != 0)
1281 return (error);
1282
1283 /* Free the probe table. */
1284 free(fbt_probetab, M_FBT);
1285 fbt_probetab = NULL;
1286 fbt_probetab_mask = 0;
1287
1288 return (error);
1289 }
1290
1291 static int
fbt_modevent(module_t mod __unused,int type,void * data __unused)1292 fbt_modevent(module_t mod __unused, int type, void *data __unused)
1293 {
1294 int error = 0;
1295
1296 switch (type) {
1297 case MOD_LOAD:
1298 break;
1299
1300 case MOD_UNLOAD:
1301 break;
1302
1303 case MOD_SHUTDOWN:
1304 break;
1305
1306 default:
1307 error = EOPNOTSUPP;
1308 break;
1309
1310 }
1311
1312 return (error);
1313 }
1314
1315 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL);
1316 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL);
1317
1318 DEV_MODULE(fbt, fbt_modevent, NULL);
1319 MODULE_VERSION(fbt, 1);
1320 MODULE_DEPEND(fbt, dtrace, 1, 1, 1);
1321 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1);
1322