1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 * 23 * $FreeBSD$ 24 */ 25 26 /* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31 /* 32 * Copyright (c) 2015, Joyent, Inc. All rights reserved. 33 */ 34 35 #include <sys/atomic.h> 36 #include <sys/errno.h> 37 #include <sys/stat.h> 38 #include <sys/modctl.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #ifdef illumos 42 #include <sys/ddi.h> 43 #endif 44 #include <sys/sunddi.h> 45 #include <sys/cpuvar.h> 46 #include <sys/kmem.h> 47 #ifdef illumos 48 #include <sys/strsubr.h> 49 #endif 50 #include <sys/fasttrap.h> 51 #include <sys/fasttrap_impl.h> 52 #include <sys/fasttrap_isa.h> 53 #include <sys/dtrace.h> 54 #include <sys/dtrace_impl.h> 55 #include <sys/sysmacros.h> 56 #include <sys/proc.h> 57 #include <sys/policy.h> 58 #ifdef illumos 59 #include <util/qsort.h> 60 #endif 61 #include <sys/mutex.h> 62 #include <sys/kernel.h> 63 #ifndef illumos 64 #include <sys/dtrace_bsd.h> 65 #include <sys/eventhandler.h> 66 #include <sys/rmlock.h> 67 #include <sys/sysent.h> 68 #include <sys/sysctl.h> 69 #include <sys/u8_textprep.h> 70 #include <sys/user.h> 71 72 #include <vm/vm.h> 73 #include <vm/pmap.h> 74 #include <vm/vm_map.h> 75 #include <vm/vm_param.h> 76 77 #include <cddl/dev/dtrace/dtrace_cddl.h> 78 #endif 79 80 /* 81 * User-Land Trap-Based Tracing 82 * ---------------------------- 83 * 84 * The fasttrap provider allows DTrace consumers to instrument any user-level 85 * instruction to gather data; this includes probes with semantic 86 * signifigance like entry and return as well as simple offsets into the 87 * function. While the specific techniques used are very ISA specific, the 88 * methodology is generalizable to any architecture. 89 * 90 * 91 * The General Methodology 92 * ----------------------- 93 * 94 * With the primary goal of tracing every user-land instruction and the 95 * limitation that we can't trust user space so don't want to rely on much 96 * information there, we begin by replacing the instructions we want to trace 97 * with trap instructions. Each instruction we overwrite is saved into a hash 98 * table keyed by process ID and pc address. When we enter the kernel due to 99 * this trap instruction, we need the effects of the replaced instruction to 100 * appear to have occurred before we proceed with the user thread's 101 * execution. 102 * 103 * Each user level thread is represented by a ulwp_t structure which is 104 * always easily accessible through a register. The most basic way to produce 105 * the effects of the instruction we replaced is to copy that instruction out 106 * to a bit of scratch space reserved in the user thread's ulwp_t structure 107 * (a sort of kernel-private thread local storage), set the PC to that 108 * scratch space and single step. When we reenter the kernel after single 109 * stepping the instruction we must then adjust the PC to point to what would 110 * normally be the next instruction. Of course, special care must be taken 111 * for branches and jumps, but these represent such a small fraction of any 112 * instruction set that writing the code to emulate these in the kernel is 113 * not too difficult. 114 * 115 * Return probes may require several tracepoints to trace every return site, 116 * and, conversely, each tracepoint may activate several probes (the entry 117 * and offset 0 probes, for example). To solve this muliplexing problem, 118 * tracepoints contain lists of probes to activate and probes contain lists 119 * of tracepoints to enable. If a probe is activated, it adds its ID to 120 * existing tracepoints or creates new ones as necessary. 121 * 122 * Most probes are activated _before_ the instruction is executed, but return 123 * probes are activated _after_ the effects of the last instruction of the 124 * function are visible. Return probes must be fired _after_ we have 125 * single-stepped the instruction whereas all other probes are fired 126 * beforehand. 127 * 128 * 129 * Lock Ordering 130 * ------------- 131 * 132 * The lock ordering below -- both internally and with respect to the DTrace 133 * framework -- is a little tricky and bears some explanation. Each provider 134 * has a lock (ftp_mtx) that protects its members including reference counts 135 * for enabled probes (ftp_rcount), consumers actively creating probes 136 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 137 * from being freed. A provider is looked up by taking the bucket lock for the 138 * provider hash table, and is returned with its lock held. The provider lock 139 * may be taken in functions invoked by the DTrace framework, but may not be 140 * held while calling functions in the DTrace framework. 141 * 142 * To ensure consistency over multiple calls to the DTrace framework, the 143 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 144 * not be taken when holding the provider lock as that would create a cyclic 145 * lock ordering. In situations where one would naturally take the provider 146 * lock and then the creation lock, we instead up a reference count to prevent 147 * the provider from disappearing, drop the provider lock, and acquire the 148 * creation lock. 149 * 150 * Briefly: 151 * bucket lock before provider lock 152 * DTrace before provider lock 153 * creation lock before DTrace 154 * never hold the provider lock and creation lock simultaneously 155 */ 156 157 static d_open_t fasttrap_open; 158 static d_ioctl_t fasttrap_ioctl; 159 160 static struct cdevsw fasttrap_cdevsw = { 161 .d_version = D_VERSION, 162 .d_open = fasttrap_open, 163 .d_ioctl = fasttrap_ioctl, 164 .d_name = "fasttrap", 165 }; 166 static struct cdev *fasttrap_cdev; 167 static dtrace_meta_provider_id_t fasttrap_meta_id; 168 169 static struct proc *fasttrap_cleanup_proc; 170 static struct mtx fasttrap_cleanup_mtx; 171 static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 172 173 /* 174 * Generation count on modifications to the global tracepoint lookup table. 175 */ 176 static volatile uint64_t fasttrap_mod_gen; 177 178 /* 179 * When the fasttrap provider is loaded, fasttrap_max is set to either 180 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 181 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 182 * Each time a probe is created, fasttrap_total is incremented by the number 183 * of tracepoints that may be associated with that probe; fasttrap_total is capped 184 * at fasttrap_max. 185 */ 186 #define FASTTRAP_MAX_DEFAULT 250000 187 static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 188 static uint32_t fasttrap_total; 189 190 /* 191 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 192 */ 193 194 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 195 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 196 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 197 198 #define FASTTRAP_PID_NAME "pid" 199 200 fasttrap_hash_t fasttrap_tpoints; 201 static fasttrap_hash_t fasttrap_provs; 202 static fasttrap_hash_t fasttrap_procs; 203 204 static uint64_t fasttrap_pid_count; /* pid ref count */ 205 static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 206 207 #define FASTTRAP_ENABLE_FAIL 1 208 #define FASTTRAP_ENABLE_PARTIAL 2 209 210 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 211 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 212 213 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 214 const dtrace_pattr_t *); 215 static void fasttrap_provider_retire(pid_t, const char *, int); 216 static void fasttrap_provider_free(fasttrap_provider_t *); 217 218 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 219 static void fasttrap_proc_release(fasttrap_proc_t *); 220 221 #ifndef illumos 222 static void fasttrap_thread_dtor(void *, struct thread *); 223 #endif 224 225 #define FASTTRAP_PROVS_INDEX(pid, name) \ 226 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 227 228 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 229 230 #ifndef illumos 231 struct rmlock fasttrap_tp_lock; 232 static eventhandler_tag fasttrap_thread_dtor_tag; 233 #endif 234 235 static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 236 237 #ifdef __FreeBSD__ 238 SYSCTL_DECL(_kern_dtrace); 239 SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters"); 240 SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 241 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 242 SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 243 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 244 #endif 245 246 static int 247 fasttrap_highbit(ulong_t i) 248 { 249 int h = 1; 250 251 if (i == 0) 252 return (0); 253 #ifdef _LP64 254 if (i & 0xffffffff00000000ul) { 255 h += 32; i >>= 32; 256 } 257 #endif 258 if (i & 0xffff0000) { 259 h += 16; i >>= 16; 260 } 261 if (i & 0xff00) { 262 h += 8; i >>= 8; 263 } 264 if (i & 0xf0) { 265 h += 4; i >>= 4; 266 } 267 if (i & 0xc) { 268 h += 2; i >>= 2; 269 } 270 if (i & 0x2) { 271 h += 1; 272 } 273 return (h); 274 } 275 276 static uint_t 277 fasttrap_hash_str(const char *p) 278 { 279 unsigned int g; 280 uint_t hval = 0; 281 282 while (*p) { 283 hval = (hval << 4) + *p++; 284 if ((g = (hval & 0xf0000000)) != 0) 285 hval ^= g >> 24; 286 hval &= ~g; 287 } 288 return (hval); 289 } 290 291 void 292 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 293 { 294 #ifdef illumos 295 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 296 297 sqp->sq_info.si_signo = SIGTRAP; 298 sqp->sq_info.si_code = TRAP_DTRACE; 299 sqp->sq_info.si_addr = (caddr_t)pc; 300 301 mutex_enter(&p->p_lock); 302 sigaddqa(p, t, sqp); 303 mutex_exit(&p->p_lock); 304 305 if (t != NULL) 306 aston(t); 307 #else 308 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 309 310 ksiginfo_init(ksi); 311 ksi->ksi_signo = SIGTRAP; 312 ksi->ksi_code = TRAP_DTRACE; 313 ksi->ksi_addr = (caddr_t)pc; 314 PROC_LOCK(p); 315 (void) tdsendsignal(p, t, SIGTRAP, ksi); 316 PROC_UNLOCK(p); 317 #endif 318 } 319 320 #ifndef illumos 321 /* 322 * Obtain a chunk of scratch space in the address space of the target process. 323 */ 324 fasttrap_scrspace_t * 325 fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 326 { 327 fasttrap_scrblock_t *scrblk; 328 fasttrap_scrspace_t *scrspc; 329 struct proc *p; 330 vm_offset_t addr; 331 int error, i; 332 333 scrspc = NULL; 334 if (td->t_dtrace_sscr != NULL) { 335 /* If the thread already has scratch space, we're done. */ 336 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 337 return (scrspc); 338 } 339 340 p = td->td_proc; 341 342 mutex_enter(&fprc->ftpc_mtx); 343 if (LIST_EMPTY(&fprc->ftpc_fscr)) { 344 /* 345 * No scratch space is available, so we'll map a new scratch 346 * space block into the traced process' address space. 347 */ 348 addr = 0; 349 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 350 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 351 VM_PROT_ALL, 0); 352 if (error != KERN_SUCCESS) 353 goto done; 354 355 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 356 scrblk->ftsb_addr = addr; 357 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 358 359 /* 360 * Carve the block up into chunks and put them on the free list. 361 */ 362 for (i = 0; 363 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 364 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 365 scrspc->ftss_addr = addr + 366 i * FASTTRAP_SCRSPACE_SIZE; 367 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 368 ftss_next); 369 } 370 } 371 372 /* 373 * Take the first scratch chunk off the free list, put it on the 374 * allocated list, and return its address. 375 */ 376 scrspc = LIST_FIRST(&fprc->ftpc_fscr); 377 LIST_REMOVE(scrspc, ftss_next); 378 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 379 380 /* 381 * This scratch space is reserved for use by td until the thread exits. 382 */ 383 td->t_dtrace_sscr = scrspc; 384 385 done: 386 mutex_exit(&fprc->ftpc_mtx); 387 388 return (scrspc); 389 } 390 391 /* 392 * Return any allocated per-thread scratch space chunks back to the process' 393 * free list. 394 */ 395 static void 396 fasttrap_thread_dtor(void *arg __unused, struct thread *td) 397 { 398 fasttrap_bucket_t *bucket; 399 fasttrap_proc_t *fprc; 400 fasttrap_scrspace_t *scrspc; 401 pid_t pid; 402 403 if (td->t_dtrace_sscr == NULL) 404 return; 405 406 pid = td->td_proc->p_pid; 407 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 408 fprc = NULL; 409 410 /* Look up the fasttrap process handle for this process. */ 411 mutex_enter(&bucket->ftb_mtx); 412 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 413 if (fprc->ftpc_pid == pid) { 414 mutex_enter(&fprc->ftpc_mtx); 415 mutex_exit(&bucket->ftb_mtx); 416 break; 417 } 418 } 419 if (fprc == NULL) { 420 mutex_exit(&bucket->ftb_mtx); 421 return; 422 } 423 424 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 425 LIST_REMOVE(scrspc, ftss_next); 426 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 427 428 mutex_exit(&fprc->ftpc_mtx); 429 } 430 #endif 431 432 /* 433 * This function ensures that no threads are actively using the memory 434 * associated with probes that were formerly live. 435 */ 436 static void 437 fasttrap_mod_barrier(uint64_t gen) 438 { 439 int i; 440 441 if (gen < fasttrap_mod_gen) 442 return; 443 444 fasttrap_mod_gen++; 445 446 #ifdef illumos 447 CPU_FOREACH(i) { 448 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 449 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 450 } 451 #else 452 rm_wlock(&fasttrap_tp_lock); 453 rm_wunlock(&fasttrap_tp_lock); 454 #endif 455 } 456 457 /* 458 * This function performs asynchronous cleanup of fasttrap providers. The 459 * Solaris implementation of this mechanism use a timeout that's activated in 460 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 461 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 462 * Thus we use a dedicated process to perform the cleanup when requested. 463 */ 464 /*ARGSUSED*/ 465 static void 466 fasttrap_pid_cleanup_cb(void *data) 467 { 468 fasttrap_provider_t **fpp, *fp; 469 fasttrap_bucket_t *bucket; 470 dtrace_provider_id_t provid; 471 int i, later = 0, rval; 472 473 mtx_lock(&fasttrap_cleanup_mtx); 474 while (!fasttrap_cleanup_drain || later > 0) { 475 fasttrap_cleanup_work = 0; 476 mtx_unlock(&fasttrap_cleanup_mtx); 477 478 later = 0; 479 480 /* 481 * Iterate over all the providers trying to remove the marked 482 * ones. If a provider is marked but not retired, we just 483 * have to take a crack at removing it -- it's no big deal if 484 * we can't. 485 */ 486 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 487 bucket = &fasttrap_provs.fth_table[i]; 488 mutex_enter(&bucket->ftb_mtx); 489 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 490 491 while ((fp = *fpp) != NULL) { 492 if (!fp->ftp_marked) { 493 fpp = &fp->ftp_next; 494 continue; 495 } 496 497 mutex_enter(&fp->ftp_mtx); 498 499 /* 500 * If this provider has consumers actively 501 * creating probes (ftp_ccount) or is a USDT 502 * provider (ftp_mcount), we can't unregister 503 * or even condense. 504 */ 505 if (fp->ftp_ccount != 0 || 506 fp->ftp_mcount != 0) { 507 mutex_exit(&fp->ftp_mtx); 508 fp->ftp_marked = 0; 509 continue; 510 } 511 512 if (!fp->ftp_retired || fp->ftp_rcount != 0) 513 fp->ftp_marked = 0; 514 515 mutex_exit(&fp->ftp_mtx); 516 517 /* 518 * If we successfully unregister this 519 * provider we can remove it from the hash 520 * chain and free the memory. If our attempt 521 * to unregister fails and this is a retired 522 * provider, increment our flag to try again 523 * pretty soon. If we've consumed more than 524 * half of our total permitted number of 525 * probes call dtrace_condense() to try to 526 * clean out the unenabled probes. 527 */ 528 provid = fp->ftp_provid; 529 if ((rval = dtrace_unregister(provid)) != 0) { 530 if (fasttrap_total > fasttrap_max / 2) 531 (void) dtrace_condense(provid); 532 533 if (rval == EAGAIN) 534 fp->ftp_marked = 1; 535 536 later += fp->ftp_marked; 537 fpp = &fp->ftp_next; 538 } else { 539 *fpp = fp->ftp_next; 540 fasttrap_provider_free(fp); 541 } 542 } 543 mutex_exit(&bucket->ftb_mtx); 544 } 545 mtx_lock(&fasttrap_cleanup_mtx); 546 547 /* 548 * If we were unable to retire a provider, try again after a 549 * second. This situation can occur in certain circumstances 550 * where providers cannot be unregistered even though they have 551 * no probes enabled because of an execution of dtrace -l or 552 * something similar. 553 */ 554 if (later > 0 || fasttrap_cleanup_work || 555 fasttrap_cleanup_drain) { 556 mtx_unlock(&fasttrap_cleanup_mtx); 557 pause("ftclean", hz); 558 mtx_lock(&fasttrap_cleanup_mtx); 559 } else 560 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 561 0, "ftcl", 0); 562 } 563 564 /* 565 * Wake up the thread in fasttrap_unload() now that we're done. 566 */ 567 wakeup(&fasttrap_cleanup_drain); 568 mtx_unlock(&fasttrap_cleanup_mtx); 569 570 kthread_exit(); 571 } 572 573 /* 574 * Activates the asynchronous cleanup mechanism. 575 */ 576 static void 577 fasttrap_pid_cleanup(void) 578 { 579 580 mtx_lock(&fasttrap_cleanup_mtx); 581 if (!fasttrap_cleanup_work) { 582 fasttrap_cleanup_work = 1; 583 wakeup(&fasttrap_cleanup_cv); 584 } 585 mtx_unlock(&fasttrap_cleanup_mtx); 586 } 587 588 /* 589 * This is called from cfork() via dtrace_fasttrap_fork(). The child 590 * process's address space is (roughly) a copy of the parent process's so 591 * we have to remove all the instrumentation we had previously enabled in the 592 * parent. 593 */ 594 static void 595 fasttrap_fork(proc_t *p, proc_t *cp) 596 { 597 #ifndef illumos 598 fasttrap_scrblock_t *scrblk; 599 fasttrap_proc_t *fprc = NULL; 600 #endif 601 pid_t ppid = p->p_pid; 602 int i; 603 604 ASSERT(curproc == p); 605 #ifdef illumos 606 ASSERT(p->p_proc_flag & P_PR_LOCK); 607 #else 608 PROC_LOCK_ASSERT(p, MA_OWNED); 609 #endif 610 #ifdef illumos 611 ASSERT(p->p_dtrace_count > 0); 612 #else 613 /* 614 * This check is purposely here instead of in kern_fork.c because, 615 * for legal resons, we cannot include the dtrace_cddl.h header 616 * inside kern_fork.c and insert if-clause there. 617 */ 618 if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL) 619 return; 620 #endif 621 622 ASSERT(cp->p_dtrace_count == 0); 623 624 /* 625 * This would be simpler and faster if we maintained per-process 626 * hash tables of enabled tracepoints. It could, however, potentially 627 * slow down execution of a tracepoint since we'd need to go 628 * through two levels of indirection. In the future, we should 629 * consider either maintaining per-process ancillary lists of 630 * enabled tracepoints or hanging a pointer to a per-process hash 631 * table of enabled tracepoints off the proc structure. 632 */ 633 634 /* 635 * We don't have to worry about the child process disappearing 636 * because we're in fork(). 637 */ 638 #ifdef illumos 639 mtx_lock_spin(&cp->p_slock); 640 sprlock_proc(cp); 641 mtx_unlock_spin(&cp->p_slock); 642 #else 643 /* 644 * fasttrap_tracepoint_remove() expects the child process to be 645 * unlocked and the VM then expects curproc to be unlocked. 646 */ 647 _PHOLD(cp); 648 PROC_UNLOCK(cp); 649 PROC_UNLOCK(p); 650 if (p->p_dtrace_count == 0) 651 goto dup_helpers; 652 #endif 653 654 /* 655 * Iterate over every tracepoint looking for ones that belong to the 656 * parent process, and remove each from the child process. 657 */ 658 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 659 fasttrap_tracepoint_t *tp; 660 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 661 662 mutex_enter(&bucket->ftb_mtx); 663 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 664 if (tp->ftt_pid == ppid && 665 tp->ftt_proc->ftpc_acount != 0) { 666 int ret = fasttrap_tracepoint_remove(cp, tp); 667 ASSERT(ret == 0); 668 669 /* 670 * The count of active providers can only be 671 * decremented (i.e. to zero) during exec, 672 * exit, and removal of a meta provider so it 673 * should be impossible to drop the count 674 * mid-fork. 675 */ 676 ASSERT(tp->ftt_proc->ftpc_acount != 0); 677 #ifndef illumos 678 fprc = tp->ftt_proc; 679 #endif 680 } 681 } 682 mutex_exit(&bucket->ftb_mtx); 683 684 #ifndef illumos 685 /* 686 * Unmap any scratch space inherited from the parent's address 687 * space. 688 */ 689 if (fprc != NULL) { 690 mutex_enter(&fprc->ftpc_mtx); 691 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 692 vm_map_remove(&cp->p_vmspace->vm_map, 693 scrblk->ftsb_addr, 694 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 695 } 696 mutex_exit(&fprc->ftpc_mtx); 697 } 698 #endif 699 } 700 701 #ifdef illumos 702 mutex_enter(&cp->p_lock); 703 sprunlock(cp); 704 #else 705 dup_helpers: 706 if (p->p_dtrace_helpers != NULL) 707 dtrace_helpers_duplicate(p, cp); 708 PROC_LOCK(p); 709 PROC_LOCK(cp); 710 _PRELE(cp); 711 #endif 712 } 713 714 /* 715 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 716 * is set on the proc structure to indicate that there is a pid provider 717 * associated with this process. 718 */ 719 static void 720 fasttrap_exec_exit(proc_t *p) 721 { 722 #ifndef illumos 723 struct thread *td; 724 #endif 725 726 #ifdef illumos 727 ASSERT(p == curproc); 728 #else 729 PROC_LOCK_ASSERT(p, MA_OWNED); 730 _PHOLD(p); 731 /* 732 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 733 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 734 * ourselves when a process exits. 735 */ 736 FOREACH_THREAD_IN_PROC(p, td) 737 td->t_dtrace_sscr = NULL; 738 PROC_UNLOCK(p); 739 #endif 740 741 /* 742 * We clean up the pid provider for this process here; user-land 743 * static probes are handled by the meta-provider remove entry point. 744 */ 745 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 746 #ifndef illumos 747 if (p->p_dtrace_helpers) 748 dtrace_helpers_destroy(p); 749 PROC_LOCK(p); 750 _PRELE(p); 751 #endif 752 } 753 754 755 /*ARGSUSED*/ 756 static void 757 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 758 { 759 /* 760 * There are no "default" pid probes. 761 */ 762 } 763 764 static int 765 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 766 { 767 fasttrap_tracepoint_t *tp, *new_tp = NULL; 768 fasttrap_bucket_t *bucket; 769 fasttrap_id_t *id; 770 pid_t pid; 771 uintptr_t pc; 772 773 ASSERT(index < probe->ftp_ntps); 774 775 pid = probe->ftp_pid; 776 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 777 id = &probe->ftp_tps[index].fit_id; 778 779 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 780 781 #ifdef illumos 782 ASSERT(!(p->p_flag & SVFORK)); 783 #endif 784 785 /* 786 * Before we make any modifications, make sure we've imposed a barrier 787 * on the generation in which this probe was last modified. 788 */ 789 fasttrap_mod_barrier(probe->ftp_gen); 790 791 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 792 793 /* 794 * If the tracepoint has already been enabled, just add our id to the 795 * list of interested probes. This may be our second time through 796 * this path in which case we'll have constructed the tracepoint we'd 797 * like to install. If we can't find a match, and have an allocated 798 * tracepoint ready to go, enable that one now. 799 * 800 * A tracepoint whose process is defunct is also considered defunct. 801 */ 802 again: 803 mutex_enter(&bucket->ftb_mtx); 804 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 805 /* 806 * Note that it's safe to access the active count on the 807 * associated proc structure because we know that at least one 808 * provider (this one) will still be around throughout this 809 * operation. 810 */ 811 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 812 tp->ftt_proc->ftpc_acount == 0) 813 continue; 814 815 /* 816 * Now that we've found a matching tracepoint, it would be 817 * a decent idea to confirm that the tracepoint is still 818 * enabled and the trap instruction hasn't been overwritten. 819 * Since this is a little hairy, we'll punt for now. 820 */ 821 822 /* 823 * This can't be the first interested probe. We don't have 824 * to worry about another thread being in the midst of 825 * deleting this tracepoint (which would be the only valid 826 * reason for a tracepoint to have no interested probes) 827 * since we're holding P_PR_LOCK for this process. 828 */ 829 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 830 831 switch (id->fti_ptype) { 832 case DTFTP_ENTRY: 833 case DTFTP_OFFSETS: 834 case DTFTP_IS_ENABLED: 835 id->fti_next = tp->ftt_ids; 836 membar_producer(); 837 tp->ftt_ids = id; 838 membar_producer(); 839 break; 840 841 case DTFTP_RETURN: 842 case DTFTP_POST_OFFSETS: 843 id->fti_next = tp->ftt_retids; 844 membar_producer(); 845 tp->ftt_retids = id; 846 membar_producer(); 847 break; 848 849 default: 850 ASSERT(0); 851 } 852 853 mutex_exit(&bucket->ftb_mtx); 854 855 if (new_tp != NULL) { 856 new_tp->ftt_ids = NULL; 857 new_tp->ftt_retids = NULL; 858 } 859 860 return (0); 861 } 862 863 /* 864 * If we have a good tracepoint ready to go, install it now while 865 * we have the lock held and no one can screw with us. 866 */ 867 if (new_tp != NULL) { 868 int rc = 0; 869 870 new_tp->ftt_next = bucket->ftb_data; 871 membar_producer(); 872 bucket->ftb_data = new_tp; 873 membar_producer(); 874 mutex_exit(&bucket->ftb_mtx); 875 876 /* 877 * Activate the tracepoint in the ISA-specific manner. 878 * If this fails, we need to report the failure, but 879 * indicate that this tracepoint must still be disabled 880 * by calling fasttrap_tracepoint_disable(). 881 */ 882 if (fasttrap_tracepoint_install(p, new_tp) != 0) 883 rc = FASTTRAP_ENABLE_PARTIAL; 884 885 /* 886 * Increment the count of the number of tracepoints active in 887 * the victim process. 888 */ 889 #ifdef illumos 890 ASSERT(p->p_proc_flag & P_PR_LOCK); 891 #endif 892 p->p_dtrace_count++; 893 894 return (rc); 895 } 896 897 mutex_exit(&bucket->ftb_mtx); 898 899 /* 900 * Initialize the tracepoint that's been preallocated with the probe. 901 */ 902 new_tp = probe->ftp_tps[index].fit_tp; 903 904 ASSERT(new_tp->ftt_pid == pid); 905 ASSERT(new_tp->ftt_pc == pc); 906 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 907 ASSERT(new_tp->ftt_ids == NULL); 908 ASSERT(new_tp->ftt_retids == NULL); 909 910 switch (id->fti_ptype) { 911 case DTFTP_ENTRY: 912 case DTFTP_OFFSETS: 913 case DTFTP_IS_ENABLED: 914 id->fti_next = NULL; 915 new_tp->ftt_ids = id; 916 break; 917 918 case DTFTP_RETURN: 919 case DTFTP_POST_OFFSETS: 920 id->fti_next = NULL; 921 new_tp->ftt_retids = id; 922 break; 923 924 default: 925 ASSERT(0); 926 } 927 928 #ifdef __FreeBSD__ 929 if (SV_PROC_FLAG(p, SV_LP64)) 930 p->p_model = DATAMODEL_LP64; 931 else 932 p->p_model = DATAMODEL_ILP32; 933 #endif 934 935 /* 936 * If the ISA-dependent initialization goes to plan, go back to the 937 * beginning and try to install this freshly made tracepoint. 938 */ 939 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 940 goto again; 941 942 new_tp->ftt_ids = NULL; 943 new_tp->ftt_retids = NULL; 944 945 return (FASTTRAP_ENABLE_FAIL); 946 } 947 948 static void 949 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 950 { 951 fasttrap_bucket_t *bucket; 952 fasttrap_provider_t *provider = probe->ftp_prov; 953 fasttrap_tracepoint_t **pp, *tp; 954 fasttrap_id_t *id, **idp = NULL; 955 pid_t pid; 956 uintptr_t pc; 957 958 ASSERT(index < probe->ftp_ntps); 959 960 pid = probe->ftp_pid; 961 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 962 id = &probe->ftp_tps[index].fit_id; 963 964 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 965 966 /* 967 * Find the tracepoint and make sure that our id is one of the 968 * ones registered with it. 969 */ 970 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 971 mutex_enter(&bucket->ftb_mtx); 972 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 973 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 974 tp->ftt_proc == provider->ftp_proc) 975 break; 976 } 977 978 /* 979 * If we somehow lost this tracepoint, we're in a world of hurt. 980 */ 981 ASSERT(tp != NULL); 982 983 switch (id->fti_ptype) { 984 case DTFTP_ENTRY: 985 case DTFTP_OFFSETS: 986 case DTFTP_IS_ENABLED: 987 ASSERT(tp->ftt_ids != NULL); 988 idp = &tp->ftt_ids; 989 break; 990 991 case DTFTP_RETURN: 992 case DTFTP_POST_OFFSETS: 993 ASSERT(tp->ftt_retids != NULL); 994 idp = &tp->ftt_retids; 995 break; 996 997 default: 998 ASSERT(0); 999 } 1000 1001 while ((*idp)->fti_probe != probe) { 1002 idp = &(*idp)->fti_next; 1003 ASSERT(*idp != NULL); 1004 } 1005 1006 id = *idp; 1007 *idp = id->fti_next; 1008 membar_producer(); 1009 1010 ASSERT(id->fti_probe == probe); 1011 1012 /* 1013 * If there are other registered enablings of this tracepoint, we're 1014 * all done, but if this was the last probe assocated with this 1015 * this tracepoint, we need to remove and free it. 1016 */ 1017 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1018 1019 /* 1020 * If the current probe's tracepoint is in use, swap it 1021 * for an unused tracepoint. 1022 */ 1023 if (tp == probe->ftp_tps[index].fit_tp) { 1024 fasttrap_probe_t *tmp_probe; 1025 fasttrap_tracepoint_t **tmp_tp; 1026 uint_t tmp_index; 1027 1028 if (tp->ftt_ids != NULL) { 1029 tmp_probe = tp->ftt_ids->fti_probe; 1030 /* LINTED - alignment */ 1031 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1032 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1033 } else { 1034 tmp_probe = tp->ftt_retids->fti_probe; 1035 /* LINTED - alignment */ 1036 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1037 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1038 } 1039 1040 ASSERT(*tmp_tp != NULL); 1041 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1042 ASSERT((*tmp_tp)->ftt_ids == NULL); 1043 ASSERT((*tmp_tp)->ftt_retids == NULL); 1044 1045 probe->ftp_tps[index].fit_tp = *tmp_tp; 1046 *tmp_tp = tp; 1047 } 1048 1049 mutex_exit(&bucket->ftb_mtx); 1050 1051 /* 1052 * Tag the modified probe with the generation in which it was 1053 * changed. 1054 */ 1055 probe->ftp_gen = fasttrap_mod_gen; 1056 return; 1057 } 1058 1059 mutex_exit(&bucket->ftb_mtx); 1060 1061 /* 1062 * We can't safely remove the tracepoint from the set of active 1063 * tracepoints until we've actually removed the fasttrap instruction 1064 * from the process's text. We can, however, operate on this 1065 * tracepoint secure in the knowledge that no other thread is going to 1066 * be looking at it since we hold P_PR_LOCK on the process if it's 1067 * live or we hold the provider lock on the process if it's dead and 1068 * gone. 1069 */ 1070 1071 /* 1072 * We only need to remove the actual instruction if we're looking 1073 * at an existing process 1074 */ 1075 if (p != NULL) { 1076 /* 1077 * If we fail to restore the instruction we need to kill 1078 * this process since it's in a completely unrecoverable 1079 * state. 1080 */ 1081 if (fasttrap_tracepoint_remove(p, tp) != 0) 1082 fasttrap_sigtrap(p, NULL, pc); 1083 1084 /* 1085 * Decrement the count of the number of tracepoints active 1086 * in the victim process. 1087 */ 1088 #ifdef illumos 1089 ASSERT(p->p_proc_flag & P_PR_LOCK); 1090 #endif 1091 p->p_dtrace_count--; 1092 1093 atomic_add_rel_64(&p->p_fasttrap_tp_gen, 1); 1094 } 1095 1096 /* 1097 * Remove the probe from the hash table of active tracepoints. 1098 */ 1099 mutex_enter(&bucket->ftb_mtx); 1100 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1101 ASSERT(*pp != NULL); 1102 while (*pp != tp) { 1103 pp = &(*pp)->ftt_next; 1104 ASSERT(*pp != NULL); 1105 } 1106 1107 *pp = tp->ftt_next; 1108 membar_producer(); 1109 1110 mutex_exit(&bucket->ftb_mtx); 1111 1112 /* 1113 * Tag the modified probe with the generation in which it was changed. 1114 */ 1115 probe->ftp_gen = fasttrap_mod_gen; 1116 } 1117 1118 static void 1119 fasttrap_enable_callbacks(void) 1120 { 1121 /* 1122 * We don't have to play the rw lock game here because we're 1123 * providing something rather than taking something away -- 1124 * we can be sure that no threads have tried to follow this 1125 * function pointer yet. 1126 */ 1127 mutex_enter(&fasttrap_count_mtx); 1128 if (fasttrap_pid_count == 0) { 1129 ASSERT(dtrace_pid_probe_ptr == NULL); 1130 ASSERT(dtrace_return_probe_ptr == NULL); 1131 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1132 dtrace_return_probe_ptr = &fasttrap_return_probe; 1133 } 1134 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1135 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1136 fasttrap_pid_count++; 1137 mutex_exit(&fasttrap_count_mtx); 1138 } 1139 1140 static void 1141 fasttrap_disable_callbacks(void) 1142 { 1143 #ifdef illumos 1144 ASSERT(MUTEX_HELD(&cpu_lock)); 1145 #endif 1146 1147 1148 mutex_enter(&fasttrap_count_mtx); 1149 ASSERT(fasttrap_pid_count > 0); 1150 fasttrap_pid_count--; 1151 if (fasttrap_pid_count == 0) { 1152 #ifdef illumos 1153 cpu_t *cur, *cpu = CPU; 1154 1155 for (cur = cpu->cpu_next_onln; cur != cpu; 1156 cur = cur->cpu_next_onln) { 1157 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1158 } 1159 #endif 1160 dtrace_pid_probe_ptr = NULL; 1161 dtrace_return_probe_ptr = NULL; 1162 #ifdef illumos 1163 for (cur = cpu->cpu_next_onln; cur != cpu; 1164 cur = cur->cpu_next_onln) { 1165 rw_exit(&cur->cpu_ft_lock); 1166 } 1167 #endif 1168 } 1169 mutex_exit(&fasttrap_count_mtx); 1170 } 1171 1172 /*ARGSUSED*/ 1173 static void 1174 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1175 { 1176 fasttrap_probe_t *probe = parg; 1177 proc_t *p = NULL; 1178 int i, rc; 1179 1180 ASSERT(probe != NULL); 1181 ASSERT(!probe->ftp_enabled); 1182 ASSERT(id == probe->ftp_id); 1183 #ifdef illumos 1184 ASSERT(MUTEX_HELD(&cpu_lock)); 1185 #endif 1186 1187 /* 1188 * Increment the count of enabled probes on this probe's provider; 1189 * the provider can't go away while the probe still exists. We 1190 * must increment this even if we aren't able to properly enable 1191 * this probe. 1192 */ 1193 mutex_enter(&probe->ftp_prov->ftp_mtx); 1194 probe->ftp_prov->ftp_rcount++; 1195 mutex_exit(&probe->ftp_prov->ftp_mtx); 1196 1197 /* 1198 * If this probe's provider is retired (meaning it was valid in a 1199 * previously exec'ed incarnation of this address space), bail out. The 1200 * provider can't go away while we're in this code path. 1201 */ 1202 if (probe->ftp_prov->ftp_retired) 1203 return; 1204 1205 /* 1206 * If we can't find the process, it may be that we're in the context of 1207 * a fork in which the traced process is being born and we're copying 1208 * USDT probes. Otherwise, the process is gone so bail. 1209 */ 1210 #ifdef illumos 1211 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1212 if ((curproc->p_flag & SFORKING) == 0) 1213 return; 1214 1215 mutex_enter(&pidlock); 1216 p = prfind(probe->ftp_pid); 1217 1218 if (p == NULL) { 1219 /* 1220 * So it's not that the target process is being born, 1221 * it's that it isn't there at all (and we simply 1222 * happen to be forking). Anyway, we know that the 1223 * target is definitely gone, so bail out. 1224 */ 1225 mutex_exit(&pidlock); 1226 return (0); 1227 } 1228 1229 /* 1230 * Confirm that curproc is indeed forking the process in which 1231 * we're trying to enable probes. 1232 */ 1233 ASSERT(p->p_parent == curproc); 1234 ASSERT(p->p_stat == SIDL); 1235 1236 mutex_enter(&p->p_lock); 1237 mutex_exit(&pidlock); 1238 1239 sprlock_proc(p); 1240 } 1241 1242 ASSERT(!(p->p_flag & SVFORK)); 1243 mutex_exit(&p->p_lock); 1244 #else 1245 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0) 1246 return; 1247 #endif 1248 1249 /* 1250 * We have to enable the trap entry point before any user threads have 1251 * the chance to execute the trap instruction we're about to place 1252 * in their process's text. 1253 */ 1254 fasttrap_enable_callbacks(); 1255 1256 /* 1257 * Enable all the tracepoints and add this probe's id to each 1258 * tracepoint's list of active probes. 1259 */ 1260 for (i = 0; i < probe->ftp_ntps; i++) { 1261 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1262 /* 1263 * If enabling the tracepoint failed completely, 1264 * we don't have to disable it; if the failure 1265 * was only partial we must disable it. 1266 */ 1267 if (rc == FASTTRAP_ENABLE_FAIL) 1268 i--; 1269 else 1270 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1271 1272 /* 1273 * Back up and pull out all the tracepoints we've 1274 * created so far for this probe. 1275 */ 1276 while (i >= 0) { 1277 fasttrap_tracepoint_disable(p, probe, i); 1278 i--; 1279 } 1280 1281 #ifdef illumos 1282 mutex_enter(&p->p_lock); 1283 sprunlock(p); 1284 #else 1285 PRELE(p); 1286 #endif 1287 1288 /* 1289 * Since we're not actually enabling this probe, 1290 * drop our reference on the trap table entry. 1291 */ 1292 fasttrap_disable_callbacks(); 1293 return; 1294 } 1295 } 1296 #ifdef illumos 1297 mutex_enter(&p->p_lock); 1298 sprunlock(p); 1299 #else 1300 PRELE(p); 1301 #endif 1302 1303 probe->ftp_enabled = 1; 1304 } 1305 1306 /*ARGSUSED*/ 1307 static void 1308 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1309 { 1310 fasttrap_probe_t *probe = parg; 1311 fasttrap_provider_t *provider = probe->ftp_prov; 1312 proc_t *p; 1313 int i, whack = 0; 1314 1315 ASSERT(id == probe->ftp_id); 1316 1317 mutex_enter(&provider->ftp_mtx); 1318 1319 /* 1320 * We won't be able to acquire a /proc-esque lock on the process 1321 * iff the process is dead and gone. In this case, we rely on the 1322 * provider lock as a point of mutual exclusion to prevent other 1323 * DTrace consumers from disabling this probe. 1324 */ 1325 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0) 1326 p = NULL; 1327 1328 /* 1329 * Disable all the associated tracepoints (for fully enabled probes). 1330 */ 1331 if (probe->ftp_enabled) { 1332 for (i = 0; i < probe->ftp_ntps; i++) { 1333 fasttrap_tracepoint_disable(p, probe, i); 1334 } 1335 } 1336 1337 ASSERT(provider->ftp_rcount > 0); 1338 provider->ftp_rcount--; 1339 1340 if (p != NULL) { 1341 /* 1342 * Even though we may not be able to remove it entirely, we 1343 * mark this retired provider to get a chance to remove some 1344 * of the associated probes. 1345 */ 1346 if (provider->ftp_retired && !provider->ftp_marked) 1347 whack = provider->ftp_marked = 1; 1348 mutex_exit(&provider->ftp_mtx); 1349 } else { 1350 /* 1351 * If the process is dead, we're just waiting for the 1352 * last probe to be disabled to be able to free it. 1353 */ 1354 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1355 whack = provider->ftp_marked = 1; 1356 mutex_exit(&provider->ftp_mtx); 1357 } 1358 1359 if (whack) 1360 fasttrap_pid_cleanup(); 1361 1362 #ifdef __FreeBSD__ 1363 if (p != NULL) 1364 PRELE(p); 1365 #endif 1366 if (!probe->ftp_enabled) 1367 return; 1368 1369 probe->ftp_enabled = 0; 1370 1371 #ifdef illumos 1372 ASSERT(MUTEX_HELD(&cpu_lock)); 1373 #endif 1374 fasttrap_disable_callbacks(); 1375 } 1376 1377 /*ARGSUSED*/ 1378 static void 1379 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1380 dtrace_argdesc_t *desc) 1381 { 1382 fasttrap_probe_t *probe = parg; 1383 char *str; 1384 int i, ndx; 1385 1386 desc->dtargd_native[0] = '\0'; 1387 desc->dtargd_xlate[0] = '\0'; 1388 1389 if (probe->ftp_prov->ftp_retired != 0 || 1390 desc->dtargd_ndx >= probe->ftp_nargs) { 1391 desc->dtargd_ndx = DTRACE_ARGNONE; 1392 return; 1393 } 1394 1395 ndx = (probe->ftp_argmap != NULL) ? 1396 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1397 1398 str = probe->ftp_ntypes; 1399 for (i = 0; i < ndx; i++) { 1400 str += strlen(str) + 1; 1401 } 1402 1403 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1404 (void) strcpy(desc->dtargd_native, str); 1405 1406 if (probe->ftp_xtypes == NULL) 1407 return; 1408 1409 str = probe->ftp_xtypes; 1410 for (i = 0; i < desc->dtargd_ndx; i++) { 1411 str += strlen(str) + 1; 1412 } 1413 1414 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1415 (void) strcpy(desc->dtargd_xlate, str); 1416 } 1417 1418 /*ARGSUSED*/ 1419 static void 1420 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1421 { 1422 fasttrap_probe_t *probe = parg; 1423 int i; 1424 size_t size; 1425 1426 ASSERT(probe != NULL); 1427 ASSERT(!probe->ftp_enabled); 1428 ASSERT(fasttrap_total >= probe->ftp_ntps); 1429 1430 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1431 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1432 1433 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1434 fasttrap_mod_barrier(probe->ftp_gen); 1435 1436 for (i = 0; i < probe->ftp_ntps; i++) { 1437 kmem_free(probe->ftp_tps[i].fit_tp, 1438 sizeof (fasttrap_tracepoint_t)); 1439 } 1440 1441 kmem_free(probe, size); 1442 } 1443 1444 1445 static const dtrace_pattr_t pid_attr = { 1446 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1447 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1448 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1449 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1450 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1451 }; 1452 1453 static dtrace_pops_t pid_pops = { 1454 .dtps_provide = fasttrap_pid_provide, 1455 .dtps_provide_module = NULL, 1456 .dtps_enable = fasttrap_pid_enable, 1457 .dtps_disable = fasttrap_pid_disable, 1458 .dtps_suspend = NULL, 1459 .dtps_resume = NULL, 1460 .dtps_getargdesc = fasttrap_pid_getargdesc, 1461 .dtps_getargval = fasttrap_pid_getarg, 1462 .dtps_usermode = NULL, 1463 .dtps_destroy = fasttrap_pid_destroy 1464 }; 1465 1466 static dtrace_pops_t usdt_pops = { 1467 .dtps_provide = fasttrap_pid_provide, 1468 .dtps_provide_module = NULL, 1469 .dtps_enable = fasttrap_pid_enable, 1470 .dtps_disable = fasttrap_pid_disable, 1471 .dtps_suspend = NULL, 1472 .dtps_resume = NULL, 1473 .dtps_getargdesc = fasttrap_pid_getargdesc, 1474 .dtps_getargval = fasttrap_usdt_getarg, 1475 .dtps_usermode = NULL, 1476 .dtps_destroy = fasttrap_pid_destroy 1477 }; 1478 1479 static fasttrap_proc_t * 1480 fasttrap_proc_lookup(pid_t pid) 1481 { 1482 fasttrap_bucket_t *bucket; 1483 fasttrap_proc_t *fprc, *new_fprc; 1484 1485 1486 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1487 mutex_enter(&bucket->ftb_mtx); 1488 1489 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1490 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1491 mutex_enter(&fprc->ftpc_mtx); 1492 mutex_exit(&bucket->ftb_mtx); 1493 fprc->ftpc_rcount++; 1494 atomic_inc_64(&fprc->ftpc_acount); 1495 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1496 mutex_exit(&fprc->ftpc_mtx); 1497 1498 return (fprc); 1499 } 1500 } 1501 1502 /* 1503 * Drop the bucket lock so we don't try to perform a sleeping 1504 * allocation under it. 1505 */ 1506 mutex_exit(&bucket->ftb_mtx); 1507 1508 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1509 new_fprc->ftpc_pid = pid; 1510 new_fprc->ftpc_rcount = 1; 1511 new_fprc->ftpc_acount = 1; 1512 #ifndef illumos 1513 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1514 NULL); 1515 #endif 1516 1517 mutex_enter(&bucket->ftb_mtx); 1518 1519 /* 1520 * Take another lap through the list to make sure a proc hasn't 1521 * been created for this pid while we weren't under the bucket lock. 1522 */ 1523 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1524 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1525 mutex_enter(&fprc->ftpc_mtx); 1526 mutex_exit(&bucket->ftb_mtx); 1527 fprc->ftpc_rcount++; 1528 atomic_inc_64(&fprc->ftpc_acount); 1529 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1530 mutex_exit(&fprc->ftpc_mtx); 1531 1532 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1533 1534 return (fprc); 1535 } 1536 } 1537 1538 new_fprc->ftpc_next = bucket->ftb_data; 1539 bucket->ftb_data = new_fprc; 1540 1541 mutex_exit(&bucket->ftb_mtx); 1542 1543 return (new_fprc); 1544 } 1545 1546 static void 1547 fasttrap_proc_release(fasttrap_proc_t *proc) 1548 { 1549 fasttrap_bucket_t *bucket; 1550 fasttrap_proc_t *fprc, **fprcp; 1551 pid_t pid = proc->ftpc_pid; 1552 #ifndef illumos 1553 fasttrap_scrblock_t *scrblk, *scrblktmp; 1554 fasttrap_scrspace_t *scrspc, *scrspctmp; 1555 struct proc *p; 1556 struct thread *td; 1557 #endif 1558 1559 mutex_enter(&proc->ftpc_mtx); 1560 1561 ASSERT(proc->ftpc_rcount != 0); 1562 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1563 1564 if (--proc->ftpc_rcount != 0) { 1565 mutex_exit(&proc->ftpc_mtx); 1566 return; 1567 } 1568 1569 #ifndef illumos 1570 /* 1571 * Free all structures used to manage per-thread scratch space. 1572 */ 1573 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1574 scrblktmp) { 1575 LIST_REMOVE(scrblk, ftsb_next); 1576 free(scrblk, M_SOLARIS); 1577 } 1578 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1579 LIST_REMOVE(scrspc, ftss_next); 1580 free(scrspc, M_SOLARIS); 1581 } 1582 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1583 LIST_REMOVE(scrspc, ftss_next); 1584 free(scrspc, M_SOLARIS); 1585 } 1586 1587 if ((p = pfind(pid)) != NULL) { 1588 FOREACH_THREAD_IN_PROC(p, td) 1589 td->t_dtrace_sscr = NULL; 1590 PROC_UNLOCK(p); 1591 } 1592 #endif 1593 1594 mutex_exit(&proc->ftpc_mtx); 1595 1596 /* 1597 * There should definitely be no live providers associated with this 1598 * process at this point. 1599 */ 1600 ASSERT(proc->ftpc_acount == 0); 1601 1602 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1603 mutex_enter(&bucket->ftb_mtx); 1604 1605 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1606 while ((fprc = *fprcp) != NULL) { 1607 if (fprc == proc) 1608 break; 1609 1610 fprcp = &fprc->ftpc_next; 1611 } 1612 1613 /* 1614 * Something strange has happened if we can't find the proc. 1615 */ 1616 ASSERT(fprc != NULL); 1617 1618 *fprcp = fprc->ftpc_next; 1619 1620 mutex_exit(&bucket->ftb_mtx); 1621 1622 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1623 } 1624 1625 /* 1626 * Lookup a fasttrap-managed provider based on its name and associated pid. 1627 * If the pattr argument is non-NULL, this function instantiates the provider 1628 * if it doesn't exist otherwise it returns NULL. The provider is returned 1629 * with its lock held. 1630 */ 1631 static fasttrap_provider_t * 1632 fasttrap_provider_lookup(pid_t pid, const char *name, 1633 const dtrace_pattr_t *pattr) 1634 { 1635 fasttrap_provider_t *fp, *new_fp = NULL; 1636 fasttrap_bucket_t *bucket; 1637 char provname[DTRACE_PROVNAMELEN]; 1638 proc_t *p; 1639 cred_t *cred; 1640 1641 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1642 ASSERT(pattr != NULL); 1643 1644 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1645 mutex_enter(&bucket->ftb_mtx); 1646 1647 /* 1648 * Take a lap through the list and return the match if we find it. 1649 */ 1650 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1651 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1652 !fp->ftp_retired) { 1653 mutex_enter(&fp->ftp_mtx); 1654 mutex_exit(&bucket->ftb_mtx); 1655 return (fp); 1656 } 1657 } 1658 1659 /* 1660 * Drop the bucket lock so we don't try to perform a sleeping 1661 * allocation under it. 1662 */ 1663 mutex_exit(&bucket->ftb_mtx); 1664 1665 /* 1666 * Make sure the process exists, isn't a child created as the result 1667 * of a vfork(2), and isn't a zombie (but may be in fork). 1668 */ 1669 if ((p = pfind(pid)) == NULL) 1670 return (NULL); 1671 1672 /* 1673 * Increment p_dtrace_probes so that the process knows to inform us 1674 * when it exits or execs. fasttrap_provider_free() decrements this 1675 * when we're done with this provider. 1676 */ 1677 p->p_dtrace_probes++; 1678 1679 /* 1680 * Grab the credentials for this process so we have 1681 * something to pass to dtrace_register(). 1682 */ 1683 PROC_LOCK_ASSERT(p, MA_OWNED); 1684 crhold(p->p_ucred); 1685 cred = p->p_ucred; 1686 PROC_UNLOCK(p); 1687 1688 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1689 new_fp->ftp_pid = pid; 1690 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1691 #ifndef illumos 1692 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1693 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1694 #endif 1695 1696 ASSERT(new_fp->ftp_proc != NULL); 1697 1698 mutex_enter(&bucket->ftb_mtx); 1699 1700 /* 1701 * Take another lap through the list to make sure a provider hasn't 1702 * been created for this pid while we weren't under the bucket lock. 1703 */ 1704 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1705 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1706 !fp->ftp_retired) { 1707 mutex_enter(&fp->ftp_mtx); 1708 mutex_exit(&bucket->ftb_mtx); 1709 fasttrap_provider_free(new_fp); 1710 crfree(cred); 1711 return (fp); 1712 } 1713 } 1714 1715 (void) strcpy(new_fp->ftp_name, name); 1716 1717 /* 1718 * Fail and return NULL if either the provider name is too long 1719 * or we fail to register this new provider with the DTrace 1720 * framework. Note that this is the only place we ever construct 1721 * the full provider name -- we keep it in pieces in the provider 1722 * structure. 1723 */ 1724 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1725 sizeof (provname) || 1726 dtrace_register(provname, pattr, 1727 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1728 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1729 &new_fp->ftp_provid) != 0) { 1730 mutex_exit(&bucket->ftb_mtx); 1731 fasttrap_provider_free(new_fp); 1732 crfree(cred); 1733 return (NULL); 1734 } 1735 1736 new_fp->ftp_next = bucket->ftb_data; 1737 bucket->ftb_data = new_fp; 1738 1739 mutex_enter(&new_fp->ftp_mtx); 1740 mutex_exit(&bucket->ftb_mtx); 1741 1742 crfree(cred); 1743 return (new_fp); 1744 } 1745 1746 static void 1747 fasttrap_provider_free(fasttrap_provider_t *provider) 1748 { 1749 pid_t pid = provider->ftp_pid; 1750 proc_t *p; 1751 1752 /* 1753 * There need to be no associated enabled probes, no consumers 1754 * creating probes, and no meta providers referencing this provider. 1755 */ 1756 ASSERT(provider->ftp_rcount == 0); 1757 ASSERT(provider->ftp_ccount == 0); 1758 ASSERT(provider->ftp_mcount == 0); 1759 1760 /* 1761 * If this provider hasn't been retired, we need to explicitly drop the 1762 * count of active providers on the associated process structure. 1763 */ 1764 if (!provider->ftp_retired) { 1765 atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1766 ASSERT(provider->ftp_proc->ftpc_acount < 1767 provider->ftp_proc->ftpc_rcount); 1768 } 1769 1770 fasttrap_proc_release(provider->ftp_proc); 1771 1772 #ifndef illumos 1773 mutex_destroy(&provider->ftp_mtx); 1774 mutex_destroy(&provider->ftp_cmtx); 1775 #endif 1776 kmem_free(provider, sizeof (fasttrap_provider_t)); 1777 1778 /* 1779 * Decrement p_dtrace_probes on the process whose provider we're 1780 * freeing. We don't have to worry about clobbering somone else's 1781 * modifications to it because we have locked the bucket that 1782 * corresponds to this process's hash chain in the provider hash 1783 * table. Don't sweat it if we can't find the process. 1784 */ 1785 if ((p = pfind(pid)) == NULL) { 1786 return; 1787 } 1788 1789 p->p_dtrace_probes--; 1790 #ifndef illumos 1791 PROC_UNLOCK(p); 1792 #endif 1793 } 1794 1795 static void 1796 fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1797 { 1798 fasttrap_provider_t *fp; 1799 fasttrap_bucket_t *bucket; 1800 dtrace_provider_id_t provid; 1801 1802 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1803 1804 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1805 mutex_enter(&bucket->ftb_mtx); 1806 1807 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1808 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1809 !fp->ftp_retired) 1810 break; 1811 } 1812 1813 if (fp == NULL) { 1814 mutex_exit(&bucket->ftb_mtx); 1815 return; 1816 } 1817 1818 mutex_enter(&fp->ftp_mtx); 1819 ASSERT(!mprov || fp->ftp_mcount > 0); 1820 if (mprov && --fp->ftp_mcount != 0) { 1821 mutex_exit(&fp->ftp_mtx); 1822 mutex_exit(&bucket->ftb_mtx); 1823 return; 1824 } 1825 1826 /* 1827 * Mark the provider to be removed in our post-processing step, mark it 1828 * retired, and drop the active count on its proc. Marking it indicates 1829 * that we should try to remove it; setting the retired flag indicates 1830 * that we're done with this provider; dropping the active the proc 1831 * releases our hold, and when this reaches zero (as it will during 1832 * exit or exec) the proc and associated providers become defunct. 1833 * 1834 * We obviously need to take the bucket lock before the provider lock 1835 * to perform the lookup, but we need to drop the provider lock 1836 * before calling into the DTrace framework since we acquire the 1837 * provider lock in callbacks invoked from the DTrace framework. The 1838 * bucket lock therefore protects the integrity of the provider hash 1839 * table. 1840 */ 1841 atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1842 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1843 1844 fp->ftp_retired = 1; 1845 fp->ftp_marked = 1; 1846 provid = fp->ftp_provid; 1847 mutex_exit(&fp->ftp_mtx); 1848 1849 /* 1850 * We don't have to worry about invalidating the same provider twice 1851 * since fasttrap_provider_lookup() will ignore provider that have 1852 * been marked as retired. 1853 */ 1854 dtrace_invalidate(provid); 1855 1856 mutex_exit(&bucket->ftb_mtx); 1857 1858 fasttrap_pid_cleanup(); 1859 } 1860 1861 static int 1862 fasttrap_uint32_cmp(const void *ap, const void *bp) 1863 { 1864 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1865 } 1866 1867 static int 1868 fasttrap_uint64_cmp(const void *ap, const void *bp) 1869 { 1870 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1871 } 1872 1873 static int 1874 fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1875 { 1876 fasttrap_provider_t *provider; 1877 fasttrap_probe_t *pp; 1878 fasttrap_tracepoint_t *tp; 1879 char *name; 1880 int i, aframes = 0, whack; 1881 1882 /* 1883 * There needs to be at least one desired trace point. 1884 */ 1885 if (pdata->ftps_noffs == 0) 1886 return (EINVAL); 1887 1888 switch (pdata->ftps_type) { 1889 case DTFTP_ENTRY: 1890 name = "entry"; 1891 aframes = FASTTRAP_ENTRY_AFRAMES; 1892 break; 1893 case DTFTP_RETURN: 1894 name = "return"; 1895 aframes = FASTTRAP_RETURN_AFRAMES; 1896 break; 1897 case DTFTP_OFFSETS: 1898 name = NULL; 1899 break; 1900 default: 1901 return (EINVAL); 1902 } 1903 1904 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1905 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1906 return (ESRCH); 1907 1908 /* 1909 * Increment this reference count to indicate that a consumer is 1910 * actively adding a new probe associated with this provider. This 1911 * prevents the provider from being deleted -- we'll need to check 1912 * for pending deletions when we drop this reference count. 1913 */ 1914 provider->ftp_ccount++; 1915 mutex_exit(&provider->ftp_mtx); 1916 1917 /* 1918 * Grab the creation lock to ensure consistency between calls to 1919 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1920 * other threads creating probes. We must drop the provider lock 1921 * before taking this lock to avoid a three-way deadlock with the 1922 * DTrace framework. 1923 */ 1924 mutex_enter(&provider->ftp_cmtx); 1925 1926 if (name == NULL) { 1927 for (i = 0; i < pdata->ftps_noffs; i++) { 1928 char name_str[17]; 1929 1930 (void) sprintf(name_str, "%llx", 1931 (unsigned long long)pdata->ftps_offs[i]); 1932 1933 if (dtrace_probe_lookup(provider->ftp_provid, 1934 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1935 continue; 1936 1937 atomic_inc_32(&fasttrap_total); 1938 1939 if (fasttrap_total > fasttrap_max) { 1940 atomic_dec_32(&fasttrap_total); 1941 goto no_mem; 1942 } 1943 1944 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1945 1946 pp->ftp_prov = provider; 1947 pp->ftp_faddr = pdata->ftps_pc; 1948 pp->ftp_fsize = pdata->ftps_size; 1949 pp->ftp_pid = pdata->ftps_pid; 1950 pp->ftp_ntps = 1; 1951 1952 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1953 KM_SLEEP); 1954 1955 tp->ftt_proc = provider->ftp_proc; 1956 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1957 tp->ftt_pid = pdata->ftps_pid; 1958 1959 pp->ftp_tps[0].fit_tp = tp; 1960 pp->ftp_tps[0].fit_id.fti_probe = pp; 1961 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1962 1963 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1964 pdata->ftps_mod, pdata->ftps_func, name_str, 1965 FASTTRAP_OFFSET_AFRAMES, pp); 1966 } 1967 1968 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1969 pdata->ftps_func, name) == 0) { 1970 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1971 1972 if (fasttrap_total > fasttrap_max) { 1973 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1974 goto no_mem; 1975 } 1976 1977 /* 1978 * Make sure all tracepoint program counter values are unique. 1979 * We later assume that each probe has exactly one tracepoint 1980 * for a given pc. 1981 */ 1982 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1983 sizeof (uint64_t), fasttrap_uint64_cmp); 1984 for (i = 1; i < pdata->ftps_noffs; i++) { 1985 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1986 continue; 1987 1988 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1989 goto no_mem; 1990 } 1991 1992 ASSERT(pdata->ftps_noffs > 0); 1993 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1994 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1995 1996 pp->ftp_prov = provider; 1997 pp->ftp_faddr = pdata->ftps_pc; 1998 pp->ftp_fsize = pdata->ftps_size; 1999 pp->ftp_pid = pdata->ftps_pid; 2000 pp->ftp_ntps = pdata->ftps_noffs; 2001 2002 for (i = 0; i < pdata->ftps_noffs; i++) { 2003 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 2004 KM_SLEEP); 2005 2006 tp->ftt_proc = provider->ftp_proc; 2007 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 2008 tp->ftt_pid = pdata->ftps_pid; 2009 2010 pp->ftp_tps[i].fit_tp = tp; 2011 pp->ftp_tps[i].fit_id.fti_probe = pp; 2012 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 2013 } 2014 2015 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 2016 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 2017 } 2018 2019 mutex_exit(&provider->ftp_cmtx); 2020 2021 /* 2022 * We know that the provider is still valid since we incremented the 2023 * creation reference count. If someone tried to clean up this provider 2024 * while we were using it (e.g. because the process called exec(2) or 2025 * exit(2)), take note of that and try to clean it up now. 2026 */ 2027 mutex_enter(&provider->ftp_mtx); 2028 provider->ftp_ccount--; 2029 whack = provider->ftp_retired; 2030 mutex_exit(&provider->ftp_mtx); 2031 2032 if (whack) 2033 fasttrap_pid_cleanup(); 2034 2035 return (0); 2036 2037 no_mem: 2038 /* 2039 * If we've exhausted the allowable resources, we'll try to remove 2040 * this provider to free some up. This is to cover the case where 2041 * the user has accidentally created many more probes than was 2042 * intended (e.g. pid123:::). 2043 */ 2044 mutex_exit(&provider->ftp_cmtx); 2045 mutex_enter(&provider->ftp_mtx); 2046 provider->ftp_ccount--; 2047 provider->ftp_marked = 1; 2048 mutex_exit(&provider->ftp_mtx); 2049 2050 fasttrap_pid_cleanup(); 2051 2052 return (ENOMEM); 2053 } 2054 2055 /*ARGSUSED*/ 2056 static void * 2057 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2058 { 2059 fasttrap_provider_t *provider; 2060 2061 /* 2062 * A 32-bit unsigned integer (like a pid for example) can be 2063 * expressed in 10 or fewer decimal digits. Make sure that we'll 2064 * have enough space for the provider name. 2065 */ 2066 if (strlen(dhpv->dthpv_provname) + 10 >= 2067 sizeof (provider->ftp_name)) { 2068 printf("failed to instantiate provider %s: " 2069 "name too long to accomodate pid", dhpv->dthpv_provname); 2070 return (NULL); 2071 } 2072 2073 /* 2074 * Don't let folks spoof the true pid provider. 2075 */ 2076 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2077 printf("failed to instantiate provider %s: " 2078 "%s is an invalid name", dhpv->dthpv_provname, 2079 FASTTRAP_PID_NAME); 2080 return (NULL); 2081 } 2082 2083 /* 2084 * The highest stability class that fasttrap supports is ISA; cap 2085 * the stability of the new provider accordingly. 2086 */ 2087 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2088 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2089 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2090 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2091 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2092 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2093 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2094 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2095 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2096 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2097 2098 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2099 &dhpv->dthpv_pattr)) == NULL) { 2100 printf("failed to instantiate provider %s for " 2101 "process %u", dhpv->dthpv_provname, (uint_t)pid); 2102 return (NULL); 2103 } 2104 2105 /* 2106 * Up the meta provider count so this provider isn't removed until 2107 * the meta provider has been told to remove it. 2108 */ 2109 provider->ftp_mcount++; 2110 2111 mutex_exit(&provider->ftp_mtx); 2112 2113 return (provider); 2114 } 2115 2116 /* 2117 * We know a few things about our context here: we know that the probe being 2118 * created doesn't already exist (DTrace won't load DOF at the same address 2119 * twice, even if explicitly told to do so) and we know that we are 2120 * single-threaded with respect to the meta provider machinery. Knowing that 2121 * this is a new probe and that there is no way for us to race with another 2122 * operation on this provider allows us an important optimization: we need not 2123 * lookup a probe before adding it. Saving this lookup is important because 2124 * this code is in the fork path for processes with USDT probes, and lookups 2125 * here are potentially very expensive because of long hash conflicts on 2126 * module, function and name (DTrace doesn't hash on provider name). 2127 */ 2128 /*ARGSUSED*/ 2129 static void 2130 fasttrap_meta_create_probe(void *arg, void *parg, 2131 dtrace_helper_probedesc_t *dhpb) 2132 { 2133 fasttrap_provider_t *provider = parg; 2134 fasttrap_probe_t *pp; 2135 fasttrap_tracepoint_t *tp; 2136 int i, j; 2137 uint32_t ntps; 2138 2139 /* 2140 * Since the meta provider count is non-zero we don't have to worry 2141 * about this provider disappearing. 2142 */ 2143 ASSERT(provider->ftp_mcount > 0); 2144 2145 /* 2146 * The offsets must be unique. 2147 */ 2148 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2149 fasttrap_uint32_cmp); 2150 for (i = 1; i < dhpb->dthpb_noffs; i++) { 2151 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2152 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2153 return; 2154 } 2155 2156 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2157 fasttrap_uint32_cmp); 2158 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2159 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2160 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2161 return; 2162 } 2163 2164 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2165 ASSERT(ntps > 0); 2166 2167 atomic_add_32(&fasttrap_total, ntps); 2168 2169 if (fasttrap_total > fasttrap_max) { 2170 atomic_add_32(&fasttrap_total, -ntps); 2171 return; 2172 } 2173 2174 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2175 2176 pp->ftp_prov = provider; 2177 pp->ftp_pid = provider->ftp_pid; 2178 pp->ftp_ntps = ntps; 2179 pp->ftp_nargs = dhpb->dthpb_xargc; 2180 pp->ftp_xtypes = dhpb->dthpb_xtypes; 2181 pp->ftp_ntypes = dhpb->dthpb_ntypes; 2182 2183 /* 2184 * First create a tracepoint for each actual point of interest. 2185 */ 2186 for (i = 0; i < dhpb->dthpb_noffs; i++) { 2187 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2188 2189 tp->ftt_proc = provider->ftp_proc; 2190 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2191 tp->ftt_pid = provider->ftp_pid; 2192 2193 pp->ftp_tps[i].fit_tp = tp; 2194 pp->ftp_tps[i].fit_id.fti_probe = pp; 2195 #ifdef __sparc 2196 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2197 #else 2198 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2199 #endif 2200 } 2201 2202 /* 2203 * Then create a tracepoint for each is-enabled point. 2204 */ 2205 for (j = 0; i < ntps; i++, j++) { 2206 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2207 2208 tp->ftt_proc = provider->ftp_proc; 2209 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2210 tp->ftt_pid = provider->ftp_pid; 2211 2212 pp->ftp_tps[i].fit_tp = tp; 2213 pp->ftp_tps[i].fit_id.fti_probe = pp; 2214 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2215 } 2216 2217 /* 2218 * If the arguments are shuffled around we set the argument remapping 2219 * table. Later, when the probe fires, we only remap the arguments 2220 * if the table is non-NULL. 2221 */ 2222 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2223 if (dhpb->dthpb_args[i] != i) { 2224 pp->ftp_argmap = dhpb->dthpb_args; 2225 break; 2226 } 2227 } 2228 2229 /* 2230 * The probe is fully constructed -- register it with DTrace. 2231 */ 2232 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2233 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2234 } 2235 2236 /*ARGSUSED*/ 2237 static void 2238 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2239 { 2240 /* 2241 * Clean up the USDT provider. There may be active consumers of the 2242 * provider busy adding probes, no damage will actually befall the 2243 * provider until that count has dropped to zero. This just puts 2244 * the provider on death row. 2245 */ 2246 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2247 } 2248 2249 static dtrace_mops_t fasttrap_mops = { 2250 .dtms_create_probe = fasttrap_meta_create_probe, 2251 .dtms_provide_pid = fasttrap_meta_provide, 2252 .dtms_remove_pid = fasttrap_meta_remove 2253 }; 2254 2255 /*ARGSUSED*/ 2256 static int 2257 fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2258 int devtype __unused, struct thread *td __unused) 2259 { 2260 return (0); 2261 } 2262 2263 /*ARGSUSED*/ 2264 static int 2265 fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2266 struct thread *td) 2267 { 2268 if (!dtrace_attached()) 2269 return (EAGAIN); 2270 2271 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2272 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2273 fasttrap_probe_spec_t *probe; 2274 uint64_t noffs; 2275 size_t size; 2276 int ret, err; 2277 2278 if (copyin(&uprobe->ftps_noffs, &noffs, 2279 sizeof (uprobe->ftps_noffs))) 2280 return (EFAULT); 2281 2282 /* 2283 * Probes must have at least one tracepoint. 2284 */ 2285 if (noffs == 0) 2286 return (EINVAL); 2287 2288 size = sizeof (fasttrap_probe_spec_t) + 2289 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2290 2291 if (size > 1024 * 1024) 2292 return (ENOMEM); 2293 2294 probe = kmem_alloc(size, KM_SLEEP); 2295 2296 if (copyin(uprobe, probe, size) != 0 || 2297 probe->ftps_noffs != noffs) { 2298 kmem_free(probe, size); 2299 return (EFAULT); 2300 } 2301 2302 /* 2303 * Verify that the function and module strings contain no 2304 * funny characters. 2305 */ 2306 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2307 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2308 ret = EINVAL; 2309 goto err; 2310 } 2311 2312 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2313 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2314 ret = EINVAL; 2315 goto err; 2316 } 2317 2318 #ifdef notyet 2319 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2320 proc_t *p; 2321 pid_t pid = probe->ftps_pid; 2322 2323 mutex_enter(&pidlock); 2324 /* 2325 * Report an error if the process doesn't exist 2326 * or is actively being birthed. 2327 */ 2328 if ((p = pfind(pid)) == NULL || p->p_stat == SIDL) { 2329 mutex_exit(&pidlock); 2330 return (ESRCH); 2331 } 2332 mutex_enter(&p->p_lock); 2333 mutex_exit(&pidlock); 2334 2335 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2336 VREAD | VWRITE)) != 0) { 2337 mutex_exit(&p->p_lock); 2338 return (ret); 2339 } 2340 mutex_exit(&p->p_lock); 2341 } 2342 #endif /* notyet */ 2343 2344 ret = fasttrap_add_probe(probe); 2345 err: 2346 kmem_free(probe, size); 2347 2348 return (ret); 2349 2350 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2351 fasttrap_instr_query_t instr; 2352 fasttrap_tracepoint_t *tp; 2353 uint_t index; 2354 #ifdef notyet 2355 int ret; 2356 #endif 2357 2358 #ifdef illumos 2359 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2360 return (EFAULT); 2361 #endif 2362 2363 #ifdef notyet 2364 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2365 proc_t *p; 2366 pid_t pid = instr.ftiq_pid; 2367 2368 mutex_enter(&pidlock); 2369 /* 2370 * Report an error if the process doesn't exist 2371 * or is actively being birthed. 2372 */ 2373 if ((p == pfind(pid)) == NULL || p->p_stat == SIDL) { 2374 mutex_exit(&pidlock); 2375 return (ESRCH); 2376 } 2377 mutex_enter(&p->p_lock); 2378 mutex_exit(&pidlock); 2379 2380 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2381 VREAD)) != 0) { 2382 mutex_exit(&p->p_lock); 2383 return (ret); 2384 } 2385 2386 mutex_exit(&p->p_lock); 2387 } 2388 #endif /* notyet */ 2389 2390 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2391 2392 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2393 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2394 while (tp != NULL) { 2395 if (instr.ftiq_pid == tp->ftt_pid && 2396 instr.ftiq_pc == tp->ftt_pc && 2397 tp->ftt_proc->ftpc_acount != 0) 2398 break; 2399 2400 tp = tp->ftt_next; 2401 } 2402 2403 if (tp == NULL) { 2404 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2405 return (ENOENT); 2406 } 2407 2408 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2409 sizeof (instr.ftiq_instr)); 2410 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2411 2412 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2413 return (EFAULT); 2414 2415 return (0); 2416 } 2417 2418 return (EINVAL); 2419 } 2420 2421 static int 2422 fasttrap_load(void) 2423 { 2424 ulong_t nent; 2425 int i, ret; 2426 2427 /* Create the /dev/dtrace/fasttrap entry. */ 2428 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2429 "dtrace/fasttrap"); 2430 2431 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2432 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2433 NULL); 2434 2435 #ifdef illumos 2436 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2437 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2438 #endif 2439 fasttrap_total = 0; 2440 2441 /* 2442 * Conjure up the tracepoints hashtable... 2443 */ 2444 #ifdef illumos 2445 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2446 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2447 #else 2448 nent = tpoints_hash_size; 2449 #endif 2450 2451 if (nent == 0 || nent > 0x1000000) 2452 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2453 2454 tpoints_hash_size = nent; 2455 2456 if (ISP2(nent)) 2457 fasttrap_tpoints.fth_nent = nent; 2458 else 2459 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2460 ASSERT(fasttrap_tpoints.fth_nent > 0); 2461 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2462 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2463 sizeof (fasttrap_bucket_t), KM_SLEEP); 2464 #ifndef illumos 2465 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2466 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2467 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2468 #endif 2469 2470 /* 2471 * ... and the providers hash table... 2472 */ 2473 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2474 if (ISP2(nent)) 2475 fasttrap_provs.fth_nent = nent; 2476 else 2477 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2478 ASSERT(fasttrap_provs.fth_nent > 0); 2479 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2480 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2481 sizeof (fasttrap_bucket_t), KM_SLEEP); 2482 #ifndef illumos 2483 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2484 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2485 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2486 #endif 2487 2488 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2489 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2490 if (ret != 0) { 2491 destroy_dev(fasttrap_cdev); 2492 #ifndef illumos 2493 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2494 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2495 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2496 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2497 #endif 2498 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2499 sizeof (fasttrap_bucket_t)); 2500 mtx_destroy(&fasttrap_cleanup_mtx); 2501 mutex_destroy(&fasttrap_count_mtx); 2502 return (ret); 2503 } 2504 2505 2506 /* 2507 * ... and the procs hash table. 2508 */ 2509 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2510 if (ISP2(nent)) 2511 fasttrap_procs.fth_nent = nent; 2512 else 2513 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2514 ASSERT(fasttrap_procs.fth_nent > 0); 2515 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2516 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2517 sizeof (fasttrap_bucket_t), KM_SLEEP); 2518 #ifndef illumos 2519 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2520 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2521 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2522 2523 rm_init(&fasttrap_tp_lock, "fasttrap tracepoint"); 2524 2525 /* 2526 * This event handler must run before kdtrace_thread_dtor() since it 2527 * accesses the thread's struct kdtrace_thread. 2528 */ 2529 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2530 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2531 #endif 2532 2533 /* 2534 * Install our hooks into fork(2), exec(2), and exit(2). 2535 */ 2536 dtrace_fasttrap_fork = &fasttrap_fork; 2537 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2538 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2539 2540 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2541 &fasttrap_meta_id); 2542 2543 return (0); 2544 } 2545 2546 static int 2547 fasttrap_unload(void) 2548 { 2549 int i, fail = 0; 2550 2551 /* 2552 * Unregister the meta-provider to make sure no new fasttrap- 2553 * managed providers come along while we're trying to close up 2554 * shop. If we fail to detach, we'll need to re-register as a 2555 * meta-provider. We can fail to unregister as a meta-provider 2556 * if providers we manage still exist. 2557 */ 2558 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2559 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2560 return (-1); 2561 2562 /* 2563 * Iterate over all of our providers. If there's still a process 2564 * that corresponds to that pid, fail to detach. 2565 */ 2566 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2567 fasttrap_provider_t **fpp, *fp; 2568 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2569 2570 mutex_enter(&bucket->ftb_mtx); 2571 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2572 while ((fp = *fpp) != NULL) { 2573 /* 2574 * Acquire and release the lock as a simple way of 2575 * waiting for any other consumer to finish with 2576 * this provider. A thread must first acquire the 2577 * bucket lock so there's no chance of another thread 2578 * blocking on the provider's lock. 2579 */ 2580 mutex_enter(&fp->ftp_mtx); 2581 mutex_exit(&fp->ftp_mtx); 2582 2583 if (dtrace_unregister(fp->ftp_provid) != 0) { 2584 fail = 1; 2585 fpp = &fp->ftp_next; 2586 } else { 2587 *fpp = fp->ftp_next; 2588 fasttrap_provider_free(fp); 2589 } 2590 } 2591 2592 mutex_exit(&bucket->ftb_mtx); 2593 } 2594 2595 if (fail) { 2596 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2597 &fasttrap_meta_id); 2598 2599 return (-1); 2600 } 2601 2602 /* 2603 * Stop new processes from entering these hooks now, before the 2604 * fasttrap_cleanup thread runs. That way all processes will hopefully 2605 * be out of these hooks before we free fasttrap_provs.fth_table 2606 */ 2607 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2608 dtrace_fasttrap_fork = NULL; 2609 2610 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2611 dtrace_fasttrap_exec = NULL; 2612 2613 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2614 dtrace_fasttrap_exit = NULL; 2615 2616 mtx_lock(&fasttrap_cleanup_mtx); 2617 fasttrap_cleanup_drain = 1; 2618 /* Wait for the cleanup thread to finish up and signal us. */ 2619 wakeup(&fasttrap_cleanup_cv); 2620 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2621 0); 2622 fasttrap_cleanup_proc = NULL; 2623 mtx_destroy(&fasttrap_cleanup_mtx); 2624 2625 #ifdef DEBUG 2626 mutex_enter(&fasttrap_count_mtx); 2627 ASSERT(fasttrap_pid_count == 0); 2628 mutex_exit(&fasttrap_count_mtx); 2629 #endif 2630 2631 #ifndef illumos 2632 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2633 2634 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2635 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2636 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2637 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2638 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2639 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2640 #endif 2641 kmem_free(fasttrap_tpoints.fth_table, 2642 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2643 fasttrap_tpoints.fth_nent = 0; 2644 2645 kmem_free(fasttrap_provs.fth_table, 2646 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2647 fasttrap_provs.fth_nent = 0; 2648 2649 kmem_free(fasttrap_procs.fth_table, 2650 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2651 fasttrap_procs.fth_nent = 0; 2652 2653 #ifndef illumos 2654 destroy_dev(fasttrap_cdev); 2655 mutex_destroy(&fasttrap_count_mtx); 2656 rm_destroy(&fasttrap_tp_lock); 2657 #endif 2658 2659 return (0); 2660 } 2661 2662 /* ARGSUSED */ 2663 static int 2664 fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2665 { 2666 int error = 0; 2667 2668 switch (type) { 2669 case MOD_LOAD: 2670 break; 2671 2672 case MOD_UNLOAD: 2673 break; 2674 2675 case MOD_SHUTDOWN: 2676 break; 2677 2678 default: 2679 error = EOPNOTSUPP; 2680 break; 2681 } 2682 return (error); 2683 } 2684 2685 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2686 NULL); 2687 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2688 fasttrap_unload, NULL); 2689 2690 DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2691 MODULE_VERSION(fasttrap, 1); 2692 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2693 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2694