191eaf3e1SJohn Birrell /* 291eaf3e1SJohn Birrell * CDDL HEADER START 391eaf3e1SJohn Birrell * 491eaf3e1SJohn Birrell * The contents of this file are subject to the terms of the 591eaf3e1SJohn Birrell * Common Development and Distribution License (the "License"). 691eaf3e1SJohn Birrell * You may not use this file except in compliance with the License. 791eaf3e1SJohn Birrell * 891eaf3e1SJohn Birrell * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 991eaf3e1SJohn Birrell * or http://www.opensolaris.org/os/licensing. 1091eaf3e1SJohn Birrell * See the License for the specific language governing permissions 1191eaf3e1SJohn Birrell * and limitations under the License. 1291eaf3e1SJohn Birrell * 1391eaf3e1SJohn Birrell * When distributing Covered Code, include this CDDL HEADER in each 1491eaf3e1SJohn Birrell * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1591eaf3e1SJohn Birrell * If applicable, add the following below this CDDL HEADER, with the 1691eaf3e1SJohn Birrell * fields enclosed by brackets "[]" replaced with your own identifying 1791eaf3e1SJohn Birrell * information: Portions Copyright [yyyy] [name of copyright owner] 1891eaf3e1SJohn Birrell * 1991eaf3e1SJohn Birrell * CDDL HEADER END 2091eaf3e1SJohn Birrell * 2191eaf3e1SJohn Birrell * Portions Copyright 2006-2008 John Birrell jb@freebsd.org 2291eaf3e1SJohn Birrell * 2391eaf3e1SJohn Birrell * $FreeBSD$ 2491eaf3e1SJohn Birrell * 2591eaf3e1SJohn Birrell */ 2691eaf3e1SJohn Birrell 2791eaf3e1SJohn Birrell /* 2891eaf3e1SJohn Birrell * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 2991eaf3e1SJohn Birrell * Use is subject to license terms. 3091eaf3e1SJohn Birrell */ 3191eaf3e1SJohn Birrell 3291eaf3e1SJohn Birrell #include <sys/cdefs.h> 3391eaf3e1SJohn Birrell #include <sys/param.h> 3491eaf3e1SJohn Birrell #include <sys/systm.h> 3591eaf3e1SJohn Birrell #include <sys/conf.h> 3691eaf3e1SJohn Birrell #include <sys/cpuvar.h> 379e5787d2SMatt Macy #include <sys/endian.h> 3891eaf3e1SJohn Birrell #include <sys/fcntl.h> 3991eaf3e1SJohn Birrell #include <sys/filio.h> 4091eaf3e1SJohn Birrell #include <sys/kdb.h> 4191eaf3e1SJohn Birrell #include <sys/kernel.h> 4291eaf3e1SJohn Birrell #include <sys/kmem.h> 4391eaf3e1SJohn Birrell #include <sys/kthread.h> 4491eaf3e1SJohn Birrell #include <sys/limits.h> 4591eaf3e1SJohn Birrell #include <sys/linker.h> 4691eaf3e1SJohn Birrell #include <sys/lock.h> 4791eaf3e1SJohn Birrell #include <sys/malloc.h> 4891eaf3e1SJohn Birrell #include <sys/module.h> 4991eaf3e1SJohn Birrell #include <sys/mutex.h> 5091eaf3e1SJohn Birrell #include <sys/poll.h> 5191eaf3e1SJohn Birrell #include <sys/proc.h> 5291eaf3e1SJohn Birrell #include <sys/selinfo.h> 5391eaf3e1SJohn Birrell #include <sys/smp.h> 545cde34a0SAndrew Turner #include <sys/sysctl.h> 5591eaf3e1SJohn Birrell #include <sys/uio.h> 5691eaf3e1SJohn Birrell #include <sys/unistd.h> 57036a8c5dSAndriy Gapon #include <machine/cpu.h> 5891eaf3e1SJohn Birrell #include <machine/stdarg.h> 5991eaf3e1SJohn Birrell 6091eaf3e1SJohn Birrell #include <sys/dtrace.h> 6191eaf3e1SJohn Birrell #include <sys/dtrace_bsd.h> 6291eaf3e1SJohn Birrell 6391eaf3e1SJohn Birrell #define PROF_NAMELEN 15 6491eaf3e1SJohn Birrell 6591eaf3e1SJohn Birrell #define PROF_PROFILE 0 6691eaf3e1SJohn Birrell #define PROF_TICK 1 6791eaf3e1SJohn Birrell #define PROF_PREFIX_PROFILE "profile-" 6891eaf3e1SJohn Birrell #define PROF_PREFIX_TICK "tick-" 6991eaf3e1SJohn Birrell 7091eaf3e1SJohn Birrell /* 7191eaf3e1SJohn Birrell * Regardless of platform, there are five artificial frames in the case of the 7291eaf3e1SJohn Birrell * profile provider: 7391eaf3e1SJohn Birrell * 7491eaf3e1SJohn Birrell * profile_fire 7591eaf3e1SJohn Birrell * cyclic_expire 7691eaf3e1SJohn Birrell * cyclic_fire 7791eaf3e1SJohn Birrell * [ cbe ] 7891eaf3e1SJohn Birrell * [ locore ] 7991eaf3e1SJohn Birrell * 8091eaf3e1SJohn Birrell * On amd64, there are two frames associated with locore: one in locore, and 8191eaf3e1SJohn Birrell * another in common interrupt dispatch code. (i386 has not been modified to 8291eaf3e1SJohn Birrell * use this common layer.) Further, on i386, the interrupted instruction 8391eaf3e1SJohn Birrell * appears as its own stack frame. All of this means that we need to add one 8491eaf3e1SJohn Birrell * frame for amd64, and then take one away for both amd64 and i386. 8591eaf3e1SJohn Birrell * 8691eaf3e1SJohn Birrell * All of the above constraints lead to the mess below. Yes, the profile 8791eaf3e1SJohn Birrell * provider should ideally figure this out on-the-fly by hiting one of its own 8891eaf3e1SJohn Birrell * probes and then walking its own stack trace. This is complicated, however, 8991eaf3e1SJohn Birrell * and the static definition doesn't seem to be overly brittle. Still, we 9091eaf3e1SJohn Birrell * allow for a manual override in case we get it completely wrong. 9191eaf3e1SJohn Birrell */ 9291eaf3e1SJohn Birrell #ifdef __amd64 93036a8c5dSAndriy Gapon #define PROF_ARTIFICIAL_FRAMES 10 9491eaf3e1SJohn Birrell #else 9591eaf3e1SJohn Birrell #ifdef __i386 9691eaf3e1SJohn Birrell #define PROF_ARTIFICIAL_FRAMES 6 9791eaf3e1SJohn Birrell #endif 9891eaf3e1SJohn Birrell #endif 9991eaf3e1SJohn Birrell 100c57e9d4eSOleksandr Tymoshenko #ifdef __mips 101c57e9d4eSOleksandr Tymoshenko /* 102c57e9d4eSOleksandr Tymoshenko * This value is bogus just to make module compilable on mips 103c57e9d4eSOleksandr Tymoshenko */ 104c57e9d4eSOleksandr Tymoshenko #define PROF_ARTIFICIAL_FRAMES 3 105c57e9d4eSOleksandr Tymoshenko #endif 106c57e9d4eSOleksandr Tymoshenko 107c7570492SJustin Hibbits #ifdef __powerpc__ 108c7570492SJustin Hibbits /* 109c7570492SJustin Hibbits * This value is bogus just to make module compilable on powerpc 110c7570492SJustin Hibbits */ 111a1a990d8SJustin Hibbits #define PROF_ARTIFICIAL_FRAMES 3 112c7570492SJustin Hibbits #endif 113c7570492SJustin Hibbits 114036a8c5dSAndriy Gapon struct profile_probe_percpu; 115036a8c5dSAndriy Gapon 116fcb56067SGeorge V. Neville-Neil #ifdef __mips 117fcb56067SGeorge V. Neville-Neil /* bogus */ 118fcb56067SGeorge V. Neville-Neil #define PROF_ARTIFICIAL_FRAMES 3 119fcb56067SGeorge V. Neville-Neil #endif 120fcb56067SGeorge V. Neville-Neil 121fcb56067SGeorge V. Neville-Neil #ifdef __arm__ 1225cde34a0SAndrew Turner #define PROF_ARTIFICIAL_FRAMES 3 123fcb56067SGeorge V. Neville-Neil #endif 124fcb56067SGeorge V. Neville-Neil 125b78ee15eSRuslan Bukin #ifdef __aarch64__ 126599fb1d1SRobert Watson #define PROF_ARTIFICIAL_FRAMES 12 127b78ee15eSRuslan Bukin #endif 128b78ee15eSRuslan Bukin 129ca20f8ecSRuslan Bukin #ifdef __riscv 130fed1ca4bSRuslan Bukin /* TODO: verify */ 131fed1ca4bSRuslan Bukin #define PROF_ARTIFICIAL_FRAMES 10 132fed1ca4bSRuslan Bukin #endif 133fed1ca4bSRuslan Bukin 13491eaf3e1SJohn Birrell typedef struct profile_probe { 13591eaf3e1SJohn Birrell char prof_name[PROF_NAMELEN]; 13691eaf3e1SJohn Birrell dtrace_id_t prof_id; 13791eaf3e1SJohn Birrell int prof_kind; 138036a8c5dSAndriy Gapon #ifdef illumos 13991eaf3e1SJohn Birrell hrtime_t prof_interval; 14091eaf3e1SJohn Birrell cyclic_id_t prof_cyclic; 141036a8c5dSAndriy Gapon #else 142036a8c5dSAndriy Gapon sbintime_t prof_interval; 143036a8c5dSAndriy Gapon struct callout prof_cyclic; 144036a8c5dSAndriy Gapon sbintime_t prof_expected; 145036a8c5dSAndriy Gapon struct profile_probe_percpu **prof_pcpus; 146036a8c5dSAndriy Gapon #endif 14791eaf3e1SJohn Birrell } profile_probe_t; 14891eaf3e1SJohn Birrell 14991eaf3e1SJohn Birrell typedef struct profile_probe_percpu { 15091eaf3e1SJohn Birrell hrtime_t profc_expected; 15191eaf3e1SJohn Birrell hrtime_t profc_interval; 15291eaf3e1SJohn Birrell profile_probe_t *profc_probe; 153036a8c5dSAndriy Gapon #ifdef __FreeBSD__ 154036a8c5dSAndriy Gapon struct callout profc_cyclic; 155036a8c5dSAndriy Gapon #endif 15691eaf3e1SJohn Birrell } profile_probe_percpu_t; 15791eaf3e1SJohn Birrell 15891eaf3e1SJohn Birrell static d_open_t profile_open; 15991eaf3e1SJohn Birrell static int profile_unload(void); 16091eaf3e1SJohn Birrell static void profile_create(hrtime_t, char *, int); 16191eaf3e1SJohn Birrell static void profile_destroy(void *, dtrace_id_t, void *); 16291eaf3e1SJohn Birrell static void profile_enable(void *, dtrace_id_t, void *); 16391eaf3e1SJohn Birrell static void profile_disable(void *, dtrace_id_t, void *); 16491eaf3e1SJohn Birrell static void profile_load(void *); 16591eaf3e1SJohn Birrell static void profile_provide(void *, dtrace_probedesc_t *); 16691eaf3e1SJohn Birrell 16791eaf3e1SJohn Birrell static int profile_rates[] = { 16891eaf3e1SJohn Birrell 97, 199, 499, 997, 1999, 16991eaf3e1SJohn Birrell 4001, 4999, 0, 0, 0, 17091eaf3e1SJohn Birrell 0, 0, 0, 0, 0, 17191eaf3e1SJohn Birrell 0, 0, 0, 0, 0 17291eaf3e1SJohn Birrell }; 17391eaf3e1SJohn Birrell 17491eaf3e1SJohn Birrell static int profile_ticks[] = { 17591eaf3e1SJohn Birrell 1, 10, 100, 500, 1000, 17691eaf3e1SJohn Birrell 5000, 0, 0, 0, 0, 17791eaf3e1SJohn Birrell 0, 0, 0, 0, 0 17891eaf3e1SJohn Birrell }; 17991eaf3e1SJohn Birrell 18091eaf3e1SJohn Birrell /* 18191eaf3e1SJohn Birrell * profile_max defines the upper bound on the number of profile probes that 18291eaf3e1SJohn Birrell * can exist (this is to prevent malicious or clumsy users from exhausing 18391eaf3e1SJohn Birrell * system resources by creating a slew of profile probes). At mod load time, 18491eaf3e1SJohn Birrell * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's 18591eaf3e1SJohn Birrell * present in the profile.conf file. 18691eaf3e1SJohn Birrell */ 18791eaf3e1SJohn Birrell #define PROFILE_MAX_DEFAULT 1000 /* default max. number of probes */ 18891eaf3e1SJohn Birrell static uint32_t profile_max = PROFILE_MAX_DEFAULT; 18991eaf3e1SJohn Birrell /* maximum number of profile probes */ 19091eaf3e1SJohn Birrell static uint32_t profile_total; /* current number of profile probes */ 19191eaf3e1SJohn Birrell 19291eaf3e1SJohn Birrell static struct cdevsw profile_cdevsw = { 19391eaf3e1SJohn Birrell .d_version = D_VERSION, 19491eaf3e1SJohn Birrell .d_open = profile_open, 19591eaf3e1SJohn Birrell .d_name = "profile", 19691eaf3e1SJohn Birrell }; 19791eaf3e1SJohn Birrell 19891eaf3e1SJohn Birrell static dtrace_pattr_t profile_attr = { 19991eaf3e1SJohn Birrell { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 20091eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 20191eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 20291eaf3e1SJohn Birrell { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 20391eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 20491eaf3e1SJohn Birrell }; 20591eaf3e1SJohn Birrell 20691eaf3e1SJohn Birrell static dtrace_pops_t profile_pops = { 20747f11baaSMark Johnston .dtps_provide = profile_provide, 20847f11baaSMark Johnston .dtps_provide_module = NULL, 20947f11baaSMark Johnston .dtps_enable = profile_enable, 21047f11baaSMark Johnston .dtps_disable = profile_disable, 21147f11baaSMark Johnston .dtps_suspend = NULL, 21247f11baaSMark Johnston .dtps_resume = NULL, 21347f11baaSMark Johnston .dtps_getargdesc = NULL, 21447f11baaSMark Johnston .dtps_getargval = NULL, 21547f11baaSMark Johnston .dtps_usermode = NULL, 21647f11baaSMark Johnston .dtps_destroy = profile_destroy 21791eaf3e1SJohn Birrell }; 21891eaf3e1SJohn Birrell 21991eaf3e1SJohn Birrell static struct cdev *profile_cdev; 22091eaf3e1SJohn Birrell static dtrace_provider_id_t profile_id; 22191eaf3e1SJohn Birrell static hrtime_t profile_interval_min = NANOSEC / 5000; /* 5000 hz */ 2225cde34a0SAndrew Turner static int profile_aframes = PROF_ARTIFICIAL_FRAMES; 2235cde34a0SAndrew Turner 2245cde34a0SAndrew Turner SYSCTL_DECL(_kern_dtrace); 2257029da5cSPawel Biernacki SYSCTL_NODE(_kern_dtrace, OID_AUTO, profile, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2267029da5cSPawel Biernacki "DTrace profile parameters"); 2275cde34a0SAndrew Turner SYSCTL_INT(_kern_dtrace_profile, OID_AUTO, aframes, CTLFLAG_RW, &profile_aframes, 2285cde34a0SAndrew Turner 0, "Skipped frames for profile provider"); 22991eaf3e1SJohn Birrell 230036a8c5dSAndriy Gapon static sbintime_t 231036a8c5dSAndriy Gapon nsec_to_sbt(hrtime_t nsec) 232036a8c5dSAndriy Gapon { 233036a8c5dSAndriy Gapon time_t sec; 234036a8c5dSAndriy Gapon 235036a8c5dSAndriy Gapon /* 236036a8c5dSAndriy Gapon * We need to calculate nsec * 2^32 / 10^9 237036a8c5dSAndriy Gapon * Seconds and nanoseconds are split to avoid overflow. 238036a8c5dSAndriy Gapon */ 239036a8c5dSAndriy Gapon sec = nsec / NANOSEC; 240036a8c5dSAndriy Gapon nsec = nsec % NANOSEC; 241036a8c5dSAndriy Gapon return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC); 242036a8c5dSAndriy Gapon } 243036a8c5dSAndriy Gapon 244036a8c5dSAndriy Gapon static hrtime_t 245036a8c5dSAndriy Gapon sbt_to_nsec(sbintime_t sbt) 246036a8c5dSAndriy Gapon { 247036a8c5dSAndriy Gapon 248036a8c5dSAndriy Gapon return ((sbt >> 32) * NANOSEC + 249036a8c5dSAndriy Gapon (((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32)); 250036a8c5dSAndriy Gapon } 251036a8c5dSAndriy Gapon 25291eaf3e1SJohn Birrell static void 253de3a96e3SMark Johnston profile_probe(profile_probe_t *prof, hrtime_t late) 25491eaf3e1SJohn Birrell { 255de3a96e3SMark Johnston struct thread *td; 256036a8c5dSAndriy Gapon struct trapframe *frame; 257036a8c5dSAndriy Gapon uintfptr_t pc, upc; 25891eaf3e1SJohn Birrell 259de3a96e3SMark Johnston td = curthread; 260de3a96e3SMark Johnston pc = upc = 0; 261036a8c5dSAndriy Gapon 262036a8c5dSAndriy Gapon /* 263de3a96e3SMark Johnston * td_intr_frame can be unset if this is a catch-up event upon waking up 264de3a96e3SMark Johnston * from idle sleep. This can only happen on a CPU idle thread. Use a 265de3a96e3SMark Johnston * representative arg0 value in this case so that one of the probe 266de3a96e3SMark Johnston * arguments is non-zero. 267036a8c5dSAndriy Gapon */ 268de3a96e3SMark Johnston frame = td->td_intr_frame; 269036a8c5dSAndriy Gapon if (frame != NULL) { 270036a8c5dSAndriy Gapon if (TRAPF_USERMODE(frame)) 271036a8c5dSAndriy Gapon upc = TRAPF_PC(frame); 272036a8c5dSAndriy Gapon else 273036a8c5dSAndriy Gapon pc = TRAPF_PC(frame); 274de3a96e3SMark Johnston } else if (TD_IS_IDLETHREAD(td)) 275de3a96e3SMark Johnston pc = (uintfptr_t)&cpu_idle; 276036a8c5dSAndriy Gapon 277de3a96e3SMark Johnston dtrace_probe(prof->prof_id, pc, upc, late, 0, 0); 278de3a96e3SMark Johnston } 279de3a96e3SMark Johnston 280de3a96e3SMark Johnston static void 281de3a96e3SMark Johnston profile_fire(void *arg) 282de3a96e3SMark Johnston { 283de3a96e3SMark Johnston profile_probe_percpu_t *pcpu = arg; 284de3a96e3SMark Johnston profile_probe_t *prof = pcpu->profc_probe; 285de3a96e3SMark Johnston hrtime_t late; 286de3a96e3SMark Johnston 287de3a96e3SMark Johnston late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected); 288de3a96e3SMark Johnston 289de3a96e3SMark Johnston profile_probe(prof, late); 290036a8c5dSAndriy Gapon pcpu->profc_expected += pcpu->profc_interval; 291036a8c5dSAndriy Gapon callout_schedule_sbt_curcpu(&pcpu->profc_cyclic, 292036a8c5dSAndriy Gapon pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE); 29391eaf3e1SJohn Birrell } 29491eaf3e1SJohn Birrell 29591eaf3e1SJohn Birrell static void 29691eaf3e1SJohn Birrell profile_tick(void *arg) 29791eaf3e1SJohn Birrell { 29891eaf3e1SJohn Birrell profile_probe_t *prof = arg; 29991eaf3e1SJohn Birrell 300de3a96e3SMark Johnston profile_probe(prof, 0); 301036a8c5dSAndriy Gapon prof->prof_expected += prof->prof_interval; 302036a8c5dSAndriy Gapon callout_schedule_sbt(&prof->prof_cyclic, 303036a8c5dSAndriy Gapon prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE); 30491eaf3e1SJohn Birrell } 30591eaf3e1SJohn Birrell 30691eaf3e1SJohn Birrell static void 30791eaf3e1SJohn Birrell profile_create(hrtime_t interval, char *name, int kind) 30891eaf3e1SJohn Birrell { 30991eaf3e1SJohn Birrell profile_probe_t *prof; 31091eaf3e1SJohn Birrell 31191eaf3e1SJohn Birrell if (interval < profile_interval_min) 31291eaf3e1SJohn Birrell return; 31391eaf3e1SJohn Birrell 31491eaf3e1SJohn Birrell if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0) 31591eaf3e1SJohn Birrell return; 31691eaf3e1SJohn Birrell 31791eaf3e1SJohn Birrell atomic_add_32(&profile_total, 1); 31891eaf3e1SJohn Birrell if (profile_total > profile_max) { 31991eaf3e1SJohn Birrell atomic_add_32(&profile_total, -1); 32091eaf3e1SJohn Birrell return; 32191eaf3e1SJohn Birrell } 32291eaf3e1SJohn Birrell 32391eaf3e1SJohn Birrell prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP); 32491eaf3e1SJohn Birrell (void) strcpy(prof->prof_name, name); 325036a8c5dSAndriy Gapon #ifdef illumos 32691eaf3e1SJohn Birrell prof->prof_interval = interval; 32791eaf3e1SJohn Birrell prof->prof_cyclic = CYCLIC_NONE; 328036a8c5dSAndriy Gapon #else 329036a8c5dSAndriy Gapon prof->prof_interval = nsec_to_sbt(interval); 330fd90e2edSJung-uk Kim callout_init(&prof->prof_cyclic, 1); 331036a8c5dSAndriy Gapon #endif 33291eaf3e1SJohn Birrell prof->prof_kind = kind; 33391eaf3e1SJohn Birrell prof->prof_id = dtrace_probe_create(profile_id, 33491eaf3e1SJohn Birrell NULL, NULL, name, 3355cde34a0SAndrew Turner profile_aframes, prof); 33691eaf3e1SJohn Birrell } 33791eaf3e1SJohn Birrell 33891eaf3e1SJohn Birrell /*ARGSUSED*/ 33991eaf3e1SJohn Birrell static void 34091eaf3e1SJohn Birrell profile_provide(void *arg, dtrace_probedesc_t *desc) 34191eaf3e1SJohn Birrell { 34291eaf3e1SJohn Birrell int i, j, rate, kind; 34391eaf3e1SJohn Birrell hrtime_t val = 0, mult = 1, len = 0; 34491eaf3e1SJohn Birrell char *name, *suffix = NULL; 34591eaf3e1SJohn Birrell 34691eaf3e1SJohn Birrell const struct { 34791eaf3e1SJohn Birrell char *prefix; 34891eaf3e1SJohn Birrell int kind; 34991eaf3e1SJohn Birrell } types[] = { 35091eaf3e1SJohn Birrell { PROF_PREFIX_PROFILE, PROF_PROFILE }, 35191eaf3e1SJohn Birrell { PROF_PREFIX_TICK, PROF_TICK }, 35291eaf3e1SJohn Birrell { 0, 0 } 35391eaf3e1SJohn Birrell }; 35491eaf3e1SJohn Birrell 35591eaf3e1SJohn Birrell const struct { 35691eaf3e1SJohn Birrell char *name; 35791eaf3e1SJohn Birrell hrtime_t mult; 35891eaf3e1SJohn Birrell } suffixes[] = { 35991eaf3e1SJohn Birrell { "ns", NANOSEC / NANOSEC }, 36091eaf3e1SJohn Birrell { "nsec", NANOSEC / NANOSEC }, 36191eaf3e1SJohn Birrell { "us", NANOSEC / MICROSEC }, 36291eaf3e1SJohn Birrell { "usec", NANOSEC / MICROSEC }, 36391eaf3e1SJohn Birrell { "ms", NANOSEC / MILLISEC }, 36491eaf3e1SJohn Birrell { "msec", NANOSEC / MILLISEC }, 36591eaf3e1SJohn Birrell { "s", NANOSEC / SEC }, 36691eaf3e1SJohn Birrell { "sec", NANOSEC / SEC }, 36791eaf3e1SJohn Birrell { "m", NANOSEC * (hrtime_t)60 }, 36891eaf3e1SJohn Birrell { "min", NANOSEC * (hrtime_t)60 }, 36991eaf3e1SJohn Birrell { "h", NANOSEC * (hrtime_t)(60 * 60) }, 37091eaf3e1SJohn Birrell { "hour", NANOSEC * (hrtime_t)(60 * 60) }, 37191eaf3e1SJohn Birrell { "d", NANOSEC * (hrtime_t)(24 * 60 * 60) }, 37291eaf3e1SJohn Birrell { "day", NANOSEC * (hrtime_t)(24 * 60 * 60) }, 37391eaf3e1SJohn Birrell { "hz", 0 }, 37491eaf3e1SJohn Birrell { NULL } 37591eaf3e1SJohn Birrell }; 37691eaf3e1SJohn Birrell 37791eaf3e1SJohn Birrell if (desc == NULL) { 37891eaf3e1SJohn Birrell char n[PROF_NAMELEN]; 37991eaf3e1SJohn Birrell 38091eaf3e1SJohn Birrell /* 38191eaf3e1SJohn Birrell * If no description was provided, provide all of our probes. 38291eaf3e1SJohn Birrell */ 38391eaf3e1SJohn Birrell for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) { 38491eaf3e1SJohn Birrell if ((rate = profile_rates[i]) == 0) 38591eaf3e1SJohn Birrell continue; 38691eaf3e1SJohn Birrell 38791eaf3e1SJohn Birrell (void) snprintf(n, PROF_NAMELEN, "%s%d", 38891eaf3e1SJohn Birrell PROF_PREFIX_PROFILE, rate); 38991eaf3e1SJohn Birrell profile_create(NANOSEC / rate, n, PROF_PROFILE); 39091eaf3e1SJohn Birrell } 39191eaf3e1SJohn Birrell 39291eaf3e1SJohn Birrell for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) { 39391eaf3e1SJohn Birrell if ((rate = profile_ticks[i]) == 0) 39491eaf3e1SJohn Birrell continue; 39591eaf3e1SJohn Birrell 39691eaf3e1SJohn Birrell (void) snprintf(n, PROF_NAMELEN, "%s%d", 39791eaf3e1SJohn Birrell PROF_PREFIX_TICK, rate); 39891eaf3e1SJohn Birrell profile_create(NANOSEC / rate, n, PROF_TICK); 39991eaf3e1SJohn Birrell } 40091eaf3e1SJohn Birrell 40191eaf3e1SJohn Birrell return; 40291eaf3e1SJohn Birrell } 40391eaf3e1SJohn Birrell 40491eaf3e1SJohn Birrell name = desc->dtpd_name; 40591eaf3e1SJohn Birrell 40691eaf3e1SJohn Birrell for (i = 0; types[i].prefix != NULL; i++) { 40791eaf3e1SJohn Birrell len = strlen(types[i].prefix); 40891eaf3e1SJohn Birrell 40991eaf3e1SJohn Birrell if (strncmp(name, types[i].prefix, len) != 0) 41091eaf3e1SJohn Birrell continue; 41191eaf3e1SJohn Birrell break; 41291eaf3e1SJohn Birrell } 41391eaf3e1SJohn Birrell 41491eaf3e1SJohn Birrell if (types[i].prefix == NULL) 41591eaf3e1SJohn Birrell return; 41691eaf3e1SJohn Birrell 41791eaf3e1SJohn Birrell kind = types[i].kind; 41891eaf3e1SJohn Birrell j = strlen(name) - len; 41991eaf3e1SJohn Birrell 42091eaf3e1SJohn Birrell /* 42191eaf3e1SJohn Birrell * We need to start before any time suffix. 42291eaf3e1SJohn Birrell */ 42391eaf3e1SJohn Birrell for (j = strlen(name); j >= len; j--) { 42491eaf3e1SJohn Birrell if (name[j] >= '0' && name[j] <= '9') 42591eaf3e1SJohn Birrell break; 42691eaf3e1SJohn Birrell suffix = &name[j]; 42791eaf3e1SJohn Birrell } 42891eaf3e1SJohn Birrell 42991eaf3e1SJohn Birrell ASSERT(suffix != NULL); 43091eaf3e1SJohn Birrell 43191eaf3e1SJohn Birrell /* 43291eaf3e1SJohn Birrell * Now determine the numerical value present in the probe name. 43391eaf3e1SJohn Birrell */ 43491eaf3e1SJohn Birrell for (; j >= len; j--) { 43591eaf3e1SJohn Birrell if (name[j] < '0' || name[j] > '9') 43691eaf3e1SJohn Birrell return; 43791eaf3e1SJohn Birrell 43891eaf3e1SJohn Birrell val += (name[j] - '0') * mult; 43991eaf3e1SJohn Birrell mult *= (hrtime_t)10; 44091eaf3e1SJohn Birrell } 44191eaf3e1SJohn Birrell 44291eaf3e1SJohn Birrell if (val == 0) 44391eaf3e1SJohn Birrell return; 44491eaf3e1SJohn Birrell 44591eaf3e1SJohn Birrell /* 44691eaf3e1SJohn Birrell * Look-up the suffix to determine the multiplier. 44791eaf3e1SJohn Birrell */ 44891eaf3e1SJohn Birrell for (i = 0, mult = 0; suffixes[i].name != NULL; i++) { 44991eaf3e1SJohn Birrell if (strcasecmp(suffixes[i].name, suffix) == 0) { 45091eaf3e1SJohn Birrell mult = suffixes[i].mult; 45191eaf3e1SJohn Birrell break; 45291eaf3e1SJohn Birrell } 45391eaf3e1SJohn Birrell } 45491eaf3e1SJohn Birrell 45591eaf3e1SJohn Birrell if (suffixes[i].name == NULL && *suffix != '\0') 45691eaf3e1SJohn Birrell return; 45791eaf3e1SJohn Birrell 45891eaf3e1SJohn Birrell if (mult == 0) { 45991eaf3e1SJohn Birrell /* 46091eaf3e1SJohn Birrell * The default is frequency-per-second. 46191eaf3e1SJohn Birrell */ 46291eaf3e1SJohn Birrell val = NANOSEC / val; 46391eaf3e1SJohn Birrell } else { 46491eaf3e1SJohn Birrell val *= mult; 46591eaf3e1SJohn Birrell } 46691eaf3e1SJohn Birrell 46791eaf3e1SJohn Birrell profile_create(val, name, kind); 46891eaf3e1SJohn Birrell } 46991eaf3e1SJohn Birrell 47091eaf3e1SJohn Birrell /* ARGSUSED */ 47191eaf3e1SJohn Birrell static void 47291eaf3e1SJohn Birrell profile_destroy(void *arg, dtrace_id_t id, void *parg) 47391eaf3e1SJohn Birrell { 47491eaf3e1SJohn Birrell profile_probe_t *prof = parg; 47591eaf3e1SJohn Birrell 476036a8c5dSAndriy Gapon #ifdef illumos 47791eaf3e1SJohn Birrell ASSERT(prof->prof_cyclic == CYCLIC_NONE); 478036a8c5dSAndriy Gapon #else 479036a8c5dSAndriy Gapon ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL); 480036a8c5dSAndriy Gapon #endif 48191eaf3e1SJohn Birrell kmem_free(prof, sizeof (profile_probe_t)); 48291eaf3e1SJohn Birrell 48391eaf3e1SJohn Birrell ASSERT(profile_total >= 1); 48491eaf3e1SJohn Birrell atomic_add_32(&profile_total, -1); 48591eaf3e1SJohn Birrell } 48691eaf3e1SJohn Birrell 487036a8c5dSAndriy Gapon #ifdef illumos 48891eaf3e1SJohn Birrell /*ARGSUSED*/ 48991eaf3e1SJohn Birrell static void 49091eaf3e1SJohn Birrell profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 49191eaf3e1SJohn Birrell { 49291eaf3e1SJohn Birrell profile_probe_t *prof = arg; 49391eaf3e1SJohn Birrell profile_probe_percpu_t *pcpu; 49491eaf3e1SJohn Birrell 49591eaf3e1SJohn Birrell pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP); 49691eaf3e1SJohn Birrell pcpu->profc_probe = prof; 49791eaf3e1SJohn Birrell 49891eaf3e1SJohn Birrell hdlr->cyh_func = profile_fire; 49991eaf3e1SJohn Birrell hdlr->cyh_arg = pcpu; 50091eaf3e1SJohn Birrell 50191eaf3e1SJohn Birrell when->cyt_interval = prof->prof_interval; 50291eaf3e1SJohn Birrell when->cyt_when = gethrtime() + when->cyt_interval; 50391eaf3e1SJohn Birrell 50491eaf3e1SJohn Birrell pcpu->profc_expected = when->cyt_when; 50591eaf3e1SJohn Birrell pcpu->profc_interval = when->cyt_interval; 50691eaf3e1SJohn Birrell } 50791eaf3e1SJohn Birrell 50891eaf3e1SJohn Birrell /*ARGSUSED*/ 50991eaf3e1SJohn Birrell static void 51091eaf3e1SJohn Birrell profile_offline(void *arg, cpu_t *cpu, void *oarg) 51191eaf3e1SJohn Birrell { 51291eaf3e1SJohn Birrell profile_probe_percpu_t *pcpu = oarg; 51391eaf3e1SJohn Birrell 51491eaf3e1SJohn Birrell ASSERT(pcpu->profc_probe == arg); 51591eaf3e1SJohn Birrell kmem_free(pcpu, sizeof (profile_probe_percpu_t)); 51691eaf3e1SJohn Birrell } 51791eaf3e1SJohn Birrell 51891eaf3e1SJohn Birrell /* ARGSUSED */ 51991eaf3e1SJohn Birrell static void 52091eaf3e1SJohn Birrell profile_enable(void *arg, dtrace_id_t id, void *parg) 52191eaf3e1SJohn Birrell { 52291eaf3e1SJohn Birrell profile_probe_t *prof = parg; 52391eaf3e1SJohn Birrell cyc_omni_handler_t omni; 52491eaf3e1SJohn Birrell cyc_handler_t hdlr; 52591eaf3e1SJohn Birrell cyc_time_t when; 52691eaf3e1SJohn Birrell 52791eaf3e1SJohn Birrell ASSERT(prof->prof_interval != 0); 52891eaf3e1SJohn Birrell ASSERT(MUTEX_HELD(&cpu_lock)); 52991eaf3e1SJohn Birrell 53091eaf3e1SJohn Birrell if (prof->prof_kind == PROF_TICK) { 53191eaf3e1SJohn Birrell hdlr.cyh_func = profile_tick; 53291eaf3e1SJohn Birrell hdlr.cyh_arg = prof; 53391eaf3e1SJohn Birrell 53491eaf3e1SJohn Birrell when.cyt_interval = prof->prof_interval; 53591eaf3e1SJohn Birrell when.cyt_when = gethrtime() + when.cyt_interval; 53691eaf3e1SJohn Birrell } else { 53791eaf3e1SJohn Birrell ASSERT(prof->prof_kind == PROF_PROFILE); 53891eaf3e1SJohn Birrell omni.cyo_online = profile_online; 53991eaf3e1SJohn Birrell omni.cyo_offline = profile_offline; 54091eaf3e1SJohn Birrell omni.cyo_arg = prof; 54191eaf3e1SJohn Birrell } 54291eaf3e1SJohn Birrell 54391eaf3e1SJohn Birrell if (prof->prof_kind == PROF_TICK) { 54491eaf3e1SJohn Birrell prof->prof_cyclic = cyclic_add(&hdlr, &when); 54591eaf3e1SJohn Birrell } else { 54691eaf3e1SJohn Birrell prof->prof_cyclic = cyclic_add_omni(&omni); 54791eaf3e1SJohn Birrell } 54891eaf3e1SJohn Birrell } 54991eaf3e1SJohn Birrell 55091eaf3e1SJohn Birrell /* ARGSUSED */ 55191eaf3e1SJohn Birrell static void 55291eaf3e1SJohn Birrell profile_disable(void *arg, dtrace_id_t id, void *parg) 55391eaf3e1SJohn Birrell { 55491eaf3e1SJohn Birrell profile_probe_t *prof = parg; 55591eaf3e1SJohn Birrell 55691eaf3e1SJohn Birrell ASSERT(prof->prof_cyclic != CYCLIC_NONE); 55791eaf3e1SJohn Birrell ASSERT(MUTEX_HELD(&cpu_lock)); 55891eaf3e1SJohn Birrell 55991eaf3e1SJohn Birrell cyclic_remove(prof->prof_cyclic); 56091eaf3e1SJohn Birrell prof->prof_cyclic = CYCLIC_NONE; 56191eaf3e1SJohn Birrell } 56291eaf3e1SJohn Birrell 563036a8c5dSAndriy Gapon #else 564036a8c5dSAndriy Gapon 565036a8c5dSAndriy Gapon static void 566036a8c5dSAndriy Gapon profile_enable_omni(profile_probe_t *prof) 567036a8c5dSAndriy Gapon { 568036a8c5dSAndriy Gapon profile_probe_percpu_t *pcpu; 569036a8c5dSAndriy Gapon int cpu; 570036a8c5dSAndriy Gapon 571036a8c5dSAndriy Gapon prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP); 572036a8c5dSAndriy Gapon CPU_FOREACH(cpu) { 573036a8c5dSAndriy Gapon pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP); 574036a8c5dSAndriy Gapon prof->prof_pcpus[cpu] = pcpu; 575036a8c5dSAndriy Gapon pcpu->profc_probe = prof; 576036a8c5dSAndriy Gapon pcpu->profc_expected = sbinuptime() + prof->prof_interval; 577036a8c5dSAndriy Gapon pcpu->profc_interval = prof->prof_interval; 578fd90e2edSJung-uk Kim callout_init(&pcpu->profc_cyclic, 1); 579036a8c5dSAndriy Gapon callout_reset_sbt_on(&pcpu->profc_cyclic, 580036a8c5dSAndriy Gapon pcpu->profc_expected, 0, profile_fire, pcpu, 581036a8c5dSAndriy Gapon cpu, C_DIRECT_EXEC | C_ABSOLUTE); 582036a8c5dSAndriy Gapon } 583036a8c5dSAndriy Gapon } 584036a8c5dSAndriy Gapon 585036a8c5dSAndriy Gapon static void 586036a8c5dSAndriy Gapon profile_disable_omni(profile_probe_t *prof) 587036a8c5dSAndriy Gapon { 588036a8c5dSAndriy Gapon profile_probe_percpu_t *pcpu; 589036a8c5dSAndriy Gapon int cpu; 590036a8c5dSAndriy Gapon 591036a8c5dSAndriy Gapon ASSERT(prof->prof_pcpus != NULL); 592036a8c5dSAndriy Gapon CPU_FOREACH(cpu) { 593036a8c5dSAndriy Gapon pcpu = prof->prof_pcpus[cpu]; 594036a8c5dSAndriy Gapon ASSERT(pcpu->profc_probe == prof); 595036a8c5dSAndriy Gapon ASSERT(callout_active(&pcpu->profc_cyclic)); 596036a8c5dSAndriy Gapon callout_stop(&pcpu->profc_cyclic); 597036a8c5dSAndriy Gapon callout_drain(&pcpu->profc_cyclic); 598036a8c5dSAndriy Gapon kmem_free(pcpu, sizeof(profile_probe_percpu_t)); 599036a8c5dSAndriy Gapon } 600036a8c5dSAndriy Gapon kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu)); 601036a8c5dSAndriy Gapon prof->prof_pcpus = NULL; 602036a8c5dSAndriy Gapon } 603036a8c5dSAndriy Gapon 604036a8c5dSAndriy Gapon /* ARGSUSED */ 605036a8c5dSAndriy Gapon static void 606036a8c5dSAndriy Gapon profile_enable(void *arg, dtrace_id_t id, void *parg) 607036a8c5dSAndriy Gapon { 608036a8c5dSAndriy Gapon profile_probe_t *prof = parg; 609036a8c5dSAndriy Gapon 610036a8c5dSAndriy Gapon if (prof->prof_kind == PROF_TICK) { 611036a8c5dSAndriy Gapon prof->prof_expected = sbinuptime() + prof->prof_interval; 612036a8c5dSAndriy Gapon callout_reset_sbt(&prof->prof_cyclic, 613036a8c5dSAndriy Gapon prof->prof_expected, 0, profile_tick, prof, 614036a8c5dSAndriy Gapon C_DIRECT_EXEC | C_ABSOLUTE); 615036a8c5dSAndriy Gapon } else { 616036a8c5dSAndriy Gapon ASSERT(prof->prof_kind == PROF_PROFILE); 617036a8c5dSAndriy Gapon profile_enable_omni(prof); 618036a8c5dSAndriy Gapon } 619036a8c5dSAndriy Gapon } 620036a8c5dSAndriy Gapon 621036a8c5dSAndriy Gapon /* ARGSUSED */ 622036a8c5dSAndriy Gapon static void 623036a8c5dSAndriy Gapon profile_disable(void *arg, dtrace_id_t id, void *parg) 624036a8c5dSAndriy Gapon { 625036a8c5dSAndriy Gapon profile_probe_t *prof = parg; 626036a8c5dSAndriy Gapon 627036a8c5dSAndriy Gapon if (prof->prof_kind == PROF_TICK) { 628036a8c5dSAndriy Gapon ASSERT(callout_active(&prof->prof_cyclic)); 629036a8c5dSAndriy Gapon callout_stop(&prof->prof_cyclic); 630036a8c5dSAndriy Gapon callout_drain(&prof->prof_cyclic); 631036a8c5dSAndriy Gapon } else { 632036a8c5dSAndriy Gapon ASSERT(prof->prof_kind == PROF_PROFILE); 633036a8c5dSAndriy Gapon profile_disable_omni(prof); 634036a8c5dSAndriy Gapon } 635036a8c5dSAndriy Gapon } 636036a8c5dSAndriy Gapon #endif 637036a8c5dSAndriy Gapon 63891eaf3e1SJohn Birrell static void 63991eaf3e1SJohn Birrell profile_load(void *dummy) 64091eaf3e1SJohn Birrell { 64191eaf3e1SJohn Birrell /* Create the /dev/dtrace/profile entry. */ 64291eaf3e1SJohn Birrell profile_cdev = make_dev(&profile_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 64391eaf3e1SJohn Birrell "dtrace/profile"); 64491eaf3e1SJohn Birrell 64591eaf3e1SJohn Birrell if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER, 64691eaf3e1SJohn Birrell NULL, &profile_pops, NULL, &profile_id) != 0) 64791eaf3e1SJohn Birrell return; 64891eaf3e1SJohn Birrell } 64991eaf3e1SJohn Birrell 65091eaf3e1SJohn Birrell 65191eaf3e1SJohn Birrell static int 65291eaf3e1SJohn Birrell profile_unload() 65391eaf3e1SJohn Birrell { 65491eaf3e1SJohn Birrell int error = 0; 65591eaf3e1SJohn Birrell 65691eaf3e1SJohn Birrell if ((error = dtrace_unregister(profile_id)) != 0) 65791eaf3e1SJohn Birrell return (error); 65891eaf3e1SJohn Birrell 65991eaf3e1SJohn Birrell destroy_dev(profile_cdev); 66091eaf3e1SJohn Birrell 66191eaf3e1SJohn Birrell return (error); 66291eaf3e1SJohn Birrell } 66391eaf3e1SJohn Birrell 66491eaf3e1SJohn Birrell /* ARGSUSED */ 66591eaf3e1SJohn Birrell static int 66691eaf3e1SJohn Birrell profile_modevent(module_t mod __unused, int type, void *data __unused) 66791eaf3e1SJohn Birrell { 66891eaf3e1SJohn Birrell int error = 0; 66991eaf3e1SJohn Birrell 67091eaf3e1SJohn Birrell switch (type) { 67191eaf3e1SJohn Birrell case MOD_LOAD: 67291eaf3e1SJohn Birrell break; 67391eaf3e1SJohn Birrell 67491eaf3e1SJohn Birrell case MOD_UNLOAD: 67591eaf3e1SJohn Birrell break; 67691eaf3e1SJohn Birrell 67791eaf3e1SJohn Birrell case MOD_SHUTDOWN: 67891eaf3e1SJohn Birrell break; 67991eaf3e1SJohn Birrell 68091eaf3e1SJohn Birrell default: 68191eaf3e1SJohn Birrell error = EOPNOTSUPP; 68291eaf3e1SJohn Birrell break; 68391eaf3e1SJohn Birrell 68491eaf3e1SJohn Birrell } 68591eaf3e1SJohn Birrell return (error); 68691eaf3e1SJohn Birrell } 68791eaf3e1SJohn Birrell 68891eaf3e1SJohn Birrell /* ARGSUSED */ 68991eaf3e1SJohn Birrell static int 69091eaf3e1SJohn Birrell profile_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) 69191eaf3e1SJohn Birrell { 69291eaf3e1SJohn Birrell return (0); 69391eaf3e1SJohn Birrell } 69491eaf3e1SJohn Birrell 69591eaf3e1SJohn Birrell SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL); 69691eaf3e1SJohn Birrell SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL); 69791eaf3e1SJohn Birrell 69891eaf3e1SJohn Birrell DEV_MODULE(profile, profile_modevent, NULL); 69991eaf3e1SJohn Birrell MODULE_VERSION(profile, 1); 70091eaf3e1SJohn Birrell MODULE_DEPEND(profile, dtrace, 1, 1, 1); 70191eaf3e1SJohn Birrell MODULE_DEPEND(profile, opensolaris, 1, 1, 1); 702