191eaf3e1SJohn Birrell /*
291eaf3e1SJohn Birrell * CDDL HEADER START
391eaf3e1SJohn Birrell *
491eaf3e1SJohn Birrell * The contents of this file are subject to the terms of the
591eaf3e1SJohn Birrell * Common Development and Distribution License (the "License").
691eaf3e1SJohn Birrell * You may not use this file except in compliance with the License.
791eaf3e1SJohn Birrell *
891eaf3e1SJohn Birrell * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
991eaf3e1SJohn Birrell * or http://www.opensolaris.org/os/licensing.
1091eaf3e1SJohn Birrell * See the License for the specific language governing permissions
1191eaf3e1SJohn Birrell * and limitations under the License.
1291eaf3e1SJohn Birrell *
1391eaf3e1SJohn Birrell * When distributing Covered Code, include this CDDL HEADER in each
1491eaf3e1SJohn Birrell * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1591eaf3e1SJohn Birrell * If applicable, add the following below this CDDL HEADER, with the
1691eaf3e1SJohn Birrell * fields enclosed by brackets "[]" replaced with your own identifying
1791eaf3e1SJohn Birrell * information: Portions Copyright [yyyy] [name of copyright owner]
1891eaf3e1SJohn Birrell *
1991eaf3e1SJohn Birrell * CDDL HEADER END
2091eaf3e1SJohn Birrell *
2191eaf3e1SJohn Birrell * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
2291eaf3e1SJohn Birrell *
2391eaf3e1SJohn Birrell */
2491eaf3e1SJohn Birrell
2591eaf3e1SJohn Birrell /*
2691eaf3e1SJohn Birrell * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
2791eaf3e1SJohn Birrell * Use is subject to license terms.
2891eaf3e1SJohn Birrell */
2991eaf3e1SJohn Birrell
3091eaf3e1SJohn Birrell #include <sys/param.h>
3191eaf3e1SJohn Birrell #include <sys/systm.h>
3291eaf3e1SJohn Birrell #include <sys/conf.h>
3391eaf3e1SJohn Birrell #include <sys/cpuvar.h>
349e5787d2SMatt Macy #include <sys/endian.h>
3591eaf3e1SJohn Birrell #include <sys/fcntl.h>
3691eaf3e1SJohn Birrell #include <sys/filio.h>
3791eaf3e1SJohn Birrell #include <sys/kdb.h>
3891eaf3e1SJohn Birrell #include <sys/kernel.h>
3991eaf3e1SJohn Birrell #include <sys/kmem.h>
4091eaf3e1SJohn Birrell #include <sys/kthread.h>
4191eaf3e1SJohn Birrell #include <sys/limits.h>
4291eaf3e1SJohn Birrell #include <sys/linker.h>
4391eaf3e1SJohn Birrell #include <sys/lock.h>
4491eaf3e1SJohn Birrell #include <sys/malloc.h>
4591eaf3e1SJohn Birrell #include <sys/module.h>
4691eaf3e1SJohn Birrell #include <sys/mutex.h>
4791eaf3e1SJohn Birrell #include <sys/poll.h>
4891eaf3e1SJohn Birrell #include <sys/proc.h>
4991eaf3e1SJohn Birrell #include <sys/selinfo.h>
5091eaf3e1SJohn Birrell #include <sys/smp.h>
515cde34a0SAndrew Turner #include <sys/sysctl.h>
5291eaf3e1SJohn Birrell #include <sys/uio.h>
5391eaf3e1SJohn Birrell #include <sys/unistd.h>
54036a8c5dSAndriy Gapon #include <machine/cpu.h>
5591eaf3e1SJohn Birrell #include <machine/stdarg.h>
5691eaf3e1SJohn Birrell
5791eaf3e1SJohn Birrell #include <sys/dtrace.h>
5891eaf3e1SJohn Birrell #include <sys/dtrace_bsd.h>
5991eaf3e1SJohn Birrell
607d35b389SMark Johnston #include <cddl/dev/dtrace/dtrace_cddl.h>
617d35b389SMark Johnston
6291eaf3e1SJohn Birrell #define PROF_NAMELEN 15
6391eaf3e1SJohn Birrell
6491eaf3e1SJohn Birrell #define PROF_PROFILE 0
6591eaf3e1SJohn Birrell #define PROF_TICK 1
6691eaf3e1SJohn Birrell #define PROF_PREFIX_PROFILE "profile-"
6791eaf3e1SJohn Birrell #define PROF_PREFIX_TICK "tick-"
6891eaf3e1SJohn Birrell
6991eaf3e1SJohn Birrell /*
7091eaf3e1SJohn Birrell * Regardless of platform, there are five artificial frames in the case of the
7191eaf3e1SJohn Birrell * profile provider:
7291eaf3e1SJohn Birrell *
7391eaf3e1SJohn Birrell * profile_fire
7491eaf3e1SJohn Birrell * cyclic_expire
7591eaf3e1SJohn Birrell * cyclic_fire
7691eaf3e1SJohn Birrell * [ cbe ]
7791eaf3e1SJohn Birrell * [ locore ]
7891eaf3e1SJohn Birrell *
7991eaf3e1SJohn Birrell * On amd64, there are two frames associated with locore: one in locore, and
8091eaf3e1SJohn Birrell * another in common interrupt dispatch code. (i386 has not been modified to
8191eaf3e1SJohn Birrell * use this common layer.) Further, on i386, the interrupted instruction
8291eaf3e1SJohn Birrell * appears as its own stack frame. All of this means that we need to add one
8391eaf3e1SJohn Birrell * frame for amd64, and then take one away for both amd64 and i386.
8491eaf3e1SJohn Birrell *
8591eaf3e1SJohn Birrell * All of the above constraints lead to the mess below. Yes, the profile
8691eaf3e1SJohn Birrell * provider should ideally figure this out on-the-fly by hiting one of its own
8791eaf3e1SJohn Birrell * probes and then walking its own stack trace. This is complicated, however,
8891eaf3e1SJohn Birrell * and the static definition doesn't seem to be overly brittle. Still, we
8991eaf3e1SJohn Birrell * allow for a manual override in case we get it completely wrong.
9091eaf3e1SJohn Birrell */
9191eaf3e1SJohn Birrell #ifdef __amd64
92036a8c5dSAndriy Gapon #define PROF_ARTIFICIAL_FRAMES 10
9391eaf3e1SJohn Birrell #else
9491eaf3e1SJohn Birrell #ifdef __i386
9591eaf3e1SJohn Birrell #define PROF_ARTIFICIAL_FRAMES 6
9691eaf3e1SJohn Birrell #endif
9791eaf3e1SJohn Birrell #endif
9891eaf3e1SJohn Birrell
99c7570492SJustin Hibbits #ifdef __powerpc__
100c7570492SJustin Hibbits /*
101c7570492SJustin Hibbits * This value is bogus just to make module compilable on powerpc
102c7570492SJustin Hibbits */
103635ecbf4SJustin Hibbits #define PROF_ARTIFICIAL_FRAMES 8
104c7570492SJustin Hibbits #endif
105c7570492SJustin Hibbits
106036a8c5dSAndriy Gapon struct profile_probe_percpu;
107036a8c5dSAndriy Gapon
108fcb56067SGeorge V. Neville-Neil #ifdef __arm__
1095cde34a0SAndrew Turner #define PROF_ARTIFICIAL_FRAMES 3
110fcb56067SGeorge V. Neville-Neil #endif
111fcb56067SGeorge V. Neville-Neil
112b78ee15eSRuslan Bukin #ifdef __aarch64__
113599fb1d1SRobert Watson #define PROF_ARTIFICIAL_FRAMES 12
114b78ee15eSRuslan Bukin #endif
115b78ee15eSRuslan Bukin
116ca20f8ecSRuslan Bukin #ifdef __riscv
11740fdda02SMitchell Horne #define PROF_ARTIFICIAL_FRAMES 12
118fed1ca4bSRuslan Bukin #endif
119fed1ca4bSRuslan Bukin
12091eaf3e1SJohn Birrell typedef struct profile_probe {
12191eaf3e1SJohn Birrell char prof_name[PROF_NAMELEN];
12291eaf3e1SJohn Birrell dtrace_id_t prof_id;
12391eaf3e1SJohn Birrell int prof_kind;
124036a8c5dSAndriy Gapon #ifdef illumos
12591eaf3e1SJohn Birrell hrtime_t prof_interval;
12691eaf3e1SJohn Birrell cyclic_id_t prof_cyclic;
127036a8c5dSAndriy Gapon #else
128036a8c5dSAndriy Gapon sbintime_t prof_interval;
129036a8c5dSAndriy Gapon struct callout prof_cyclic;
130036a8c5dSAndriy Gapon sbintime_t prof_expected;
131036a8c5dSAndriy Gapon struct profile_probe_percpu **prof_pcpus;
132036a8c5dSAndriy Gapon #endif
13391eaf3e1SJohn Birrell } profile_probe_t;
13491eaf3e1SJohn Birrell
13591eaf3e1SJohn Birrell typedef struct profile_probe_percpu {
13691eaf3e1SJohn Birrell hrtime_t profc_expected;
13791eaf3e1SJohn Birrell hrtime_t profc_interval;
13891eaf3e1SJohn Birrell profile_probe_t *profc_probe;
139036a8c5dSAndriy Gapon #ifdef __FreeBSD__
140036a8c5dSAndriy Gapon struct callout profc_cyclic;
141036a8c5dSAndriy Gapon #endif
14291eaf3e1SJohn Birrell } profile_probe_percpu_t;
14391eaf3e1SJohn Birrell
14491eaf3e1SJohn Birrell static int profile_unload(void);
14591eaf3e1SJohn Birrell static void profile_create(hrtime_t, char *, int);
14691eaf3e1SJohn Birrell static void profile_destroy(void *, dtrace_id_t, void *);
14791eaf3e1SJohn Birrell static void profile_enable(void *, dtrace_id_t, void *);
14891eaf3e1SJohn Birrell static void profile_disable(void *, dtrace_id_t, void *);
14991eaf3e1SJohn Birrell static void profile_load(void *);
15091eaf3e1SJohn Birrell static void profile_provide(void *, dtrace_probedesc_t *);
15191eaf3e1SJohn Birrell
15291eaf3e1SJohn Birrell static int profile_rates[] = {
15391eaf3e1SJohn Birrell 97, 199, 499, 997, 1999,
15491eaf3e1SJohn Birrell 4001, 4999, 0, 0, 0,
15591eaf3e1SJohn Birrell 0, 0, 0, 0, 0,
15691eaf3e1SJohn Birrell 0, 0, 0, 0, 0
15791eaf3e1SJohn Birrell };
15891eaf3e1SJohn Birrell
15991eaf3e1SJohn Birrell static int profile_ticks[] = {
16091eaf3e1SJohn Birrell 1, 10, 100, 500, 1000,
16191eaf3e1SJohn Birrell 5000, 0, 0, 0, 0,
16291eaf3e1SJohn Birrell 0, 0, 0, 0, 0
16391eaf3e1SJohn Birrell };
16491eaf3e1SJohn Birrell
16591eaf3e1SJohn Birrell /*
16691eaf3e1SJohn Birrell * profile_max defines the upper bound on the number of profile probes that
16791eaf3e1SJohn Birrell * can exist (this is to prevent malicious or clumsy users from exhausing
16891eaf3e1SJohn Birrell * system resources by creating a slew of profile probes). At mod load time,
16991eaf3e1SJohn Birrell * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's
17091eaf3e1SJohn Birrell * present in the profile.conf file.
17191eaf3e1SJohn Birrell */
17291eaf3e1SJohn Birrell #define PROFILE_MAX_DEFAULT 1000 /* default max. number of probes */
17391eaf3e1SJohn Birrell static uint32_t profile_max = PROFILE_MAX_DEFAULT;
17491eaf3e1SJohn Birrell /* maximum number of profile probes */
17591eaf3e1SJohn Birrell static uint32_t profile_total; /* current number of profile probes */
17691eaf3e1SJohn Birrell
17791eaf3e1SJohn Birrell static dtrace_pattr_t profile_attr = {
17891eaf3e1SJohn Birrell { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
17991eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
18091eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
18191eaf3e1SJohn Birrell { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
18291eaf3e1SJohn Birrell { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
18391eaf3e1SJohn Birrell };
18491eaf3e1SJohn Birrell
18591eaf3e1SJohn Birrell static dtrace_pops_t profile_pops = {
18647f11baaSMark Johnston .dtps_provide = profile_provide,
18747f11baaSMark Johnston .dtps_provide_module = NULL,
18847f11baaSMark Johnston .dtps_enable = profile_enable,
18947f11baaSMark Johnston .dtps_disable = profile_disable,
19047f11baaSMark Johnston .dtps_suspend = NULL,
19147f11baaSMark Johnston .dtps_resume = NULL,
19247f11baaSMark Johnston .dtps_getargdesc = NULL,
19347f11baaSMark Johnston .dtps_getargval = NULL,
19447f11baaSMark Johnston .dtps_usermode = NULL,
19547f11baaSMark Johnston .dtps_destroy = profile_destroy
19691eaf3e1SJohn Birrell };
19791eaf3e1SJohn Birrell
19891eaf3e1SJohn Birrell static dtrace_provider_id_t profile_id;
19991eaf3e1SJohn Birrell static hrtime_t profile_interval_min = NANOSEC / 5000; /* 5000 hz */
2005cde34a0SAndrew Turner static int profile_aframes = PROF_ARTIFICIAL_FRAMES;
2015cde34a0SAndrew Turner
2025cde34a0SAndrew Turner SYSCTL_DECL(_kern_dtrace);
2037029da5cSPawel Biernacki SYSCTL_NODE(_kern_dtrace, OID_AUTO, profile, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
2047029da5cSPawel Biernacki "DTrace profile parameters");
2055cde34a0SAndrew Turner SYSCTL_INT(_kern_dtrace_profile, OID_AUTO, aframes, CTLFLAG_RW, &profile_aframes,
2065cde34a0SAndrew Turner 0, "Skipped frames for profile provider");
20791eaf3e1SJohn Birrell
208036a8c5dSAndriy Gapon static sbintime_t
nsec_to_sbt(hrtime_t nsec)209036a8c5dSAndriy Gapon nsec_to_sbt(hrtime_t nsec)
210036a8c5dSAndriy Gapon {
211036a8c5dSAndriy Gapon time_t sec;
212036a8c5dSAndriy Gapon
213036a8c5dSAndriy Gapon /*
214036a8c5dSAndriy Gapon * We need to calculate nsec * 2^32 / 10^9
215036a8c5dSAndriy Gapon * Seconds and nanoseconds are split to avoid overflow.
216036a8c5dSAndriy Gapon */
217036a8c5dSAndriy Gapon sec = nsec / NANOSEC;
218036a8c5dSAndriy Gapon nsec = nsec % NANOSEC;
219036a8c5dSAndriy Gapon return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC);
220036a8c5dSAndriy Gapon }
221036a8c5dSAndriy Gapon
222036a8c5dSAndriy Gapon static hrtime_t
sbt_to_nsec(sbintime_t sbt)223036a8c5dSAndriy Gapon sbt_to_nsec(sbintime_t sbt)
224036a8c5dSAndriy Gapon {
225036a8c5dSAndriy Gapon
226036a8c5dSAndriy Gapon return ((sbt >> 32) * NANOSEC +
227036a8c5dSAndriy Gapon (((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32));
228036a8c5dSAndriy Gapon }
229036a8c5dSAndriy Gapon
23091eaf3e1SJohn Birrell static void
profile_probe(profile_probe_t * prof,hrtime_t late)231de3a96e3SMark Johnston profile_probe(profile_probe_t *prof, hrtime_t late)
23291eaf3e1SJohn Birrell {
233de3a96e3SMark Johnston struct thread *td;
234036a8c5dSAndriy Gapon struct trapframe *frame;
235036a8c5dSAndriy Gapon uintfptr_t pc, upc;
23691eaf3e1SJohn Birrell
237de3a96e3SMark Johnston td = curthread;
238de3a96e3SMark Johnston pc = upc = 0;
239036a8c5dSAndriy Gapon
240036a8c5dSAndriy Gapon /*
241de3a96e3SMark Johnston * td_intr_frame can be unset if this is a catch-up event upon waking up
242de3a96e3SMark Johnston * from idle sleep. This can only happen on a CPU idle thread. Use a
243de3a96e3SMark Johnston * representative arg0 value in this case so that one of the probe
244de3a96e3SMark Johnston * arguments is non-zero.
245036a8c5dSAndriy Gapon */
246de3a96e3SMark Johnston frame = td->td_intr_frame;
247036a8c5dSAndriy Gapon if (frame != NULL) {
248036a8c5dSAndriy Gapon if (TRAPF_USERMODE(frame))
249036a8c5dSAndriy Gapon upc = TRAPF_PC(frame);
2507d35b389SMark Johnston else {
251036a8c5dSAndriy Gapon pc = TRAPF_PC(frame);
2527d35b389SMark Johnston td->t_dtrace_trapframe = frame;
2537d35b389SMark Johnston }
254de3a96e3SMark Johnston } else if (TD_IS_IDLETHREAD(td))
255de3a96e3SMark Johnston pc = (uintfptr_t)&cpu_idle;
256036a8c5dSAndriy Gapon
257de3a96e3SMark Johnston dtrace_probe(prof->prof_id, pc, upc, late, 0, 0);
2587d35b389SMark Johnston td->t_dtrace_trapframe = NULL;
259de3a96e3SMark Johnston }
260de3a96e3SMark Johnston
261de3a96e3SMark Johnston static void
profile_fire(void * arg)262de3a96e3SMark Johnston profile_fire(void *arg)
263de3a96e3SMark Johnston {
264de3a96e3SMark Johnston profile_probe_percpu_t *pcpu = arg;
265de3a96e3SMark Johnston profile_probe_t *prof = pcpu->profc_probe;
266de3a96e3SMark Johnston hrtime_t late;
267de3a96e3SMark Johnston
268de3a96e3SMark Johnston late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected);
269de3a96e3SMark Johnston
270de3a96e3SMark Johnston profile_probe(prof, late);
271036a8c5dSAndriy Gapon pcpu->profc_expected += pcpu->profc_interval;
272036a8c5dSAndriy Gapon callout_schedule_sbt_curcpu(&pcpu->profc_cyclic,
273036a8c5dSAndriy Gapon pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
27491eaf3e1SJohn Birrell }
27591eaf3e1SJohn Birrell
27691eaf3e1SJohn Birrell static void
profile_tick(void * arg)27791eaf3e1SJohn Birrell profile_tick(void *arg)
27891eaf3e1SJohn Birrell {
27991eaf3e1SJohn Birrell profile_probe_t *prof = arg;
28091eaf3e1SJohn Birrell
281de3a96e3SMark Johnston profile_probe(prof, 0);
282036a8c5dSAndriy Gapon prof->prof_expected += prof->prof_interval;
283036a8c5dSAndriy Gapon callout_schedule_sbt(&prof->prof_cyclic,
284036a8c5dSAndriy Gapon prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
28591eaf3e1SJohn Birrell }
28691eaf3e1SJohn Birrell
28791eaf3e1SJohn Birrell static void
profile_create(hrtime_t interval,char * name,int kind)28891eaf3e1SJohn Birrell profile_create(hrtime_t interval, char *name, int kind)
28991eaf3e1SJohn Birrell {
29091eaf3e1SJohn Birrell profile_probe_t *prof;
29191eaf3e1SJohn Birrell
29291eaf3e1SJohn Birrell if (interval < profile_interval_min)
29391eaf3e1SJohn Birrell return;
29491eaf3e1SJohn Birrell
29591eaf3e1SJohn Birrell if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
29691eaf3e1SJohn Birrell return;
29791eaf3e1SJohn Birrell
29891eaf3e1SJohn Birrell atomic_add_32(&profile_total, 1);
29991eaf3e1SJohn Birrell if (profile_total > profile_max) {
30091eaf3e1SJohn Birrell atomic_add_32(&profile_total, -1);
30191eaf3e1SJohn Birrell return;
30291eaf3e1SJohn Birrell }
30391eaf3e1SJohn Birrell
30491eaf3e1SJohn Birrell prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
30591eaf3e1SJohn Birrell (void) strcpy(prof->prof_name, name);
306036a8c5dSAndriy Gapon #ifdef illumos
30791eaf3e1SJohn Birrell prof->prof_interval = interval;
30891eaf3e1SJohn Birrell prof->prof_cyclic = CYCLIC_NONE;
309036a8c5dSAndriy Gapon #else
310036a8c5dSAndriy Gapon prof->prof_interval = nsec_to_sbt(interval);
311fd90e2edSJung-uk Kim callout_init(&prof->prof_cyclic, 1);
312036a8c5dSAndriy Gapon #endif
31391eaf3e1SJohn Birrell prof->prof_kind = kind;
31491eaf3e1SJohn Birrell prof->prof_id = dtrace_probe_create(profile_id,
31591eaf3e1SJohn Birrell NULL, NULL, name,
3165cde34a0SAndrew Turner profile_aframes, prof);
31791eaf3e1SJohn Birrell }
31891eaf3e1SJohn Birrell
31991eaf3e1SJohn Birrell /*ARGSUSED*/
32091eaf3e1SJohn Birrell static void
profile_provide(void * arg,dtrace_probedesc_t * desc)32191eaf3e1SJohn Birrell profile_provide(void *arg, dtrace_probedesc_t *desc)
32291eaf3e1SJohn Birrell {
32391eaf3e1SJohn Birrell int i, j, rate, kind;
32491eaf3e1SJohn Birrell hrtime_t val = 0, mult = 1, len = 0;
32591eaf3e1SJohn Birrell char *name, *suffix = NULL;
32691eaf3e1SJohn Birrell
32791eaf3e1SJohn Birrell const struct {
32891eaf3e1SJohn Birrell char *prefix;
32991eaf3e1SJohn Birrell int kind;
33091eaf3e1SJohn Birrell } types[] = {
33191eaf3e1SJohn Birrell { PROF_PREFIX_PROFILE, PROF_PROFILE },
33291eaf3e1SJohn Birrell { PROF_PREFIX_TICK, PROF_TICK },
33391eaf3e1SJohn Birrell { 0, 0 }
33491eaf3e1SJohn Birrell };
33591eaf3e1SJohn Birrell
33691eaf3e1SJohn Birrell const struct {
33791eaf3e1SJohn Birrell char *name;
33891eaf3e1SJohn Birrell hrtime_t mult;
33991eaf3e1SJohn Birrell } suffixes[] = {
34091eaf3e1SJohn Birrell { "ns", NANOSEC / NANOSEC },
34191eaf3e1SJohn Birrell { "nsec", NANOSEC / NANOSEC },
34291eaf3e1SJohn Birrell { "us", NANOSEC / MICROSEC },
34391eaf3e1SJohn Birrell { "usec", NANOSEC / MICROSEC },
34491eaf3e1SJohn Birrell { "ms", NANOSEC / MILLISEC },
34591eaf3e1SJohn Birrell { "msec", NANOSEC / MILLISEC },
34691eaf3e1SJohn Birrell { "s", NANOSEC / SEC },
34791eaf3e1SJohn Birrell { "sec", NANOSEC / SEC },
34891eaf3e1SJohn Birrell { "m", NANOSEC * (hrtime_t)60 },
34991eaf3e1SJohn Birrell { "min", NANOSEC * (hrtime_t)60 },
35091eaf3e1SJohn Birrell { "h", NANOSEC * (hrtime_t)(60 * 60) },
35191eaf3e1SJohn Birrell { "hour", NANOSEC * (hrtime_t)(60 * 60) },
35291eaf3e1SJohn Birrell { "d", NANOSEC * (hrtime_t)(24 * 60 * 60) },
35391eaf3e1SJohn Birrell { "day", NANOSEC * (hrtime_t)(24 * 60 * 60) },
35491eaf3e1SJohn Birrell { "hz", 0 },
35591eaf3e1SJohn Birrell { NULL }
35691eaf3e1SJohn Birrell };
35791eaf3e1SJohn Birrell
35891eaf3e1SJohn Birrell if (desc == NULL) {
35991eaf3e1SJohn Birrell char n[PROF_NAMELEN];
36091eaf3e1SJohn Birrell
36191eaf3e1SJohn Birrell /*
36291eaf3e1SJohn Birrell * If no description was provided, provide all of our probes.
36391eaf3e1SJohn Birrell */
36491eaf3e1SJohn Birrell for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) {
36591eaf3e1SJohn Birrell if ((rate = profile_rates[i]) == 0)
36691eaf3e1SJohn Birrell continue;
36791eaf3e1SJohn Birrell
36891eaf3e1SJohn Birrell (void) snprintf(n, PROF_NAMELEN, "%s%d",
36991eaf3e1SJohn Birrell PROF_PREFIX_PROFILE, rate);
37091eaf3e1SJohn Birrell profile_create(NANOSEC / rate, n, PROF_PROFILE);
37191eaf3e1SJohn Birrell }
37291eaf3e1SJohn Birrell
37391eaf3e1SJohn Birrell for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) {
37491eaf3e1SJohn Birrell if ((rate = profile_ticks[i]) == 0)
37591eaf3e1SJohn Birrell continue;
37691eaf3e1SJohn Birrell
37791eaf3e1SJohn Birrell (void) snprintf(n, PROF_NAMELEN, "%s%d",
37891eaf3e1SJohn Birrell PROF_PREFIX_TICK, rate);
37991eaf3e1SJohn Birrell profile_create(NANOSEC / rate, n, PROF_TICK);
38091eaf3e1SJohn Birrell }
38191eaf3e1SJohn Birrell
38291eaf3e1SJohn Birrell return;
38391eaf3e1SJohn Birrell }
38491eaf3e1SJohn Birrell
38591eaf3e1SJohn Birrell name = desc->dtpd_name;
38691eaf3e1SJohn Birrell
38791eaf3e1SJohn Birrell for (i = 0; types[i].prefix != NULL; i++) {
38891eaf3e1SJohn Birrell len = strlen(types[i].prefix);
38991eaf3e1SJohn Birrell
39091eaf3e1SJohn Birrell if (strncmp(name, types[i].prefix, len) != 0)
39191eaf3e1SJohn Birrell continue;
39291eaf3e1SJohn Birrell break;
39391eaf3e1SJohn Birrell }
39491eaf3e1SJohn Birrell
39591eaf3e1SJohn Birrell if (types[i].prefix == NULL)
39691eaf3e1SJohn Birrell return;
39791eaf3e1SJohn Birrell
39891eaf3e1SJohn Birrell kind = types[i].kind;
39991eaf3e1SJohn Birrell j = strlen(name) - len;
40091eaf3e1SJohn Birrell
40191eaf3e1SJohn Birrell /*
40291eaf3e1SJohn Birrell * We need to start before any time suffix.
40391eaf3e1SJohn Birrell */
40491eaf3e1SJohn Birrell for (j = strlen(name); j >= len; j--) {
40591eaf3e1SJohn Birrell if (name[j] >= '0' && name[j] <= '9')
40691eaf3e1SJohn Birrell break;
40791eaf3e1SJohn Birrell suffix = &name[j];
40891eaf3e1SJohn Birrell }
40991eaf3e1SJohn Birrell
41091eaf3e1SJohn Birrell ASSERT(suffix != NULL);
41191eaf3e1SJohn Birrell
41291eaf3e1SJohn Birrell /*
41391eaf3e1SJohn Birrell * Now determine the numerical value present in the probe name.
41491eaf3e1SJohn Birrell */
41591eaf3e1SJohn Birrell for (; j >= len; j--) {
41691eaf3e1SJohn Birrell if (name[j] < '0' || name[j] > '9')
41791eaf3e1SJohn Birrell return;
41891eaf3e1SJohn Birrell
41991eaf3e1SJohn Birrell val += (name[j] - '0') * mult;
42091eaf3e1SJohn Birrell mult *= (hrtime_t)10;
42191eaf3e1SJohn Birrell }
42291eaf3e1SJohn Birrell
42391eaf3e1SJohn Birrell if (val == 0)
42491eaf3e1SJohn Birrell return;
42591eaf3e1SJohn Birrell
42691eaf3e1SJohn Birrell /*
42791eaf3e1SJohn Birrell * Look-up the suffix to determine the multiplier.
42891eaf3e1SJohn Birrell */
42991eaf3e1SJohn Birrell for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
43091eaf3e1SJohn Birrell if (strcasecmp(suffixes[i].name, suffix) == 0) {
43191eaf3e1SJohn Birrell mult = suffixes[i].mult;
43291eaf3e1SJohn Birrell break;
43391eaf3e1SJohn Birrell }
43491eaf3e1SJohn Birrell }
43591eaf3e1SJohn Birrell
43691eaf3e1SJohn Birrell if (suffixes[i].name == NULL && *suffix != '\0')
43791eaf3e1SJohn Birrell return;
43891eaf3e1SJohn Birrell
43991eaf3e1SJohn Birrell if (mult == 0) {
44091eaf3e1SJohn Birrell /*
44191eaf3e1SJohn Birrell * The default is frequency-per-second.
44291eaf3e1SJohn Birrell */
44391eaf3e1SJohn Birrell val = NANOSEC / val;
44491eaf3e1SJohn Birrell } else {
44591eaf3e1SJohn Birrell val *= mult;
44691eaf3e1SJohn Birrell }
44791eaf3e1SJohn Birrell
44891eaf3e1SJohn Birrell profile_create(val, name, kind);
44991eaf3e1SJohn Birrell }
45091eaf3e1SJohn Birrell
45191eaf3e1SJohn Birrell /* ARGSUSED */
45291eaf3e1SJohn Birrell static void
profile_destroy(void * arg,dtrace_id_t id,void * parg)45391eaf3e1SJohn Birrell profile_destroy(void *arg, dtrace_id_t id, void *parg)
45491eaf3e1SJohn Birrell {
45591eaf3e1SJohn Birrell profile_probe_t *prof = parg;
45691eaf3e1SJohn Birrell
457036a8c5dSAndriy Gapon #ifdef illumos
45891eaf3e1SJohn Birrell ASSERT(prof->prof_cyclic == CYCLIC_NONE);
459036a8c5dSAndriy Gapon #else
460036a8c5dSAndriy Gapon ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL);
461036a8c5dSAndriy Gapon #endif
46291eaf3e1SJohn Birrell kmem_free(prof, sizeof (profile_probe_t));
46391eaf3e1SJohn Birrell
46491eaf3e1SJohn Birrell ASSERT(profile_total >= 1);
46591eaf3e1SJohn Birrell atomic_add_32(&profile_total, -1);
46691eaf3e1SJohn Birrell }
46791eaf3e1SJohn Birrell
468036a8c5dSAndriy Gapon #ifdef illumos
46991eaf3e1SJohn Birrell /*ARGSUSED*/
47091eaf3e1SJohn Birrell static void
profile_online(void * arg,cpu_t * cpu,cyc_handler_t * hdlr,cyc_time_t * when)47191eaf3e1SJohn Birrell profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
47291eaf3e1SJohn Birrell {
47391eaf3e1SJohn Birrell profile_probe_t *prof = arg;
47491eaf3e1SJohn Birrell profile_probe_percpu_t *pcpu;
47591eaf3e1SJohn Birrell
47691eaf3e1SJohn Birrell pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP);
47791eaf3e1SJohn Birrell pcpu->profc_probe = prof;
47891eaf3e1SJohn Birrell
47991eaf3e1SJohn Birrell hdlr->cyh_func = profile_fire;
48091eaf3e1SJohn Birrell hdlr->cyh_arg = pcpu;
48191eaf3e1SJohn Birrell
48291eaf3e1SJohn Birrell when->cyt_interval = prof->prof_interval;
48391eaf3e1SJohn Birrell when->cyt_when = gethrtime() + when->cyt_interval;
48491eaf3e1SJohn Birrell
48591eaf3e1SJohn Birrell pcpu->profc_expected = when->cyt_when;
48691eaf3e1SJohn Birrell pcpu->profc_interval = when->cyt_interval;
48791eaf3e1SJohn Birrell }
48891eaf3e1SJohn Birrell
48991eaf3e1SJohn Birrell /*ARGSUSED*/
49091eaf3e1SJohn Birrell static void
profile_offline(void * arg,cpu_t * cpu,void * oarg)49191eaf3e1SJohn Birrell profile_offline(void *arg, cpu_t *cpu, void *oarg)
49291eaf3e1SJohn Birrell {
49391eaf3e1SJohn Birrell profile_probe_percpu_t *pcpu = oarg;
49491eaf3e1SJohn Birrell
49591eaf3e1SJohn Birrell ASSERT(pcpu->profc_probe == arg);
49691eaf3e1SJohn Birrell kmem_free(pcpu, sizeof (profile_probe_percpu_t));
49791eaf3e1SJohn Birrell }
49891eaf3e1SJohn Birrell
49991eaf3e1SJohn Birrell /* ARGSUSED */
50091eaf3e1SJohn Birrell static void
profile_enable(void * arg,dtrace_id_t id,void * parg)50191eaf3e1SJohn Birrell profile_enable(void *arg, dtrace_id_t id, void *parg)
50291eaf3e1SJohn Birrell {
50391eaf3e1SJohn Birrell profile_probe_t *prof = parg;
50491eaf3e1SJohn Birrell cyc_omni_handler_t omni;
50591eaf3e1SJohn Birrell cyc_handler_t hdlr;
50691eaf3e1SJohn Birrell cyc_time_t when;
50791eaf3e1SJohn Birrell
50891eaf3e1SJohn Birrell ASSERT(prof->prof_interval != 0);
50991eaf3e1SJohn Birrell ASSERT(MUTEX_HELD(&cpu_lock));
51091eaf3e1SJohn Birrell
51191eaf3e1SJohn Birrell if (prof->prof_kind == PROF_TICK) {
51291eaf3e1SJohn Birrell hdlr.cyh_func = profile_tick;
51391eaf3e1SJohn Birrell hdlr.cyh_arg = prof;
51491eaf3e1SJohn Birrell
51591eaf3e1SJohn Birrell when.cyt_interval = prof->prof_interval;
51691eaf3e1SJohn Birrell when.cyt_when = gethrtime() + when.cyt_interval;
51791eaf3e1SJohn Birrell } else {
51891eaf3e1SJohn Birrell ASSERT(prof->prof_kind == PROF_PROFILE);
51991eaf3e1SJohn Birrell omni.cyo_online = profile_online;
52091eaf3e1SJohn Birrell omni.cyo_offline = profile_offline;
52191eaf3e1SJohn Birrell omni.cyo_arg = prof;
52291eaf3e1SJohn Birrell }
52391eaf3e1SJohn Birrell
52491eaf3e1SJohn Birrell if (prof->prof_kind == PROF_TICK) {
52591eaf3e1SJohn Birrell prof->prof_cyclic = cyclic_add(&hdlr, &when);
52691eaf3e1SJohn Birrell } else {
52791eaf3e1SJohn Birrell prof->prof_cyclic = cyclic_add_omni(&omni);
52891eaf3e1SJohn Birrell }
52991eaf3e1SJohn Birrell }
53091eaf3e1SJohn Birrell
53191eaf3e1SJohn Birrell /* ARGSUSED */
53291eaf3e1SJohn Birrell static void
profile_disable(void * arg,dtrace_id_t id,void * parg)53391eaf3e1SJohn Birrell profile_disable(void *arg, dtrace_id_t id, void *parg)
53491eaf3e1SJohn Birrell {
53591eaf3e1SJohn Birrell profile_probe_t *prof = parg;
53691eaf3e1SJohn Birrell
53791eaf3e1SJohn Birrell ASSERT(prof->prof_cyclic != CYCLIC_NONE);
53891eaf3e1SJohn Birrell ASSERT(MUTEX_HELD(&cpu_lock));
53991eaf3e1SJohn Birrell
54091eaf3e1SJohn Birrell cyclic_remove(prof->prof_cyclic);
54191eaf3e1SJohn Birrell prof->prof_cyclic = CYCLIC_NONE;
54291eaf3e1SJohn Birrell }
54391eaf3e1SJohn Birrell
544036a8c5dSAndriy Gapon #else
545036a8c5dSAndriy Gapon
546036a8c5dSAndriy Gapon static void
profile_enable_omni(profile_probe_t * prof)547036a8c5dSAndriy Gapon profile_enable_omni(profile_probe_t *prof)
548036a8c5dSAndriy Gapon {
549036a8c5dSAndriy Gapon profile_probe_percpu_t *pcpu;
550036a8c5dSAndriy Gapon int cpu;
551036a8c5dSAndriy Gapon
552036a8c5dSAndriy Gapon prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP);
553036a8c5dSAndriy Gapon CPU_FOREACH(cpu) {
554036a8c5dSAndriy Gapon pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP);
555036a8c5dSAndriy Gapon prof->prof_pcpus[cpu] = pcpu;
556036a8c5dSAndriy Gapon pcpu->profc_probe = prof;
557036a8c5dSAndriy Gapon pcpu->profc_expected = sbinuptime() + prof->prof_interval;
558036a8c5dSAndriy Gapon pcpu->profc_interval = prof->prof_interval;
559fd90e2edSJung-uk Kim callout_init(&pcpu->profc_cyclic, 1);
560036a8c5dSAndriy Gapon callout_reset_sbt_on(&pcpu->profc_cyclic,
561036a8c5dSAndriy Gapon pcpu->profc_expected, 0, profile_fire, pcpu,
562036a8c5dSAndriy Gapon cpu, C_DIRECT_EXEC | C_ABSOLUTE);
563036a8c5dSAndriy Gapon }
564036a8c5dSAndriy Gapon }
565036a8c5dSAndriy Gapon
566036a8c5dSAndriy Gapon static void
profile_disable_omni(profile_probe_t * prof)567036a8c5dSAndriy Gapon profile_disable_omni(profile_probe_t *prof)
568036a8c5dSAndriy Gapon {
569036a8c5dSAndriy Gapon profile_probe_percpu_t *pcpu;
570036a8c5dSAndriy Gapon int cpu;
571036a8c5dSAndriy Gapon
572036a8c5dSAndriy Gapon ASSERT(prof->prof_pcpus != NULL);
573036a8c5dSAndriy Gapon CPU_FOREACH(cpu) {
574036a8c5dSAndriy Gapon pcpu = prof->prof_pcpus[cpu];
575036a8c5dSAndriy Gapon ASSERT(pcpu->profc_probe == prof);
576036a8c5dSAndriy Gapon ASSERT(callout_active(&pcpu->profc_cyclic));
577036a8c5dSAndriy Gapon callout_stop(&pcpu->profc_cyclic);
578036a8c5dSAndriy Gapon callout_drain(&pcpu->profc_cyclic);
579036a8c5dSAndriy Gapon kmem_free(pcpu, sizeof(profile_probe_percpu_t));
580036a8c5dSAndriy Gapon }
581036a8c5dSAndriy Gapon kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu));
582036a8c5dSAndriy Gapon prof->prof_pcpus = NULL;
583036a8c5dSAndriy Gapon }
584036a8c5dSAndriy Gapon
585036a8c5dSAndriy Gapon /* ARGSUSED */
586036a8c5dSAndriy Gapon static void
profile_enable(void * arg,dtrace_id_t id,void * parg)587036a8c5dSAndriy Gapon profile_enable(void *arg, dtrace_id_t id, void *parg)
588036a8c5dSAndriy Gapon {
589036a8c5dSAndriy Gapon profile_probe_t *prof = parg;
590036a8c5dSAndriy Gapon
591036a8c5dSAndriy Gapon if (prof->prof_kind == PROF_TICK) {
592036a8c5dSAndriy Gapon prof->prof_expected = sbinuptime() + prof->prof_interval;
593036a8c5dSAndriy Gapon callout_reset_sbt(&prof->prof_cyclic,
594036a8c5dSAndriy Gapon prof->prof_expected, 0, profile_tick, prof,
595036a8c5dSAndriy Gapon C_DIRECT_EXEC | C_ABSOLUTE);
596036a8c5dSAndriy Gapon } else {
597036a8c5dSAndriy Gapon ASSERT(prof->prof_kind == PROF_PROFILE);
598036a8c5dSAndriy Gapon profile_enable_omni(prof);
599036a8c5dSAndriy Gapon }
600036a8c5dSAndriy Gapon }
601036a8c5dSAndriy Gapon
602036a8c5dSAndriy Gapon /* ARGSUSED */
603036a8c5dSAndriy Gapon static void
profile_disable(void * arg,dtrace_id_t id,void * parg)604036a8c5dSAndriy Gapon profile_disable(void *arg, dtrace_id_t id, void *parg)
605036a8c5dSAndriy Gapon {
606036a8c5dSAndriy Gapon profile_probe_t *prof = parg;
607036a8c5dSAndriy Gapon
608036a8c5dSAndriy Gapon if (prof->prof_kind == PROF_TICK) {
609036a8c5dSAndriy Gapon ASSERT(callout_active(&prof->prof_cyclic));
610036a8c5dSAndriy Gapon callout_stop(&prof->prof_cyclic);
611036a8c5dSAndriy Gapon callout_drain(&prof->prof_cyclic);
612036a8c5dSAndriy Gapon } else {
613036a8c5dSAndriy Gapon ASSERT(prof->prof_kind == PROF_PROFILE);
614036a8c5dSAndriy Gapon profile_disable_omni(prof);
615036a8c5dSAndriy Gapon }
616036a8c5dSAndriy Gapon }
617036a8c5dSAndriy Gapon #endif
618036a8c5dSAndriy Gapon
61991eaf3e1SJohn Birrell static void
profile_load(void * dummy)62091eaf3e1SJohn Birrell profile_load(void *dummy)
62191eaf3e1SJohn Birrell {
62291eaf3e1SJohn Birrell if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER,
62391eaf3e1SJohn Birrell NULL, &profile_pops, NULL, &profile_id) != 0)
62491eaf3e1SJohn Birrell return;
62591eaf3e1SJohn Birrell }
62691eaf3e1SJohn Birrell
62791eaf3e1SJohn Birrell
62891eaf3e1SJohn Birrell static int
profile_unload(void)6292d03b58fSDimitry Andric profile_unload(void)
63091eaf3e1SJohn Birrell {
63191eaf3e1SJohn Birrell int error = 0;
63291eaf3e1SJohn Birrell
63391eaf3e1SJohn Birrell if ((error = dtrace_unregister(profile_id)) != 0)
63491eaf3e1SJohn Birrell return (error);
63591eaf3e1SJohn Birrell
63691eaf3e1SJohn Birrell return (error);
63791eaf3e1SJohn Birrell }
63891eaf3e1SJohn Birrell
63991eaf3e1SJohn Birrell /* ARGSUSED */
64091eaf3e1SJohn Birrell static int
profile_modevent(module_t mod __unused,int type,void * data __unused)64191eaf3e1SJohn Birrell profile_modevent(module_t mod __unused, int type, void *data __unused)
64291eaf3e1SJohn Birrell {
64391eaf3e1SJohn Birrell int error = 0;
64491eaf3e1SJohn Birrell
64591eaf3e1SJohn Birrell switch (type) {
64691eaf3e1SJohn Birrell case MOD_LOAD:
64791eaf3e1SJohn Birrell break;
64891eaf3e1SJohn Birrell
64991eaf3e1SJohn Birrell case MOD_UNLOAD:
65091eaf3e1SJohn Birrell break;
65191eaf3e1SJohn Birrell
65291eaf3e1SJohn Birrell case MOD_SHUTDOWN:
65391eaf3e1SJohn Birrell break;
65491eaf3e1SJohn Birrell
65591eaf3e1SJohn Birrell default:
65691eaf3e1SJohn Birrell error = EOPNOTSUPP;
65791eaf3e1SJohn Birrell break;
65891eaf3e1SJohn Birrell
65991eaf3e1SJohn Birrell }
66091eaf3e1SJohn Birrell return (error);
66191eaf3e1SJohn Birrell }
66291eaf3e1SJohn Birrell
66391eaf3e1SJohn Birrell SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL);
66491eaf3e1SJohn Birrell SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL);
66591eaf3e1SJohn Birrell
66691eaf3e1SJohn Birrell DEV_MODULE(profile, profile_modevent, NULL);
66791eaf3e1SJohn Birrell MODULE_VERSION(profile, 1);
66891eaf3e1SJohn Birrell MODULE_DEPEND(profile, dtrace, 1, 1, 1);
66991eaf3e1SJohn Birrell MODULE_DEPEND(profile, opensolaris, 1, 1, 1);
670