1 /*- 2 * Copyright (c) 2003-2007, Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 33 /* 34 * PMC interface used by the base kernel. 35 */ 36 37 #ifndef _SYS_PMCKERN_H_ 38 #define _SYS_PMCKERN_H_ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/lock.h> 44 #include <sys/proc.h> 45 #include <sys/sx.h> 46 #include <sys/pmc.h> 47 48 #include <machine/cpufunc.h> 49 50 #define PMC_FN_PROCESS_EXEC 1 51 #define PMC_FN_CSW_IN 2 52 #define PMC_FN_CSW_OUT 3 53 #define PMC_FN_DO_SAMPLES 4 54 #define PMC_FN_UNUSED1 5 55 #define PMC_FN_UNUSED2 6 56 #define PMC_FN_MMAP 7 57 #define PMC_FN_MUNMAP 8 58 #define PMC_FN_USER_CALLCHAIN 9 59 #define PMC_FN_USER_CALLCHAIN_SOFT 10 60 #define PMC_FN_SOFT_SAMPLING 11 61 62 #define PMC_HR 0 /* Hardware ring buffer */ 63 #define PMC_SR 1 /* Software ring buffer */ 64 65 struct pmckern_procexec { 66 int pm_credentialschanged; 67 uintfptr_t pm_entryaddr; 68 }; 69 70 struct pmckern_map_in { 71 void *pm_file; /* filename or vnode pointer */ 72 uintfptr_t pm_address; /* address object is loaded at */ 73 }; 74 75 struct pmckern_map_out { 76 uintfptr_t pm_address; /* start address of region */ 77 size_t pm_size; /* size of unmapped region */ 78 }; 79 80 struct pmckern_soft { 81 enum pmc_event pm_ev; 82 int pm_cpu; 83 struct trapframe *pm_tf; 84 }; 85 86 /* 87 * Soft PMC. 88 */ 89 90 #define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ 91 struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ 92 { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ 93 SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ 94 SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ 95 &pmc_##prov##_##mod##_##func##_##name ); \ 96 SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ 97 SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ 98 &pmc_##prov##_##mod##_##func##_##name ) 99 100 #define PMC_SOFT_DEFINE(prov, mod, func, name) \ 101 PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) 102 103 #define PMC_SOFT_DECLARE(prov, mod, func, name) \ 104 extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name 105 106 /* 107 * PMC_SOFT_CALL can be used anywhere in the kernel. 108 * Require md defined PMC_FAKE_TRAPFRAME. 109 */ 110 #ifdef PMC_FAKE_TRAPFRAME 111 #define PMC_SOFT_CALL(pr, mo, fu, na) \ 112 do { \ 113 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 114 struct pmckern_soft ks; \ 115 register_t intr; \ 116 intr = intr_disable(); \ 117 PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ 118 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 119 ks.pm_cpu = PCPU_GET(cpuid); \ 120 ks.pm_tf = &pmc_tf[curcpu]; \ 121 PMC_CALL_HOOK_UNLOCKED(curthread, \ 122 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 123 intr_restore(intr); \ 124 } \ 125 } while (0) 126 #else 127 #define PMC_SOFT_CALL(pr, mo, fu, na) \ 128 do { \ 129 } while (0) 130 #endif 131 132 /* 133 * PMC_SOFT_CALL_TF need to be used carefully. 134 * Userland capture will be done during AST processing. 135 */ 136 #define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ 137 do { \ 138 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 139 struct pmckern_soft ks; \ 140 register_t intr; \ 141 intr = intr_disable(); \ 142 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 143 ks.pm_cpu = PCPU_GET(cpuid); \ 144 ks.pm_tf = tf; \ 145 PMC_CALL_HOOK_UNLOCKED(curthread, \ 146 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 147 intr_restore(intr); \ 148 } \ 149 } while (0) 150 151 struct pmc_soft { 152 int ps_running; 153 void (*ps_alloc)(void); 154 void (*ps_release)(void); 155 struct pmc_dyn_event_descr ps_ev; 156 }; 157 158 /* hook */ 159 extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); 160 extern int (*pmc_intr)(int _cpu, struct trapframe *_frame); 161 162 /* SX lock protecting the hook */ 163 extern struct sx pmc_sx; 164 165 /* Per-cpu flags indicating availability of sampling data */ 166 extern volatile cpuset_t pmc_cpumask; 167 168 /* Count of system-wide sampling PMCs in existence */ 169 extern volatile int pmc_ss_count; 170 171 /* kernel version number */ 172 extern const int pmc_kernel_version; 173 174 /* PMC soft per cpu trapframe */ 175 extern struct trapframe pmc_tf[MAXCPU]; 176 177 /* Quick check if preparatory work is necessary */ 178 #define PMC_HOOK_INSTALLED(cmd) __predict_false(pmc_hook != NULL) 179 180 /* Hook invocation; for use within the kernel */ 181 #define PMC_CALL_HOOK(t, cmd, arg) \ 182 do { \ 183 sx_slock(&pmc_sx); \ 184 if (pmc_hook != NULL) \ 185 (pmc_hook)((t), (cmd), (arg)); \ 186 sx_sunlock(&pmc_sx); \ 187 } while (0) 188 189 /* Hook invocation that needs an exclusive lock */ 190 #define PMC_CALL_HOOK_X(t, cmd, arg) \ 191 do { \ 192 sx_xlock(&pmc_sx); \ 193 if (pmc_hook != NULL) \ 194 (pmc_hook)((t), (cmd), (arg)); \ 195 sx_xunlock(&pmc_sx); \ 196 } while (0) 197 198 /* 199 * Some hook invocations (e.g., from context switch and clock handling 200 * code) need to be lock-free. 201 */ 202 #define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ 203 do { \ 204 if (pmc_hook != NULL) \ 205 (pmc_hook)((t), (cmd), (arg)); \ 206 } while (0) 207 208 #define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) 209 210 /* Check if a process is using HWPMCs.*/ 211 #define PMC_PROC_IS_USING_PMCS(p) \ 212 (__predict_false(p->p_flag & P_HWPMC)) 213 214 /* Check if a thread have pending user capture. */ 215 #define PMC_IS_PENDING_CALLCHAIN(p) \ 216 (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) 217 218 #define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) 219 220 /* Check if a CPU has recorded samples. */ 221 #define PMC_CPU_HAS_SAMPLES(C) (__predict_false(CPU_ISSET(C, &pmc_cpumask))) 222 223 /* 224 * Helper functions. 225 */ 226 int pmc_cpu_is_disabled(int _cpu); /* deprecated */ 227 int pmc_cpu_is_active(int _cpu); 228 int pmc_cpu_is_present(int _cpu); 229 int pmc_cpu_is_primary(int _cpu); 230 unsigned int pmc_cpu_max(void); 231 232 #ifdef INVARIANTS 233 int pmc_cpu_max_active(void); 234 #endif 235 236 /* 237 * Soft events functions. 238 */ 239 void pmc_soft_ev_register(struct pmc_soft *ps); 240 void pmc_soft_ev_deregister(struct pmc_soft *ps); 241 struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); 242 void pmc_soft_ev_release(struct pmc_soft *ps); 243 244 #endif /* _SYS_PMCKERN_H_ */ 245