1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2007, Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $FreeBSD$ 33 */ 34 35 /* 36 * PMC interface used by the base kernel. 37 */ 38 39 #ifndef _SYS_PMCKERN_H_ 40 #define _SYS_PMCKERN_H_ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/proc.h> 47 #include <sys/sx.h> 48 #include <sys/pmc.h> 49 50 #include <machine/cpufunc.h> 51 52 #define PMC_FN_PROCESS_EXEC 1 53 #define PMC_FN_CSW_IN 2 54 #define PMC_FN_CSW_OUT 3 55 #define PMC_FN_DO_SAMPLES 4 56 #define PMC_FN_UNUSED1 5 57 #define PMC_FN_UNUSED2 6 58 #define PMC_FN_MMAP 7 59 #define PMC_FN_MUNMAP 8 60 #define PMC_FN_USER_CALLCHAIN 9 61 #define PMC_FN_USER_CALLCHAIN_SOFT 10 62 #define PMC_FN_SOFT_SAMPLING 11 63 #define PMC_FN_THR_CREATE 12 64 #define PMC_FN_THR_EXIT 13 65 #define PMC_FN_THR_USERRET 14 66 #define PMC_FN_THR_CREATE_LOG 15 67 #define PMC_FN_THR_EXIT_LOG 16 68 #define PMC_FN_PROC_CREATE_LOG 17 69 70 #define PMC_HR 0 /* Hardware ring buffer */ 71 #define PMC_SR 1 /* Software ring buffer */ 72 #define PMC_UR 2 /* userret ring buffer */ 73 #define PMC_NUM_SR (PMC_UR+1) 74 75 struct pmckern_procexec { 76 int pm_credentialschanged; 77 uintfptr_t pm_entryaddr; 78 }; 79 80 struct pmckern_map_in { 81 void *pm_file; /* filename or vnode pointer */ 82 uintfptr_t pm_address; /* address object is loaded at */ 83 }; 84 85 struct pmckern_map_out { 86 uintfptr_t pm_address; /* start address of region */ 87 size_t pm_size; /* size of unmapped region */ 88 }; 89 90 struct pmckern_soft { 91 enum pmc_event pm_ev; 92 int pm_cpu; 93 struct trapframe *pm_tf; 94 }; 95 96 /* 97 * Soft PMC. 98 */ 99 100 #define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ 101 struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ 102 { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ 103 SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ 104 SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ 105 &pmc_##prov##_##mod##_##func##_##name ); \ 106 SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ 107 SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ 108 &pmc_##prov##_##mod##_##func##_##name ) 109 110 #define PMC_SOFT_DEFINE(prov, mod, func, name) \ 111 PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) 112 113 #define PMC_SOFT_DECLARE(prov, mod, func, name) \ 114 extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name 115 116 /* 117 * PMC_SOFT_CALL can be used anywhere in the kernel. 118 * Require md defined PMC_FAKE_TRAPFRAME. 119 */ 120 #ifdef PMC_FAKE_TRAPFRAME 121 #define PMC_SOFT_CALL(pr, mo, fu, na) \ 122 do { \ 123 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 124 struct pmckern_soft ks; \ 125 register_t intr; \ 126 intr = intr_disable(); \ 127 PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ 128 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 129 ks.pm_cpu = PCPU_GET(cpuid); \ 130 ks.pm_tf = &pmc_tf[curcpu]; \ 131 PMC_CALL_HOOK_UNLOCKED(curthread, \ 132 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 133 intr_restore(intr); \ 134 } \ 135 } while (0) 136 #else 137 #define PMC_SOFT_CALL(pr, mo, fu, na) \ 138 do { \ 139 } while (0) 140 #endif 141 142 /* 143 * PMC_SOFT_CALL_TF need to be used carefully. 144 * Userland capture will be done during AST processing. 145 */ 146 #define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ 147 do { \ 148 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 149 struct pmckern_soft ks; \ 150 register_t intr; \ 151 intr = intr_disable(); \ 152 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 153 ks.pm_cpu = PCPU_GET(cpuid); \ 154 ks.pm_tf = tf; \ 155 PMC_CALL_HOOK_UNLOCKED(curthread, \ 156 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 157 intr_restore(intr); \ 158 } \ 159 } while (0) 160 161 struct pmc_soft { 162 int ps_running; 163 void (*ps_alloc)(void); 164 void (*ps_release)(void); 165 struct pmc_dyn_event_descr ps_ev; 166 }; 167 168 struct pmclog_buffer; 169 170 struct pmc_domain_buffer_header { 171 struct mtx pdbh_mtx; 172 TAILQ_HEAD(, pmclog_buffer) pdbh_head; 173 struct pmclog_buffer *pdbh_plbs; 174 int pdbh_ncpus; 175 } __aligned(CACHE_LINE_SIZE); 176 177 /* hook */ 178 extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); 179 extern int (*pmc_intr)(struct trapframe *_frame); 180 181 /* SX lock protecting the hook */ 182 extern struct sx pmc_sx; 183 184 /* Per-cpu flags indicating availability of sampling data */ 185 DPCPU_DECLARE(uint8_t, pmc_sampled); 186 187 /* Count of system-wide sampling PMCs in existence */ 188 extern volatile int pmc_ss_count; 189 190 /* kernel version number */ 191 extern const int pmc_kernel_version; 192 193 /* PMC soft per cpu trapframe */ 194 extern struct trapframe pmc_tf[MAXCPU]; 195 196 /* per domain buffer header list */ 197 extern struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; 198 199 /* Quick check if preparatory work is necessary */ 200 #define PMC_HOOK_INSTALLED(cmd) __predict_false(pmc_hook != NULL) 201 202 /* Hook invocation; for use within the kernel */ 203 #define PMC_CALL_HOOK(t, cmd, arg) \ 204 do { \ 205 struct epoch_tracker et; \ 206 epoch_enter_preempt(global_epoch_preempt, &et); \ 207 if (pmc_hook != NULL) \ 208 (pmc_hook)((t), (cmd), (arg)); \ 209 epoch_exit_preempt(global_epoch_preempt, &et); \ 210 } while (0) 211 212 /* Hook invocation that needs an exclusive lock */ 213 #define PMC_CALL_HOOK_X(t, cmd, arg) \ 214 do { \ 215 sx_xlock(&pmc_sx); \ 216 if (pmc_hook != NULL) \ 217 (pmc_hook)((t), (cmd), (arg)); \ 218 sx_xunlock(&pmc_sx); \ 219 } while (0) 220 221 /* 222 * Some hook invocations (e.g., from context switch and clock handling 223 * code) need to be lock-free. 224 */ 225 #define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ 226 do { \ 227 if (pmc_hook != NULL) \ 228 (pmc_hook)((t), (cmd), (arg)); \ 229 } while (0) 230 231 #define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) 232 233 /* Check if a process is using HWPMCs.*/ 234 #define PMC_PROC_IS_USING_PMCS(p) \ 235 (__predict_false(p->p_flag & P_HWPMC)) 236 237 #define PMC_THREAD_HAS_SAMPLES(td) \ 238 (__predict_false((td)->td_pmcpend)) 239 240 /* Check if a thread have pending user capture. */ 241 #define PMC_IS_PENDING_CALLCHAIN(p) \ 242 (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) 243 244 #define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) 245 246 /* Check if a CPU has recorded samples. */ 247 #define PMC_CPU_HAS_SAMPLES(C) (__predict_false(DPCPU_ID_GET((C), pmc_sampled))) 248 249 /* 250 * Helper functions. 251 */ 252 int pmc_cpu_is_disabled(int _cpu); /* deprecated */ 253 int pmc_cpu_is_active(int _cpu); 254 int pmc_cpu_is_present(int _cpu); 255 int pmc_cpu_is_primary(int _cpu); 256 unsigned int pmc_cpu_max(void); 257 258 #ifdef INVARIANTS 259 int pmc_cpu_max_active(void); 260 #endif 261 262 /* 263 * Soft events functions. 264 */ 265 void pmc_soft_ev_register(struct pmc_soft *ps); 266 void pmc_soft_ev_deregister(struct pmc_soft *ps); 267 struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); 268 void pmc_soft_ev_release(struct pmc_soft *ps); 269 270 #endif /* _SYS_PMCKERN_H_ */ 271