1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright 2022 Mark Johnston <markj@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/bitset.h> 10 #include <sys/cred.h> 11 #include <sys/eventhandler.h> 12 #include <sys/kernel.h> 13 #include <sys/lock.h> 14 #include <sys/malloc.h> 15 #include <sys/proc.h> 16 #include <sys/queue.h> 17 #include <sys/sx.h> 18 19 #include <vm/vm.h> 20 #include <vm/vm_param.h> 21 #include <vm/pmap.h> 22 #include <vm/vm_map.h> 23 #include <vm/vm_kern.h> 24 #include <vm/vm_object.h> 25 26 #include <cddl/dev/dtrace/dtrace_cddl.h> 27 28 #include "kinst.h" 29 #include "kinst_isa.h" 30 31 /* 32 * We can have 4KB/32B = 128 trampolines per chunk. 33 */ 34 #define KINST_TRAMPS_PER_CHUNK (KINST_TRAMPCHUNK_SIZE / KINST_TRAMP_SIZE) 35 36 struct trampchunk { 37 TAILQ_ENTRY(trampchunk) next; 38 uint8_t *addr; 39 /* 0 -> allocated, 1 -> free */ 40 BITSET_DEFINE(, KINST_TRAMPS_PER_CHUNK) free; 41 }; 42 43 static TAILQ_HEAD(, trampchunk) kinst_trampchunks = 44 TAILQ_HEAD_INITIALIZER(kinst_trampchunks); 45 static struct sx kinst_tramp_sx; 46 SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp"); 47 static eventhandler_tag kinst_thread_ctor_handler; 48 static eventhandler_tag kinst_thread_dtor_handler; 49 50 static struct trampchunk * 51 kinst_trampchunk_alloc(void) 52 { 53 struct trampchunk *chunk; 54 vm_offset_t trampaddr; 55 int error __diagused; 56 57 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 58 59 /* 60 * Allocate virtual memory for the trampoline chunk. The returned 61 * address is saved in "trampaddr". To simplify population of 62 * trampolines, we follow the amd64 kernel's code model and allocate 63 * them above KERNBASE, i.e., in the top 2GB of the kernel's virtual 64 * address space. Trampolines must be executable so max_prot must 65 * include VM_PROT_EXECUTE. 66 */ 67 trampaddr = KERNBASE; 68 error = vm_map_find(kernel_map, NULL, 0, &trampaddr, 69 KINST_TRAMPCHUNK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, 70 0); 71 if (error != KERN_SUCCESS) { 72 KINST_LOG("trampoline chunk allocation failed: %d", error); 73 return (NULL); 74 } 75 76 error = kmem_back(kernel_object, trampaddr, KINST_TRAMPCHUNK_SIZE, 77 M_WAITOK | M_EXEC); 78 KASSERT(error == KERN_SUCCESS, ("kmem_back failed: %d", error)); 79 80 KINST_TRAMP_INIT((void *)trampaddr, KINST_TRAMPCHUNK_SIZE); 81 82 /* Allocate a tracker for this chunk. */ 83 chunk = malloc(sizeof(*chunk), M_KINST, M_WAITOK); 84 chunk->addr = (void *)trampaddr; 85 BIT_FILL(KINST_TRAMPS_PER_CHUNK, &chunk->free); 86 87 TAILQ_INSERT_HEAD(&kinst_trampchunks, chunk, next); 88 89 return (chunk); 90 } 91 92 static void 93 kinst_trampchunk_free(struct trampchunk *chunk) 94 { 95 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 96 97 TAILQ_REMOVE(&kinst_trampchunks, chunk, next); 98 kmem_unback(kernel_object, (vm_offset_t)chunk->addr, 99 KINST_TRAMPCHUNK_SIZE); 100 (void)vm_map_remove(kernel_map, (vm_offset_t)chunk->addr, 101 (vm_offset_t)(chunk->addr + KINST_TRAMPCHUNK_SIZE)); 102 free(chunk, M_KINST); 103 } 104 105 static uint8_t * 106 kinst_trampoline_alloc_locked(int how) 107 { 108 struct trampchunk *chunk; 109 uint8_t *tramp; 110 int off; 111 112 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 113 114 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 115 /* All trampolines from this chunk are already allocated. */ 116 if ((off = BIT_FFS(KINST_TRAMPS_PER_CHUNK, &chunk->free)) == 0) 117 continue; 118 /* BIT_FFS() returns indices starting at 1 instead of 0. */ 119 off--; 120 break; 121 } 122 if (chunk == NULL) { 123 if ((how & M_NOWAIT) != 0) 124 return (NULL); 125 126 /* 127 * We didn't find any free trampoline in the current list, 128 * allocate a new one. If that fails the provider will no 129 * longer be reliable, so try to warn the user. 130 */ 131 if ((chunk = kinst_trampchunk_alloc()) == NULL) { 132 static bool once = true; 133 134 if (once) { 135 once = false; 136 KINST_LOG( 137 "kinst: failed to allocate trampoline, " 138 "probes may not fire"); 139 } 140 return (NULL); 141 } 142 off = 0; 143 } 144 BIT_CLR(KINST_TRAMPS_PER_CHUNK, off, &chunk->free); 145 tramp = chunk->addr + off * KINST_TRAMP_SIZE; 146 return (tramp); 147 } 148 149 uint8_t * 150 kinst_trampoline_alloc(int how) 151 { 152 uint8_t *tramp; 153 154 sx_xlock(&kinst_tramp_sx); 155 tramp = kinst_trampoline_alloc_locked(how); 156 sx_xunlock(&kinst_tramp_sx); 157 return (tramp); 158 } 159 160 static void 161 kinst_trampoline_dealloc_locked(uint8_t *tramp, bool freechunks) 162 { 163 struct trampchunk *chunk; 164 int off; 165 166 if (tramp == NULL) 167 return; 168 169 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 170 for (off = 0; off < KINST_TRAMPS_PER_CHUNK; off++) { 171 if (chunk->addr + off * KINST_TRAMP_SIZE == tramp) { 172 KINST_TRAMP_INIT(tramp, KINST_TRAMP_SIZE); 173 BIT_SET(KINST_TRAMPS_PER_CHUNK, off, 174 &chunk->free); 175 if (freechunks && 176 BIT_ISFULLSET(KINST_TRAMPS_PER_CHUNK, 177 &chunk->free)) 178 kinst_trampchunk_free(chunk); 179 return; 180 } 181 } 182 } 183 panic("%s: did not find trampoline chunk for %p", __func__, tramp); 184 } 185 186 void 187 kinst_trampoline_dealloc(uint8_t *tramp) 188 { 189 sx_xlock(&kinst_tramp_sx); 190 kinst_trampoline_dealloc_locked(tramp, true); 191 sx_xunlock(&kinst_tramp_sx); 192 } 193 194 static void 195 kinst_thread_ctor(void *arg __unused, struct thread *td) 196 { 197 td->t_kinst = kinst_trampoline_alloc(M_WAITOK); 198 } 199 200 static void 201 kinst_thread_dtor(void *arg __unused, struct thread *td) 202 { 203 void *tramp; 204 205 tramp = td->t_kinst; 206 td->t_kinst = NULL; 207 208 /* 209 * This assumes that the thread_dtor event permits sleeping, which 210 * appears to be true for the time being. 211 */ 212 kinst_trampoline_dealloc(tramp); 213 } 214 215 int 216 kinst_trampoline_init(void) 217 { 218 struct proc *p; 219 struct thread *td; 220 void *tramp; 221 int error; 222 223 kinst_thread_ctor_handler = EVENTHANDLER_REGISTER(thread_ctor, 224 kinst_thread_ctor, NULL, EVENTHANDLER_PRI_ANY); 225 kinst_thread_dtor_handler = EVENTHANDLER_REGISTER(thread_dtor, 226 kinst_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 227 228 error = 0; 229 tramp = NULL; 230 231 sx_slock(&allproc_lock); 232 sx_xlock(&kinst_tramp_sx); 233 FOREACH_PROC_IN_SYSTEM(p) { 234 retry: 235 PROC_LOCK(p); 236 FOREACH_THREAD_IN_PROC(p, td) { 237 if (td->t_kinst != NULL) 238 continue; 239 if (tramp == NULL) { 240 /* 241 * Try to allocate a trampoline without dropping 242 * the process lock. If all chunks are fully 243 * utilized, we must release the lock and try 244 * again. 245 */ 246 tramp = kinst_trampoline_alloc_locked(M_NOWAIT); 247 if (tramp == NULL) { 248 PROC_UNLOCK(p); 249 tramp = kinst_trampoline_alloc_locked( 250 M_WAITOK); 251 if (tramp == NULL) { 252 /* 253 * Let the unload handler clean 254 * up. 255 */ 256 error = ENOMEM; 257 goto out; 258 } else 259 goto retry; 260 } 261 } 262 td->t_kinst = tramp; 263 tramp = NULL; 264 } 265 PROC_UNLOCK(p); 266 } 267 out: 268 sx_xunlock(&kinst_tramp_sx); 269 sx_sunlock(&allproc_lock); 270 return (error); 271 } 272 273 int 274 kinst_trampoline_deinit(void) 275 { 276 struct trampchunk *chunk, *tmp; 277 struct proc *p; 278 struct thread *td; 279 280 EVENTHANDLER_DEREGISTER(thread_ctor, kinst_thread_ctor_handler); 281 EVENTHANDLER_DEREGISTER(thread_dtor, kinst_thread_dtor_handler); 282 283 sx_slock(&allproc_lock); 284 sx_xlock(&kinst_tramp_sx); 285 FOREACH_PROC_IN_SYSTEM(p) { 286 PROC_LOCK(p); 287 FOREACH_THREAD_IN_PROC(p, td) { 288 kinst_trampoline_dealloc_locked(td->t_kinst, false); 289 td->t_kinst = NULL; 290 } 291 PROC_UNLOCK(p); 292 } 293 sx_sunlock(&allproc_lock); 294 TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) 295 kinst_trampchunk_free(chunk); 296 sx_xunlock(&kinst_tramp_sx); 297 298 return (0); 299 } 300