1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright 2022 Christos Margiolis <christos@FreeBSD.org> 5 */ 6 7 #include <sys/param.h> 8 #include <sys/systm.h> 9 #include <sys/conf.h> 10 #include <sys/kernel.h> 11 #include <sys/linker.h> 12 #include <sys/module.h> 13 14 #include <sys/dtrace.h> 15 16 #include "kinst.h" 17 18 MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing"); 19 20 static d_open_t kinst_open; 21 static d_close_t kinst_close; 22 static d_ioctl_t kinst_ioctl; 23 24 static void kinst_provide_module(void *, modctl_t *); 25 static void kinst_getargdesc(void *, dtrace_id_t, void *, 26 dtrace_argdesc_t *); 27 static void kinst_destroy(void *, dtrace_id_t, void *); 28 static void kinst_enable(void *, dtrace_id_t, void *); 29 static void kinst_disable(void *, dtrace_id_t, void *); 30 static int kinst_load(void *); 31 static int kinst_unload(void *); 32 static int kinst_modevent(module_t, int, void *); 33 34 static dtrace_pattr_t kinst_attr = { 35 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 36 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 37 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 38 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 39 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 40 }; 41 42 static const dtrace_pops_t kinst_pops = { 43 .dtps_provide = NULL, 44 .dtps_provide_module = kinst_provide_module, 45 .dtps_enable = kinst_enable, 46 .dtps_disable = kinst_disable, 47 .dtps_suspend = NULL, 48 .dtps_resume = NULL, 49 .dtps_getargdesc = kinst_getargdesc, 50 .dtps_getargval = NULL, 51 .dtps_usermode = NULL, 52 .dtps_destroy = kinst_destroy 53 }; 54 55 static struct cdevsw kinst_cdevsw = { 56 .d_name = "kinst", 57 .d_version = D_VERSION, 58 .d_flags = D_TRACKCLOSE, 59 .d_open = kinst_open, 60 .d_close = kinst_close, 61 .d_ioctl = kinst_ioctl, 62 }; 63 64 static dtrace_provider_id_t kinst_id; 65 struct kinst_probe_list *kinst_probetab; 66 static struct cdev *kinst_cdev; 67 68 /* 69 * Tracing memcpy() will crash the kernel when kinst tries to trace an instance 70 * of the memcpy() calls in kinst_invop(). To fix this, we can use 71 * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to 72 * "outsmart" the compiler and avoid having it replaced by a regular memcpy(). 73 */ 74 volatile void * 75 kinst_memcpy(volatile void *dst, volatile const void *src, size_t len) 76 { 77 volatile const unsigned char *src0; 78 volatile unsigned char *dst0; 79 80 src0 = src; 81 dst0 = dst; 82 83 while (len--) 84 *dst0++ = *src0++; 85 86 return (dst); 87 } 88 89 bool 90 kinst_excluded(const char *name) 91 { 92 if (kinst_md_excluded(name)) 93 return (true); 94 95 /* 96 * Anything beginning with "dtrace_" may be called from probe context 97 * unless it explicitly indicates that it won't be called from probe 98 * context by using the prefix "dtrace_safe_". 99 */ 100 if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 && 101 strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0) 102 return (true); 103 104 /* 105 * Omit instrumentation of functions that are probably in DDB. It 106 * makes it too hard to debug broken kinst. 107 * 108 * NB: kdb_enter() can be excluded, but its call to printf() can't be. 109 * This is generally OK since we're not yet in debugging context. 110 */ 111 if (strncmp(name, "db_", strlen("db_")) == 0 || 112 strncmp(name, "kdb_", strlen("kdb_")) == 0) 113 return (true); 114 115 /* 116 * Lock owner methods may be called from probe context. 117 */ 118 if (strcmp(name, "owner_mtx") == 0 || 119 strcmp(name, "owner_rm") == 0 || 120 strcmp(name, "owner_rw") == 0 || 121 strcmp(name, "owner_sx") == 0) 122 return (true); 123 124 /* 125 * When DTrace is built into the kernel we need to exclude the kinst 126 * functions from instrumentation. 127 */ 128 #ifndef _KLD_MODULE 129 if (strncmp(name, "kinst_", strlen("kinst_")) == 0) 130 return (true); 131 #endif 132 133 if (strcmp(name, "trap_check") == 0) 134 return (true); 135 136 return (false); 137 } 138 139 void 140 kinst_probe_create(struct kinst_probe *kp, linker_file_t lf) 141 { 142 kp->kp_id = dtrace_probe_create(kinst_id, lf->filename, 143 kp->kp_func, kp->kp_name, 3, kp); 144 145 LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext); 146 } 147 148 static int 149 kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, 150 struct thread *td __unused) 151 { 152 return (0); 153 } 154 155 static int 156 kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused, 157 struct thread *td __unused) 158 { 159 dtrace_condense(kinst_id); 160 return (0); 161 } 162 163 static int 164 kinst_linker_file_cb(linker_file_t lf, void *arg) 165 { 166 dtrace_kinst_probedesc_t *pd; 167 168 pd = arg; 169 if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0) 170 return (0); 171 172 /* 173 * Invoke kinst_make_probe_function() once for each function symbol in 174 * the module "lf". 175 */ 176 return (linker_file_function_listall(lf, kinst_make_probe, arg)); 177 } 178 179 static int 180 kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr, 181 int flags __unused, struct thread *td __unused) 182 { 183 dtrace_kinst_probedesc_t *pd; 184 int error = 0; 185 186 switch (cmd) { 187 case KINSTIOC_MAKEPROBE: 188 pd = (dtrace_kinst_probedesc_t *)addr; 189 pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0'; 190 pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0'; 191 192 /* Loop over all functions in the kernel and loaded modules. */ 193 error = linker_file_foreach(kinst_linker_file_cb, pd); 194 break; 195 default: 196 error = ENOTTY; 197 break; 198 } 199 200 return (error); 201 } 202 203 static void 204 kinst_provide_module(void *arg, modctl_t *lf) 205 { 206 } 207 208 static void 209 kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 210 { 211 desc->dtargd_ndx = DTRACE_ARGNONE; 212 } 213 214 static void 215 kinst_destroy(void *arg, dtrace_id_t id, void *parg) 216 { 217 struct kinst_probe *kp = parg; 218 219 LIST_REMOVE(kp, kp_hashnext); 220 free(kp, M_KINST); 221 } 222 223 static void 224 kinst_enable(void *arg, dtrace_id_t id, void *parg) 225 { 226 struct kinst_probe *kp = parg; 227 static bool warned = false; 228 229 if (!warned) { 230 KINST_LOG( 231 "kinst: This provider is experimental, exercise caution"); 232 warned = true; 233 } 234 235 kinst_patch_tracepoint(kp, kp->kp_patchval); 236 } 237 238 static void 239 kinst_disable(void *arg, dtrace_id_t id, void *parg) 240 { 241 struct kinst_probe *kp = parg; 242 243 kinst_patch_tracepoint(kp, kp->kp_savedval); 244 } 245 246 static int 247 kinst_load(void *dummy) 248 { 249 int error; 250 251 error = kinst_trampoline_init(); 252 if (error != 0) 253 return (error); 254 error = kinst_md_init(); 255 if (error != 0) { 256 kinst_trampoline_deinit(); 257 return (error); 258 } 259 260 error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL, 261 &kinst_pops, NULL, &kinst_id); 262 if (error != 0) { 263 kinst_md_deinit(); 264 kinst_trampoline_deinit(); 265 return (error); 266 } 267 kinst_probetab = malloc(KINST_PROBETAB_MAX * 268 sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO); 269 for (int i = 0; i < KINST_PROBETAB_MAX; i++) 270 LIST_INIT(&kinst_probetab[i]); 271 kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 272 "dtrace/kinst"); 273 dtrace_invop_add(kinst_invop); 274 return (0); 275 } 276 277 static int 278 kinst_unload(void *dummy) 279 { 280 free(kinst_probetab, M_KINST); 281 kinst_md_deinit(); 282 kinst_trampoline_deinit(); 283 dtrace_invop_remove(kinst_invop); 284 destroy_dev(kinst_cdev); 285 286 return (dtrace_unregister(kinst_id)); 287 } 288 289 static int 290 kinst_modevent(module_t mod __unused, int type, void *data __unused) 291 { 292 int error = 0; 293 294 switch (type) { 295 case MOD_LOAD: 296 break; 297 case MOD_UNLOAD: 298 break; 299 case MOD_SHUTDOWN: 300 break; 301 default: 302 error = EOPNOTSUPP; 303 break; 304 } 305 306 return (error); 307 } 308 309 SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL); 310 SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload, 311 NULL); 312 313 DEV_MODULE(kinst, kinst_modevent, NULL); 314 MODULE_VERSION(kinst, 1); 315 MODULE_DEPEND(kinst, dtrace, 1, 1, 1); 316 MODULE_DEPEND(kinst, opensolaris, 1, 1, 1); 317