1 /* $NetBSD: dtrace_subr.c,v 1.15 2021/04/06 12:48:59 simonb Exp $ */
2
3 /*
4 * CDDL HEADER START
5 *
6 * The contents of this file are subject to the terms of the
7 * Common Development and Distribution License, Version 1.0 only
8 * (the "License"). You may not use this file except in compliance
9 * with the License.
10 *
11 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12 * or http://www.opensolaris.org/os/licensing.
13 * See the License for the specific language governing permissions
14 * and limitations under the License.
15 *
16 * When distributing Covered Code, include this CDDL HEADER in each
17 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18 * If applicable, add the following below this CDDL HEADER, with the
19 * fields enclosed by brackets "[]" replaced with your own identifying
20 * information: Portions Copyright [yyyy] [name of copyright owner]
21 *
22 * CDDL HEADER END
23 *
24 * $FreeBSD: head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c 313850 2017-02-17 03:27:20Z markj $
25 *
26 */
27 /*
28 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
29 * Use is subject to license terms.
30 */
31
32 /*
33 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
34 */
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/types.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/kmem.h>
42 #include <sys/xcall.h>
43 #include <sys/cpu.h>
44 #include <sys/cpuvar.h>
45 #include <sys/dtrace_impl.h>
46 #include <sys/dtrace_bsd.h>
47 #include <machine/frame.h>
48 #include <machine/cpu_counter.h>
49 #include <machine/cpufunc.h>
50
51 int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t);
52
53 typedef struct dtrace_invop_hdlr {
54 int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t);
55 struct dtrace_invop_hdlr *dtih_next;
56 } dtrace_invop_hdlr_t;
57
58 dtrace_invop_hdlr_t *dtrace_invop_hdlr;
59
60 void dtrace_gethrtime_init(void *);
61 void dtrace_getnanotime(struct timespec *);
62
63 int
dtrace_invop(uintptr_t addr,struct trapframe * frame,uintptr_t eax)64 dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax)
65 {
66 dtrace_invop_hdlr_t *hdlr;
67 int rval;
68
69 for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
70 if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0)
71 return (rval);
72
73 return (0);
74 }
75
76 void
dtrace_invop_add(int (* func)(uintptr_t,struct trapframe *,uintptr_t))77 dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
78 {
79 dtrace_invop_hdlr_t *hdlr;
80
81 hdlr = kmem_alloc(sizeof(*hdlr), KM_SLEEP);
82 hdlr->dtih_func = func;
83 hdlr->dtih_next = dtrace_invop_hdlr;
84 dtrace_invop_hdlr = hdlr;
85 }
86
87 void
dtrace_invop_remove(int (* func)(uintptr_t,struct trapframe *,uintptr_t))88 dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
89 {
90 dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL;
91
92 for (;;) {
93 if (hdlr == NULL)
94 panic("attempt to remove non-existent invop handler");
95
96 if (hdlr->dtih_func == func)
97 break;
98
99 prev = hdlr;
100 hdlr = hdlr->dtih_next;
101 }
102
103 if (prev == NULL) {
104 ASSERT(dtrace_invop_hdlr == hdlr);
105 dtrace_invop_hdlr = hdlr->dtih_next;
106 } else {
107 ASSERT(dtrace_invop_hdlr != hdlr);
108 prev->dtih_next = hdlr->dtih_next;
109 }
110
111 kmem_free(hdlr, sizeof(*hdlr));
112 }
113
114 /*ARGSUSED*/
115 void
dtrace_toxic_ranges(void (* func)(uintptr_t base,uintptr_t limit))116 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
117 {
118 (*func)(0, VM_MIN_KERNEL_ADDRESS_DEFAULT);
119 }
120
121 static void
xcall_func(void * arg0,void * arg1)122 xcall_func(void *arg0, void *arg1)
123 {
124 dtrace_xcall_t func = arg0;
125
126 (*func)(arg1);
127 }
128
129 void
dtrace_xcall(processorid_t cpu,dtrace_xcall_t func,void * arg)130 dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
131 {
132 uint64_t where;
133
134 if (cpu == DTRACE_CPUALL) {
135 where = xc_broadcast(0, xcall_func, func, arg);
136 } else {
137 struct cpu_info *cinfo = cpu_lookup(cpu);
138
139 KASSERT(cinfo != NULL);
140 where = xc_unicast(0, xcall_func, func, arg, cinfo);
141 }
142 xc_wait(where);
143
144 /* XXX Q. Do we really need the other cpus to wait also?
145 * (see solaris:xc_sync())
146 */
147 }
148
149 static void
dtrace_sync_func(void)150 dtrace_sync_func(void)
151 {
152 }
153
154 void
dtrace_sync(void)155 dtrace_sync(void)
156 {
157 dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
158 }
159
160 #ifdef notyet
161 void
dtrace_safe_synchronous_signal(void)162 dtrace_safe_synchronous_signal(void)
163 {
164 kthread_t *t = curthread;
165 struct regs *rp = lwptoregs(ttolwp(t));
166 size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
167
168 ASSERT(t->t_dtrace_on);
169
170 /*
171 * If we're not in the range of scratch addresses, we're not actually
172 * tracing user instructions so turn off the flags. If the instruction
173 * we copied out caused a synchonous trap, reset the pc back to its
174 * original value and turn off the flags.
175 */
176 if (rp->r_pc < t->t_dtrace_scrpc ||
177 rp->r_pc > t->t_dtrace_astpc + isz) {
178 t->t_dtrace_ft = 0;
179 } else if (rp->r_pc == t->t_dtrace_scrpc ||
180 rp->r_pc == t->t_dtrace_astpc) {
181 rp->r_pc = t->t_dtrace_pc;
182 t->t_dtrace_ft = 0;
183 }
184 }
185
186 int
dtrace_safe_defer_signal(void)187 dtrace_safe_defer_signal(void)
188 {
189 kthread_t *t = curthread;
190 struct regs *rp = lwptoregs(ttolwp(t));
191 size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
192
193 ASSERT(t->t_dtrace_on);
194
195 /*
196 * If we're not in the range of scratch addresses, we're not actually
197 * tracing user instructions so turn off the flags.
198 */
199 if (rp->r_pc < t->t_dtrace_scrpc ||
200 rp->r_pc > t->t_dtrace_astpc + isz) {
201 t->t_dtrace_ft = 0;
202 return (0);
203 }
204
205 /*
206 * If we have executed the original instruction, but we have performed
207 * neither the jmp back to t->t_dtrace_npc nor the clean up of any
208 * registers used to emulate %rip-relative instructions in 64-bit mode,
209 * we'll save ourselves some effort by doing that here and taking the
210 * signal right away. We detect this condition by seeing if the program
211 * counter is the range [scrpc + isz, astpc).
212 */
213 if (rp->r_pc >= t->t_dtrace_scrpc + isz &&
214 rp->r_pc < t->t_dtrace_astpc) {
215 #ifdef __amd64
216 /*
217 * If there is a scratch register and we're on the
218 * instruction immediately after the modified instruction,
219 * restore the value of that scratch register.
220 */
221 if (t->t_dtrace_reg != 0 &&
222 rp->r_pc == t->t_dtrace_scrpc + isz) {
223 switch (t->t_dtrace_reg) {
224 case REG_RAX:
225 rp->r_rax = t->t_dtrace_regv;
226 break;
227 case REG_RCX:
228 rp->r_rcx = t->t_dtrace_regv;
229 break;
230 case REG_R8:
231 rp->r_r8 = t->t_dtrace_regv;
232 break;
233 case REG_R9:
234 rp->r_r9 = t->t_dtrace_regv;
235 break;
236 }
237 }
238 #endif
239 rp->r_pc = t->t_dtrace_npc;
240 t->t_dtrace_ft = 0;
241 return (0);
242 }
243
244 /*
245 * Otherwise, make sure we'll return to the kernel after executing
246 * the copied out instruction and defer the signal.
247 */
248 if (!t->t_dtrace_step) {
249 ASSERT(rp->r_pc < t->t_dtrace_astpc);
250 rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc;
251 t->t_dtrace_step = 1;
252 }
253
254 t->t_dtrace_ast = 1;
255
256 return (1);
257 }
258 #endif
259
260 #ifdef __NetBSD__
261 static __inline uint64_t
dtrace_rdtsc(void)262 dtrace_rdtsc(void)
263 {
264 uint32_t hi, lo;
265
266 __asm volatile("rdtsc" : "=d" (hi), "=a" (lo));
267 return (((uint64_t)hi << 32) | (uint64_t) lo);
268 }
269 #define rdtsc dtrace_rdtsc
270 #endif
271
272 #ifdef notyet
273 static int64_t tgt_cpu_tsc;
274 static int64_t hst_cpu_tsc;
275 #endif
276 static int64_t tsc_skew[MAXCPUS];
277 static uint64_t nsec_scale;
278
279 /* See below for the explanation of this macro. */
280 #define SCALE_SHIFT 28
281
282 #ifdef notyet
283 static void
dtrace_gethrtime_init_sync(void * arg)284 dtrace_gethrtime_init_sync(void *arg)
285 {
286 #ifdef CHECK_SYNC
287 /*
288 * Delay this function from returning on one
289 * of the CPUs to check that the synchronisation
290 * works.
291 */
292 uintptr_t cpu = (uintptr_t) arg;
293
294 if (cpu == cpu_number()) {
295 int i;
296 for (i = 0; i < 1000000000; i++)
297 tgt_cpu_tsc = rdtsc();
298 tgt_cpu_tsc = 0;
299 }
300 #endif
301 }
302
303 static void
dtrace_gethrtime_init_cpu(void * arg)304 dtrace_gethrtime_init_cpu(void *arg)
305 {
306 uintptr_t cpu = (uintptr_t) arg;
307
308 if (cpu == cpu_number())
309 tgt_cpu_tsc = rdtsc();
310 else
311 hst_cpu_tsc = rdtsc();
312 }
313 #endif
314
315 void
dtrace_gethrtime_init(void * arg)316 dtrace_gethrtime_init(void *arg)
317 {
318 uint64_t tsc_f;
319 CPU_INFO_ITERATOR cpuind;
320 struct cpu_info *cinfo = curcpu();
321 cpuid_t cur_cpuid = cpu_number(); /* current cpu id */
322
323 /*
324 * Get TSC frequency known at this moment.
325 * This should be constant if TSC is invariant.
326 * Otherwise tick->time conversion will be inaccurate, but
327 * will preserve monotonic property of TSC.
328 */
329 tsc_f = cpu_frequency(cinfo);
330
331 /*
332 * The following line checks that nsec_scale calculated below
333 * doesn't overflow 32-bit unsigned integer, so that it can multiply
334 * another 32-bit integer without overflowing 64-bit.
335 * Thus minimum supported TSC frequency is 62.5MHz.
336 */
337 KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)));
338
339 /*
340 * We scale up NANOSEC/tsc_f ratio to preserve as much precision
341 * as possible.
342 * 2^28 factor was chosen quite arbitrarily from practical
343 * considerations:
344 * - it supports TSC frequencies as low as 62.5MHz (see above);
345 * - it provides quite good precision (e < 0.01%) up to THz
346 * (terahertz) values;
347 */
348 nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f;
349
350 /* The current CPU is the reference one. */
351 tsc_skew[cur_cpuid] = 0;
352
353 for (CPU_INFO_FOREACH(cpuind, cinfo)) {
354 /* use skew relative to cpu 0 */
355 tsc_skew[cpu_index(cinfo)] = cinfo->ci_data.cpu_cc_skew;
356 }
357 }
358
359 /*
360 * DTrace needs a high resolution time function which can
361 * be called from a probe context and guaranteed not to have
362 * instrumented with probes itself.
363 *
364 * Returns nanoseconds since boot.
365 */
366 uint64_t
dtrace_gethrtime()367 dtrace_gethrtime()
368 {
369 uint64_t tsc;
370 uint32_t lo;
371 uint32_t hi;
372
373 /*
374 * We split TSC value into lower and higher 32-bit halves and separately
375 * scale them with nsec_scale, then we scale them down by 2^28
376 * (see nsec_scale calculations) taking into account 32-bit shift of
377 * the higher half and finally add.
378 */
379 tsc = rdtsc() + tsc_skew[cpu_number()];
380 lo = tsc;
381 hi = tsc >> 32;
382 return (((lo * nsec_scale) >> SCALE_SHIFT) +
383 ((hi * nsec_scale) << (32 - SCALE_SHIFT)));
384 }
385
386 uint64_t
dtrace_gethrestime(void)387 dtrace_gethrestime(void)
388 {
389 struct timespec current_time;
390
391 dtrace_getnanotime(¤t_time);
392
393 return (current_time.tv_sec * 1000000000ULL + current_time.tv_nsec);
394 }
395
396 /* Function to handle DTrace traps during probes. See amd64/amd64/trap.c */
397 int
dtrace_trap(struct trapframe * frame,u_int type)398 dtrace_trap(struct trapframe *frame, u_int type)
399 {
400 bool nofault;
401 cpuid_t cpuid = cpu_number(); /* current cpu id */
402
403 /*
404 * A trap can occur while DTrace executes a probe. Before
405 * executing the probe, DTrace blocks re-scheduling and sets
406 * a flag in it's per-cpu flags to indicate that it doesn't
407 * want to fault. On returning from the the probe, the no-fault
408 * flag is cleared and finally re-scheduling is enabled.
409 *
410 * Check if DTrace has enabled 'no-fault' mode:
411 *
412 */
413 nofault = (cpu_core[cpuid].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0;
414 if (nofault) {
415 #if 0
416 This assertion would always fire, we get called from
417 alltraps() -> trap() with interrupts enabled.
418 KASSERTMSG((x86_read_flags() & PSL_I) == 0, "interrupts enabled");
419 #endif
420
421 /*
422 * There are only a couple of trap types that are expected.
423 * All the rest will be handled in the usual way.
424 */
425 switch (type) {
426 /* General protection fault. */
427 case T_PROTFLT:
428 /* Flag an illegal operation. */
429 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
430
431 /*
432 * Offset the instruction pointer to the instruction
433 * following the one causing the fault.
434 */
435 frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip);
436 return (1);
437 /* Page fault. */
438 case T_PAGEFLT:
439 /* Flag a bad address. */
440 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
441 cpu_core[cpuid].cpuc_dtrace_illval = rcr2();
442
443 /*
444 * Offset the instruction pointer to the instruction
445 * following the one causing the fault.
446 */
447 frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip);
448 return (1);
449 default:
450 /* Handle all other traps in the usual way. */
451 break;
452 }
453 }
454
455 /* Handle the trap in the usual way. */
456 return (0);
457 }
458