1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  *
22  * $FreeBSD$
23  *
24  */
25 /*
26  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
27  * Use is subject to license terms.
28  */
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/malloc.h>
34 #include <sys/kmem.h>
35 #include <sys/proc.h>
36 #include <sys/smp.h>
37 #include <sys/dtrace_impl.h>
38 #include <sys/dtrace_bsd.h>
39 #include <cddl/dev/dtrace/dtrace_cddl.h>
40 #include <machine/armreg.h>
41 #include <machine/clock.h>
42 #include <machine/frame.h>
43 #include <machine/trap.h>
44 #include <machine/vmparam.h>
45 #include <vm/pmap.h>
46 
47 extern dtrace_id_t	dtrace_probeid_error;
48 extern int (*dtrace_invop_jump_addr)(struct trapframe *);
49 extern void dtrace_getnanotime(struct timespec *tsp);
50 extern void dtrace_getnanouptime(struct timespec *tsp);
51 
52 int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t);
53 void dtrace_invop_init(void);
54 void dtrace_invop_uninit(void);
55 
56 typedef struct dtrace_invop_hdlr {
57 	int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t);
58 	struct dtrace_invop_hdlr *dtih_next;
59 } dtrace_invop_hdlr_t;
60 
61 dtrace_invop_hdlr_t *dtrace_invop_hdlr;
62 
63 int
64 dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax)
65 {
66 	struct thread *td;
67 	dtrace_invop_hdlr_t *hdlr;
68 	int rval;
69 
70 	rval = 0;
71 	td = curthread;
72 	td->t_dtrace_trapframe = frame;
73 	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
74 		if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0)
75 			break;
76 	td->t_dtrace_trapframe = NULL;
77 	return (rval);
78 }
79 
80 void
81 dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
82 {
83 	dtrace_invop_hdlr_t *hdlr;
84 
85 	hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
86 	hdlr->dtih_func = func;
87 	hdlr->dtih_next = dtrace_invop_hdlr;
88 	dtrace_invop_hdlr = hdlr;
89 }
90 
91 void
92 dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
93 {
94 	dtrace_invop_hdlr_t *hdlr, *prev;
95 
96 	hdlr = dtrace_invop_hdlr;
97 	prev = NULL;
98 
99 	for (;;) {
100 		if (hdlr == NULL)
101 			panic("attempt to remove non-existent invop handler");
102 
103 		if (hdlr->dtih_func == func)
104 			break;
105 
106 		prev = hdlr;
107 		hdlr = hdlr->dtih_next;
108 	}
109 
110 	if (prev == NULL) {
111 		ASSERT(dtrace_invop_hdlr == hdlr);
112 		dtrace_invop_hdlr = hdlr->dtih_next;
113 	} else {
114 		ASSERT(dtrace_invop_hdlr != hdlr);
115 		prev->dtih_next = hdlr->dtih_next;
116 	}
117 
118 	kmem_free(hdlr, 0);
119 }
120 
121 /*ARGSUSED*/
122 void
123 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
124 {
125 
126 	(*func)(0, (uintptr_t)VM_MIN_KERNEL_ADDRESS);
127 }
128 
129 void
130 dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
131 {
132 	cpuset_t cpus;
133 
134 	if (cpu == DTRACE_CPUALL)
135 		cpus = all_cpus;
136 	else
137 		CPU_SETOF(cpu, &cpus);
138 
139 	smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func,
140 	    smp_no_rendezvous_barrier, arg);
141 }
142 
143 static void
144 dtrace_sync_func(void)
145 {
146 
147 }
148 
149 void
150 dtrace_sync(void)
151 {
152 
153 	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
154 }
155 
156 /*
157  * DTrace needs a high resolution time function which can be called from a
158  * probe context and guaranteed not to have instrumented with probes itself.
159  *
160  * Returns nanoseconds since some arbitrary point in time (likely SoC reset?).
161  */
162 uint64_t
163 dtrace_gethrtime(void)
164 {
165 	uint64_t count, freq;
166 
167 	count = READ_SPECIALREG(cntvct_el0);
168 	freq = READ_SPECIALREG(cntfrq_el0);
169 	return ((1000000000UL * count) / freq);
170 }
171 
172 /*
173  * Return a much lower resolution wallclock time based on the system clock
174  * updated by the timer.  If needed, we could add a version interpolated from
175  * the system clock as is the case with dtrace_gethrtime().
176  */
177 uint64_t
178 dtrace_gethrestime(void)
179 {
180 	struct timespec current_time;
181 
182 	dtrace_getnanotime(&current_time);
183 
184 	return (current_time.tv_sec * 1000000000UL + current_time.tv_nsec);
185 }
186 
187 /* Function to handle DTrace traps during probes. See arm64/arm64/trap.c */
188 int
189 dtrace_trap(struct trapframe *frame, u_int type)
190 {
191 	/*
192 	 * A trap can occur while DTrace executes a probe. Before
193 	 * executing the probe, DTrace blocks re-scheduling and sets
194 	 * a flag in its per-cpu flags to indicate that it doesn't
195 	 * want to fault. On returning from the probe, the no-fault
196 	 * flag is cleared and finally re-scheduling is enabled.
197 	 *
198 	 * Check if DTrace has enabled 'no-fault' mode:
199 	 *
200 	 */
201 
202 	if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) {
203 		/*
204 		 * There are only a couple of trap types that are expected.
205 		 * All the rest will be handled in the usual way.
206 		 */
207 		switch (type) {
208 		case EXCP_DATA_ABORT:
209 			/* Flag a bad address. */
210 			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
211 			cpu_core[curcpu].cpuc_dtrace_illval = 0;
212 
213 			/*
214 			 * Offset the instruction pointer to the instruction
215 			 * following the one causing the fault.
216 			 */
217 			frame->tf_elr += 4;
218 			return (1);
219 		default:
220 			/* Handle all other traps in the usual way. */
221 			break;
222 		}
223 	}
224 
225 	/* Handle the trap in the usual way. */
226 	return (0);
227 }
228 
229 void
230 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
231     int fault, int fltoffs, uintptr_t illval)
232 {
233 
234 	dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state,
235 	    (uintptr_t)epid,
236 	    (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs);
237 }
238 
239 static void
240 dtrace_load64(uint64_t *addr, struct trapframe *frame, u_int reg)
241 {
242 
243 	KASSERT(reg <= 31, ("dtrace_load64: Invalid register %u", reg));
244 	if (reg < nitems(frame->tf_x))
245 		frame->tf_x[reg] = *addr;
246 	else if (reg == 30) /* lr */
247 		frame->tf_lr = *addr;
248 	/* Nothing to do for load to xzr */
249 }
250 
251 static void
252 dtrace_store64(uint64_t *addr, struct trapframe *frame, u_int reg)
253 {
254 
255 	KASSERT(reg <= 31, ("dtrace_store64: Invalid register %u", reg));
256 	if (reg < nitems(frame->tf_x))
257 		*addr = frame->tf_x[reg];
258 	else if (reg == 30) /* lr */
259 		*addr = frame->tf_lr;
260 	else if (reg == 31) /* xzr */
261 		*addr = 0;
262 }
263 
264 static int
265 dtrace_invop_start(struct trapframe *frame)
266 {
267 	int data, invop, reg, update_sp;
268 	register_t arg1, arg2;
269 	register_t *sp;
270 	int offs;
271 	int tmp;
272 	int i;
273 
274 	invop = dtrace_invop(frame->tf_elr, frame, frame->tf_x[0]);
275 
276 	tmp = (invop & LDP_STP_MASK);
277 	if (tmp == STP_64 || tmp == LDP_64) {
278 		sp = (register_t *)frame->tf_sp;
279 		data = invop;
280 		arg1 = (data >> ARG1_SHIFT) & ARG1_MASK;
281 		arg2 = (data >> ARG2_SHIFT) & ARG2_MASK;
282 
283 		offs = (data >> OFFSET_SHIFT) & OFFSET_MASK;
284 
285 		switch (tmp) {
286 		case STP_64:
287 			if (offs >> (OFFSET_SIZE - 1))
288 				sp -= (~offs & OFFSET_MASK) + 1;
289 			else
290 				sp += (offs);
291 			dtrace_store64(sp + 0, frame, arg1);
292 			dtrace_store64(sp + 1, frame, arg2);
293 			break;
294 		case LDP_64:
295 			dtrace_load64(sp + 0, frame, arg1);
296 			dtrace_load64(sp + 1, frame, arg2);
297 			if (offs >> (OFFSET_SIZE - 1))
298 				sp -= (~offs & OFFSET_MASK) + 1;
299 			else
300 				sp += (offs);
301 			break;
302 		default:
303 			break;
304 		}
305 
306 		/* Update the stack pointer and program counter to continue */
307 		frame->tf_sp = (register_t)sp;
308 		frame->tf_elr += INSN_SIZE;
309 		return (0);
310 	}
311 
312 	if ((invop & SUB_MASK) == SUB_INSTR) {
313 		frame->tf_sp -= (invop >> SUB_IMM_SHIFT) & SUB_IMM_MASK;
314 		frame->tf_elr += INSN_SIZE;
315 		return (0);
316 	}
317 
318 	if (invop == NOP_INSTR) {
319 		frame->tf_elr += INSN_SIZE;
320 		return (0);
321 	}
322 
323 	if ((invop & B_MASK) == B_INSTR) {
324 		data = (invop & B_DATA_MASK);
325 		/* The data is the number of 4-byte words to change the pc */
326 		data *= 4;
327 		frame->tf_elr += data;
328 		return (0);
329 	}
330 
331 	if (invop == RET_INSTR) {
332 		frame->tf_elr = frame->tf_lr;
333 		return (0);
334 	}
335 
336 	return (-1);
337 }
338 
339 void
340 dtrace_invop_init(void)
341 {
342 
343 	dtrace_invop_jump_addr = dtrace_invop_start;
344 }
345 
346 void
347 dtrace_invop_uninit(void)
348 {
349 
350 	dtrace_invop_jump_addr = 0;
351 }
352