xref: /freebsd/sys/arm64/arm64/trap.c (revision c697fb7f) 
1 /*-
2  * Copyright (c) 2014 Andrew Turner
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/pioctl.h>
38 #include <sys/proc.h>
39 #include <sys/ptrace.h>
40 #include <sys/syscall.h>
41 #include <sys/sysent.h>
42 #ifdef KDB
43 #include <sys/kdb.h>
44 #endif
45 
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_extern.h>
52 
53 #include <machine/frame.h>
54 #include <machine/pcb.h>
55 #include <machine/pcpu.h>
56 #include <machine/undefined.h>
57 
58 #ifdef KDTRACE_HOOKS
59 #include <sys/dtrace_bsd.h>
60 #endif
61 
62 #ifdef VFP
63 #include <machine/vfp.h>
64 #endif
65 
66 #ifdef KDB
67 #include <machine/db_machdep.h>
68 #endif
69 
70 #ifdef DDB
71 #include <ddb/db_output.h>
72 #endif
73 
74 extern register_t fsu_intr_fault;
75 
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82 
83 static void print_registers(struct trapframe *frame);
84 
85 int (*dtrace_invop_jump_addr)(struct trapframe *);
86 
87 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
88     uint64_t, int);
89 
90 static abort_handler align_abort;
91 static abort_handler data_abort;
92 
93 static abort_handler *abort_handlers[] = {
94 	[ISS_DATA_DFSC_TF_L0] = data_abort,
95 	[ISS_DATA_DFSC_TF_L1] = data_abort,
96 	[ISS_DATA_DFSC_TF_L2] = data_abort,
97 	[ISS_DATA_DFSC_TF_L3] = data_abort,
98 	[ISS_DATA_DFSC_AFF_L1] = data_abort,
99 	[ISS_DATA_DFSC_AFF_L2] = data_abort,
100 	[ISS_DATA_DFSC_AFF_L3] = data_abort,
101 	[ISS_DATA_DFSC_PF_L1] = data_abort,
102 	[ISS_DATA_DFSC_PF_L2] = data_abort,
103 	[ISS_DATA_DFSC_PF_L3] = data_abort,
104 	[ISS_DATA_DFSC_ALIGN] = align_abort,
105 };
106 
107 static __inline void
108 call_trapsignal(struct thread *td, int sig, int code, void *addr)
109 {
110 	ksiginfo_t ksi;
111 
112 	ksiginfo_init_trap(&ksi);
113 	ksi.ksi_signo = sig;
114 	ksi.ksi_code = code;
115 	ksi.ksi_addr = addr;
116 	trapsignal(td, &ksi);
117 }
118 
119 int
120 cpu_fetch_syscall_args(struct thread *td)
121 {
122 	struct proc *p;
123 	register_t *ap;
124 	struct syscall_args *sa;
125 	int nap;
126 
127 	nap = 8;
128 	p = td->td_proc;
129 	ap = td->td_frame->tf_x;
130 	sa = &td->td_sa;
131 
132 	sa->code = td->td_frame->tf_x[8];
133 
134 	if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
135 		sa->code = *ap++;
136 		nap--;
137 	}
138 
139 	if (sa->code >= p->p_sysent->sv_size)
140 		sa->callp = &p->p_sysent->sv_table[0];
141 	else
142 		sa->callp = &p->p_sysent->sv_table[sa->code];
143 
144 	sa->narg = sa->callp->sy_narg;
145 	memcpy(sa->args, ap, nap * sizeof(register_t));
146 	if (sa->narg > nap)
147 		panic("ARM64TODO: Could we have more than 8 args?");
148 
149 	td->td_retval[0] = 0;
150 	td->td_retval[1] = 0;
151 
152 	return (0);
153 }
154 
155 #include "../../kern/subr_syscall.c"
156 
157 static void
158 svc_handler(struct thread *td, struct trapframe *frame)
159 {
160 
161 	if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
162 		syscallenter(td);
163 		syscallret(td);
164 	} else {
165 		call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr);
166 		userret(td, frame);
167 	}
168 }
169 
170 static void
171 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
172     uint64_t far, int lower)
173 {
174 	if (!lower)
175 		panic("Misaligned access from kernel space!");
176 
177 	call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr);
178 	userret(td, frame);
179 }
180 
181 static void
182 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
183     uint64_t far, int lower)
184 {
185 	struct vm_map *map;
186 	struct proc *p;
187 	struct pcb *pcb;
188 	vm_prot_t ftype;
189 	int error, sig, ucode;
190 #ifdef KDB
191 	bool handled;
192 #endif
193 
194 	/*
195 	 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
196 	 * and Store-Exclusive instruction usage restrictions", state
197 	 * of the exclusive monitors after data abort exception is unknown.
198 	 */
199 	clrex();
200 
201 #ifdef KDB
202 	if (kdb_active) {
203 		kdb_reenter();
204 		return;
205 	}
206 #endif
207 
208 	pcb = td->td_pcb;
209 	p = td->td_proc;
210 	if (lower)
211 		map = &p->p_vmspace->vm_map;
212 	else {
213 		intr_enable();
214 
215 		/* The top bit tells us which range to use */
216 		if (far >= VM_MAXUSER_ADDRESS) {
217 			map = kernel_map;
218 		} else {
219 			map = &p->p_vmspace->vm_map;
220 			if (map == NULL)
221 				map = kernel_map;
222 		}
223 	}
224 
225 	/*
226 	 * Try to handle translation, access flag, and permission faults.
227 	 * Translation faults may occur as a result of the required
228 	 * break-before-make sequence used when promoting or demoting
229 	 * superpages.  Such faults must not occur while holding the pmap lock,
230 	 * or pmap_fault() will recurse on that lock.
231 	 */
232 	if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
233 	    pmap_fault(map->pmap, esr, far) == KERN_SUCCESS)
234 		return;
235 
236 	KASSERT(td->td_md.md_spinlock_count == 0,
237 	    ("data abort with spinlock held"));
238 	if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
239 	    WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
240 		print_registers(frame);
241 		printf(" far: %16lx\n", far);
242 		printf(" esr:         %.8lx\n", esr);
243 		panic("data abort in critical section or under mutex");
244 	}
245 
246 	switch (ESR_ELx_EXCEPTION(esr)) {
247 	case EXCP_INSN_ABORT:
248 	case EXCP_INSN_ABORT_L:
249 		ftype = VM_PROT_EXECUTE;
250 		break;
251 	default:
252 		ftype = (esr & ISS_DATA_WnR) == 0 ? VM_PROT_READ :
253 		    VM_PROT_READ | VM_PROT_WRITE;
254 		break;
255 	}
256 
257 	/* Fault in the page. */
258 	error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode);
259 	if (error != KERN_SUCCESS) {
260 		if (lower) {
261 			call_trapsignal(td, sig, ucode, (void *)far);
262 		} else {
263 			if (td->td_intr_nesting_level == 0 &&
264 			    pcb->pcb_onfault != 0) {
265 				frame->tf_x[0] = error;
266 				frame->tf_elr = pcb->pcb_onfault;
267 				return;
268 			}
269 
270 			printf("Fatal data abort:\n");
271 			print_registers(frame);
272 			printf(" far: %16lx\n", far);
273 			printf(" esr:         %.8lx\n", esr);
274 
275 #ifdef KDB
276 			if (debugger_on_trap) {
277 				kdb_why = KDB_WHY_TRAP;
278 				handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
279 				    frame);
280 				kdb_why = KDB_WHY_UNSET;
281 				if (handled)
282 					return;
283 			}
284 #endif
285 			panic("vm_fault failed: %lx", frame->tf_elr);
286 		}
287 	}
288 
289 	if (lower)
290 		userret(td, frame);
291 }
292 
293 static void
294 print_registers(struct trapframe *frame)
295 {
296 	u_int reg;
297 
298 	for (reg = 0; reg < nitems(frame->tf_x); reg++) {
299 		printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg,
300 		    frame->tf_x[reg]);
301 	}
302 	printf("  sp: %16lx\n", frame->tf_sp);
303 	printf("  lr: %16lx\n", frame->tf_lr);
304 	printf(" elr: %16lx\n", frame->tf_elr);
305 	printf("spsr:         %8x\n", frame->tf_spsr);
306 }
307 
308 void
309 do_el1h_sync(struct thread *td, struct trapframe *frame)
310 {
311 	struct trapframe *oframe;
312 	uint32_t exception;
313 	uint64_t esr, far;
314 	int dfsc;
315 
316 	/* Read the esr register to get the exception details */
317 	esr = frame->tf_esr;
318 	exception = ESR_ELx_EXCEPTION(esr);
319 
320 #ifdef KDTRACE_HOOKS
321 	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
322 		return;
323 #endif
324 
325 	CTR4(KTR_TRAP,
326 	    "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td,
327 	    esr, frame->tf_elr, frame);
328 
329 	oframe = td->td_frame;
330 
331 	switch (exception) {
332 	case EXCP_BRK:
333 	case EXCP_WATCHPT_EL1:
334 	case EXCP_SOFTSTP_EL1:
335 		break;
336 	default:
337 		td->td_frame = frame;
338 		break;
339 	}
340 
341 	switch(exception) {
342 	case EXCP_FP_SIMD:
343 	case EXCP_TRAP_FP:
344 #ifdef VFP
345 		if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
346 			vfp_restore_state();
347 		} else
348 #endif
349 		{
350 			print_registers(frame);
351 			printf(" esr:         %.8lx\n", esr);
352 			panic("VFP exception in the kernel");
353 		}
354 		break;
355 	case EXCP_INSN_ABORT:
356 	case EXCP_DATA_ABORT:
357 		far = READ_SPECIALREG(far_el1);
358 		dfsc = esr & ISS_DATA_DFSC_MASK;
359 		if (dfsc < nitems(abort_handlers) &&
360 		    abort_handlers[dfsc] != NULL) {
361 			abort_handlers[dfsc](td, frame, esr, far, 0);
362 		} else {
363 			print_registers(frame);
364 			printf(" far: %16lx\n", far);
365 			panic("Unhandled EL1 %s abort: %x",
366 			    exception == EXCP_INSN_ABORT ? "instruction" :
367 			    "data", dfsc);
368 		}
369 		break;
370 	case EXCP_BRK:
371 #ifdef KDTRACE_HOOKS
372 		if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \
373 		    dtrace_invop_jump_addr != 0) {
374 			dtrace_invop_jump_addr(frame);
375 			break;
376 		}
377 #endif
378 #ifdef KDB
379 		kdb_trap(exception, 0,
380 		    (td->td_frame != NULL) ? td->td_frame : frame);
381 #else
382 		panic("No debugger in kernel.\n");
383 #endif
384 		frame->tf_elr += 4;
385 		break;
386 	case EXCP_WATCHPT_EL1:
387 	case EXCP_SOFTSTP_EL1:
388 #ifdef KDB
389 		kdb_trap(exception, 0,
390 		    (td->td_frame != NULL) ? td->td_frame : frame);
391 #else
392 		panic("No debugger in kernel.\n");
393 #endif
394 		break;
395 	case EXCP_UNKNOWN:
396 		if (undef_insn(1, frame))
397 			break;
398 		/* FALLTHROUGH */
399 	default:
400 		print_registers(frame);
401 		panic("Unknown kernel exception %x esr_el1 %lx\n", exception,
402 		    esr);
403 	}
404 
405 	td->td_frame = oframe;
406 }
407 
408 void
409 do_el0_sync(struct thread *td, struct trapframe *frame)
410 {
411 	pcpu_bp_harden bp_harden;
412 	uint32_t exception;
413 	uint64_t esr, far;
414 	int dfsc;
415 
416 	/* Check we have a sane environment when entering from userland */
417 	KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
418 	    ("Invalid pcpu address from userland: %p (tpidr %lx)",
419 	     get_pcpu(), READ_SPECIALREG(tpidr_el1)));
420 
421 	esr = frame->tf_esr;
422 	exception = ESR_ELx_EXCEPTION(esr);
423 	switch (exception) {
424 	case EXCP_INSN_ABORT_L:
425 		far = READ_SPECIALREG(far_el1);
426 
427 		/*
428 		 * Userspace may be trying to train the branch predictor to
429 		 * attack the kernel. If we are on a CPU affected by this
430 		 * call the handler to clear the branch predictor state.
431 		 */
432 		if (far > VM_MAXUSER_ADDRESS) {
433 			bp_harden = PCPU_GET(bp_harden);
434 			if (bp_harden != NULL)
435 				bp_harden();
436 		}
437 		break;
438 	case EXCP_UNKNOWN:
439 	case EXCP_DATA_ABORT_L:
440 	case EXCP_DATA_ABORT:
441 		far = READ_SPECIALREG(far_el1);
442 		break;
443 	}
444 	intr_enable();
445 
446 	CTR4(KTR_TRAP,
447 	    "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr,
448 	    frame->tf_elr, frame);
449 
450 	switch(exception) {
451 	case EXCP_FP_SIMD:
452 	case EXCP_TRAP_FP:
453 #ifdef VFP
454 		vfp_restore_state();
455 #else
456 		panic("VFP exception in userland");
457 #endif
458 		break;
459 	case EXCP_SVC32:
460 	case EXCP_SVC64:
461 		svc_handler(td, frame);
462 		break;
463 	case EXCP_INSN_ABORT_L:
464 	case EXCP_DATA_ABORT_L:
465 	case EXCP_DATA_ABORT:
466 		dfsc = esr & ISS_DATA_DFSC_MASK;
467 		if (dfsc < nitems(abort_handlers) &&
468 		    abort_handlers[dfsc] != NULL)
469 			abort_handlers[dfsc](td, frame, esr, far, 1);
470 		else
471 			panic("Unhandled EL0 %s abort: %x",
472 			    exception == EXCP_INSN_ABORT_L ? "instruction" :
473 			    "data", dfsc);
474 		break;
475 	case EXCP_UNKNOWN:
476 		if (!undef_insn(0, frame))
477 			call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far);
478 		userret(td, frame);
479 		break;
480 	case EXCP_SP_ALIGN:
481 		call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp);
482 		userret(td, frame);
483 		break;
484 	case EXCP_PC_ALIGN:
485 		call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr);
486 		userret(td, frame);
487 		break;
488 	case EXCP_BRKPT_EL0:
489 	case EXCP_BRK:
490 		call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr);
491 		userret(td, frame);
492 		break;
493 	case EXCP_MSR:
494 		call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr);
495 		userret(td, frame);
496 		break;
497 	case EXCP_SOFTSTP_EL0:
498 		td->td_frame->tf_spsr &= ~PSR_SS;
499 		td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
500 		WRITE_SPECIALREG(mdscr_el1,
501 		    READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_SS);
502 		call_trapsignal(td, SIGTRAP, TRAP_TRACE,
503 		    (void *)frame->tf_elr);
504 		userret(td, frame);
505 		break;
506 	default:
507 		call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr);
508 		userret(td, frame);
509 		break;
510 	}
511 
512 	KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
513 	    ("Kernel VFP flags set while entering userspace"));
514 	KASSERT(
515 	    td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
516 	    ("Kernel VFP state in use when entering userspace"));
517 }
518 
519 /*
520  * TODO: We will need to handle these later when we support ARMv8.2 RAS.
521  */
522 void
523 do_serror(struct trapframe *frame)
524 {
525 	uint64_t esr, far;
526 
527 	far = READ_SPECIALREG(far_el1);
528 	esr = frame->tf_esr;
529 
530 	print_registers(frame);
531 	printf(" far: %16lx\n", far);
532 	printf(" esr:         %.8lx\n", esr);
533 	panic("Unhandled System Error");
534 }
535 
536 void
537 unhandled_exception(struct trapframe *frame)
538 {
539 	uint64_t esr, far;
540 
541 	far = READ_SPECIALREG(far_el1);
542 	esr = frame->tf_esr;
543 
544 	print_registers(frame);
545 	printf(" far: %16lx\n", far);
546 	printf(" esr:         %.8lx\n", esr);
547 	panic("Unhandled exception");
548 }
549