xref: /freebsd/sys/arm/arm/trap-v6.c (revision d0b2dbfa)
1 /*-
2  * Copyright 2014 Olivier Houchard <cognet@FreeBSD.org>
3  * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com>
4  * Copyright 2014 Michal Meloun <meloun@miracle.cz>
5  * Copyright 2014 Andrew Turner <andrew@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include "opt_ktrace.h"
31 
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/systm.h>
36 #include <sys/proc.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/mutex.h>
40 #include <sys/signalvar.h>
41 #include <sys/ktr.h>
42 #include <sys/vmmeter.h>
43 #ifdef KTRACE
44 #include <sys/uio.h>
45 #include <sys/ktrace.h>
46 #endif
47 
48 #include <vm/vm.h>
49 #include <vm/pmap.h>
50 #include <vm/vm_kern.h>
51 #include <vm/vm_map.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_param.h>
54 
55 #include <machine/cpu.h>
56 #include <machine/frame.h>
57 #include <machine/machdep.h>
58 #include <machine/pcb.h>
59 
60 #ifdef KDB
61 #include <sys/kdb.h>
62 #include <machine/db_machdep.h>
63 #endif
64 
65 #ifdef KDTRACE_HOOKS
66 #include <sys/dtrace_bsd.h>
67 #endif
68 
69 extern char cachebailout[];
70 
71 struct ksig {
72 	int sig;
73 	u_long code;
74 	vm_offset_t	addr;
75 };
76 
77 typedef int abort_func_t(struct trapframe *, u_int, u_int, u_int, u_int,
78     struct thread *, struct ksig *);
79 
80 static abort_func_t abort_fatal;
81 static abort_func_t abort_align;
82 static abort_func_t abort_icache;
83 
84 struct abort {
85 	abort_func_t	*func;
86 	const char	*desc;
87 };
88 
89 /*
90  * How are the aborts handled?
91  *
92  * Undefined Code:
93  *  - Always fatal as we do not know what does it mean.
94  * Imprecise External Abort:
95  *  - Always fatal, but can be handled somehow in the future.
96  *    Now, due to PCIe buggy hardware, ignored.
97  * Precise External Abort:
98  *  - Always fatal, but who knows in the future???
99  * Debug Event:
100  *  - Special handling.
101  * External Translation Abort (L1 & L2)
102  *  - Always fatal as something is screwed up in page tables or hardware.
103  * Domain Fault (L1 & L2):
104  *  - Always fatal as we do not play game with domains.
105  * Alignment Fault:
106  *  - Everything should be aligned in kernel with exception of user to kernel
107  *    and vice versa data copying, so if pcb_onfault is not set, it's fatal.
108  *    We generate signal in case of abort from user mode.
109  * Instruction cache maintenance:
110  *  - According to manual, this is translation fault during cache maintenance
111  *    operation. So, it could be really complex in SMP case and fuzzy too
112  *    for cache operations working on virtual addresses. For now, we will
113  *    consider this abort as fatal. In fact, no cache maintenance on
114  *    not mapped virtual addresses should be called. As cache maintenance
115  *    operation (except DMB, DSB, and Flush Prefetch Buffer) are privileged,
116  *    the abort is fatal for user mode as well for now. (This is good place to
117  *    note that cache maintenance on virtual address fill TLB.)
118  * Acces Bit (L1 & L2):
119  *  - Fast hardware emulation for kernel and user mode.
120  * Translation Fault (L1 & L2):
121  *  - Standard fault mechanism is held including vm_fault().
122  * Permission Fault (L1 & L2):
123  *  - Fast hardware emulation of modify bits and in other cases, standard
124  *    fault mechanism is held including vm_fault().
125  */
126 
127 static const struct abort aborts[] = {
128 	{abort_fatal,	"Undefined Code (0x000)"},
129 	{abort_align,	"Alignment Fault"},
130 	{abort_fatal,	"Debug Event"},
131 	{NULL,		"Access Bit (L1)"},
132 	{NULL,		"Instruction cache maintenance"},
133 	{NULL,		"Translation Fault (L1)"},
134 	{NULL,		"Access Bit (L2)"},
135 	{NULL,		"Translation Fault (L2)"},
136 
137 	{abort_fatal,	"External Abort"},
138 	{abort_fatal,	"Domain Fault (L1)"},
139 	{abort_fatal,	"Undefined Code (0x00A)"},
140 	{abort_fatal,	"Domain Fault (L2)"},
141 	{abort_fatal,	"External Translation Abort (L1)"},
142 	{NULL,		"Permission Fault (L1)"},
143 	{abort_fatal,	"External Translation Abort (L2)"},
144 	{NULL,		"Permission Fault (L2)"},
145 
146 	{abort_fatal,	"TLB Conflict Abort"},
147 	{abort_fatal,	"Undefined Code (0x401)"},
148 	{abort_fatal,	"Undefined Code (0x402)"},
149 	{abort_fatal,	"Undefined Code (0x403)"},
150 	{abort_fatal,	"Undefined Code (0x404)"},
151 	{abort_fatal,	"Undefined Code (0x405)"},
152 	{abort_fatal,	"Asynchronous External Abort"},
153 	{abort_fatal,	"Undefined Code (0x407)"},
154 
155 	{abort_fatal,	"Asynchronous Parity Error on Memory Access"},
156 	{abort_fatal,	"Parity Error on Memory Access"},
157 	{abort_fatal,	"Undefined Code (0x40A)"},
158 	{abort_fatal,	"Undefined Code (0x40B)"},
159 	{abort_fatal,	"Parity Error on Translation (L1)"},
160 	{abort_fatal,	"Undefined Code (0x40D)"},
161 	{abort_fatal,	"Parity Error on Translation (L2)"},
162 	{abort_fatal,	"Undefined Code (0x40F)"}
163 };
164 
165 static __inline void
166 call_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr,
167     int trapno)
168 {
169 	ksiginfo_t ksi;
170 
171 	CTR4(KTR_TRAP, "%s: addr: %#x, sig: %d, code: %d",
172 	   __func__, addr, sig, code);
173 
174 	/*
175 	 * TODO: some info would be nice to know
176 	 * if we are serving data or prefetch abort.
177 	 */
178 
179 	ksiginfo_init_trap(&ksi);
180 	ksi.ksi_signo = sig;
181 	ksi.ksi_code = code;
182 	ksi.ksi_addr = (void *)addr;
183 	ksi.ksi_trapno = trapno;
184 	trapsignal(td, &ksi);
185 }
186 
187 /*
188  * abort_imprecise() handles the following abort:
189  *
190  *  FAULT_EA_IMPREC - Imprecise External Abort
191  *
192  * The imprecise means that we don't know where the abort happened,
193  * thus FAR is undefined. The abort should not never fire, but hot
194  * plugging or accidental hardware failure can be the cause of it.
195  * If the abort happens, it can even be on different (thread) context.
196  * Without any additional support, the abort is fatal, as we do not
197  * know what really happened.
198  *
199  * QQQ: Some additional functionality, like pcb_onfault but global,
200  *      can be implemented. Imprecise handlers could be registered
201  *      which tell us if the abort is caused by something they know
202  *      about. They should return one of three codes like:
203  *		FAULT_IS_MINE,
204  *		FAULT_CAN_BE_MINE,
205  *		FAULT_IS_NOT_MINE.
206  *      The handlers should be called until some of them returns
207  *      FAULT_IS_MINE value or all was called. If all handlers return
208  *	FAULT_IS_NOT_MINE value, then the abort is fatal.
209  */
210 static __inline void
211 abort_imprecise(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode)
212 {
213 
214 	/*
215 	 * XXX - We can got imprecise abort as result of access
216 	 * to not-present PCI/PCIe configuration space.
217 	 */
218 #if 0
219 	goto out;
220 #endif
221 	abort_fatal(tf, FAULT_EA_IMPREC, fsr, 0, prefetch, curthread, NULL);
222 
223 	/*
224 	 * Returning from this function means that we ignore
225 	 * the abort for good reason. Note that imprecise abort
226 	 * could fire any time even in user mode.
227 	 */
228 
229 #if 0
230 out:
231 	if (usermode)
232 		userret(curthread, tf);
233 #endif
234 }
235 
236 /*
237  * abort_debug() handles the following abort:
238  *
239  *  FAULT_DEBUG - Debug Event
240  *
241  */
242 static __inline void
243 abort_debug(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode,
244     u_int far)
245 {
246 
247 	if (usermode) {
248 		struct thread *td;
249 
250 		td = curthread;
251 		call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far, FAULT_DEBUG);
252 		userret(td, tf);
253 	} else {
254 #ifdef KDB
255 		kdb_trap((prefetch) ? T_BREAKPOINT : T_WATCHPOINT, 0, tf);
256 #else
257 		printf("No debugger in kernel.\n");
258 #endif
259 	}
260 }
261 
262 /*
263  * Abort handler.
264  *
265  * FAR, FSR, and everything what can be lost after enabling
266  * interrupts must be grabbed before the interrupts will be
267  * enabled. Note that when interrupts will be enabled, we
268  * could even migrate to another CPU ...
269  *
270  * TODO: move quick cases to ASM
271  */
272 void
273 abort_handler(struct trapframe *tf, int prefetch)
274 {
275 	struct thread *td;
276 	vm_offset_t far, va;
277 	int idx, rv;
278 	uint32_t fsr;
279 	struct ksig ksig;
280 	struct proc *p;
281 	struct pcb *pcb;
282 	struct vm_map *map;
283 	struct vmspace *vm;
284 	vm_prot_t ftype;
285 	bool usermode;
286 	int bp_harden, ucode;
287 #ifdef INVARIANTS
288 	void *onfault;
289 #endif
290 
291 	VM_CNT_INC(v_trap);
292 	td = curthread;
293 
294 	fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get();
295 #if __ARM_ARCH >= 7
296 	far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get();
297 #else
298 	far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get();
299 #endif
300 
301 	idx = FSR_TO_FAULT(fsr);
302 	usermode = TRAPF_USERMODE(tf);	/* Abort came from user mode? */
303 
304 	/*
305 	 * Apply BP hardening by flushing the branch prediction cache
306 	 * for prefaults on kernel addresses.
307 	 */
308 	if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS &&
309 	    (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) {
310 		bp_harden = PCPU_GET(bp_harden_kind);
311 		if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL)
312 			_CP15_BPIALL();
313 		else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU)
314 			_CP15_ICIALLU();
315 	}
316 
317 	if (usermode)
318 		td->td_frame = tf;
319 
320 	CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d",
321 	    __func__, fsr, idx, far, prefetch, usermode);
322 
323 	/*
324 	 * Firstly, handle aborts that are not directly related to mapping.
325 	 */
326 	if (__predict_false(idx == FAULT_EA_IMPREC)) {
327 		abort_imprecise(tf, fsr, prefetch, usermode);
328 		return;
329 	}
330 
331 	if (__predict_false(idx == FAULT_DEBUG)) {
332 		abort_debug(tf, fsr, prefetch, usermode, far);
333 		return;
334 	}
335 
336 	/*
337 	 * ARM has a set of unprivileged load and store instructions
338 	 * (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other
339 	 * than user mode and OS should recognize their aborts and behave
340 	 * appropriately. However, there is no way how to do that reasonably
341 	 * in general unless we restrict the handling somehow.
342 	 *
343 	 * For now, these instructions are used only in copyin()/copyout()
344 	 * like functions where usermode buffers are checked in advance that
345 	 * they are not from KVA space. Thus, no action is needed here.
346 	 */
347 
348 	/*
349 	 * (1) Handle access and R/W hardware emulation aborts.
350 	 * (2) Check that abort is not on pmap essential address ranges.
351 	 *     There is no way how to fix it, so we don't even try.
352 	 */
353 	rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode);
354 	if (rv == KERN_SUCCESS)
355 		return;
356 #ifdef KDB
357 	if (kdb_active) {
358 		kdb_reenter();
359 		goto out;
360 	}
361 #endif
362 	if (rv == KERN_INVALID_ADDRESS)
363 		goto nogo;
364 
365 	if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
366 		/*
367 		 * Due to both processor errata and lazy TLB invalidation when
368 		 * access restrictions are removed from virtual pages, memory
369 		 * accesses that are allowed by the physical mapping layer may
370 		 * nonetheless cause one spurious page fault per virtual page.
371 		 * When the thread is executing a "no faulting" section that
372 		 * is bracketed by vm_fault_{disable,enable}_pagefaults(),
373 		 * every page fault is treated as a spurious page fault,
374 		 * unless it accesses the same virtual address as the most
375 		 * recent page fault within the same "no faulting" section.
376 		 */
377 		if (td->td_md.md_spurflt_addr != far ||
378 		    (td->td_pflags & TDP_RESETSPUR) != 0) {
379 			td->td_md.md_spurflt_addr = far;
380 			td->td_pflags &= ~TDP_RESETSPUR;
381 
382 			tlb_flush_local(far & ~PAGE_MASK);
383 			return;
384 		}
385 	} else {
386 		/*
387 		 * If we get a page fault while in a critical section, then
388 		 * it is most likely a fatal kernel page fault.  The kernel
389 		 * is already going to panic trying to get a sleep lock to
390 		 * do the VM lookup, so just consider it a fatal trap so the
391 		 * kernel can print out a useful trap message and even get
392 		 * to the debugger.
393 		 *
394 		 * If we get a page fault while holding a non-sleepable
395 		 * lock, then it is most likely a fatal kernel page fault.
396 		 * If WITNESS is enabled, then it's going to whine about
397 		 * bogus LORs with various VM locks, so just skip to the
398 		 * fatal trap handling directly.
399 		 */
400 		if (td->td_critnest != 0 ||
401 		    WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
402 		    "Kernel page fault") != 0) {
403 			abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
404 			return;
405 		}
406 	}
407 
408 	/* Re-enable interrupts if they were enabled previously. */
409 	if (td->td_md.md_spinlock_count == 0) {
410 		if (__predict_true(tf->tf_spsr & PSR_I) == 0)
411 			enable_interrupts(PSR_I);
412 		if (__predict_true(tf->tf_spsr & PSR_F) == 0)
413 			enable_interrupts(PSR_F);
414 	}
415 
416 	p = td->td_proc;
417 	if (usermode) {
418 		td->td_pticks = 0;
419 		if (td->td_cowgen != atomic_load_int(&p->p_cowgen))
420 			thread_cow_update(td);
421 	}
422 
423 	/* Invoke the appropriate handler, if necessary. */
424 	if (__predict_false(aborts[idx].func != NULL)) {
425 		if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig))
426 			goto do_trapsignal;
427 		goto out;
428 	}
429 
430 	/*
431 	 * At this point, we're dealing with one of the following aborts:
432 	 *
433 	 *  FAULT_ICACHE   - I-cache maintenance
434 	 *  FAULT_TRAN_xx  - Translation
435 	 *  FAULT_PERM_xx  - Permission
436 	 */
437 
438 	/*
439 	 * Don't pass faulting cache operation to vm_fault(). We don't want
440 	 * to handle all vm stuff at this moment.
441 	 */
442 	pcb = td->td_pcb;
443 	if (__predict_false(pcb->pcb_onfault == cachebailout)) {
444 		tf->tf_r0 = far;		/* return failing address */
445 		tf->tf_pc = (register_t)pcb->pcb_onfault;
446 		return;
447 	}
448 
449 	/* Handle remaining I-cache aborts. */
450 	if (idx == FAULT_ICACHE) {
451 		if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig))
452 			goto do_trapsignal;
453 		goto out;
454 	}
455 
456 	va = trunc_page(far);
457 	if (va >= KERNBASE) {
458 		/*
459 		 * Don't allow user-mode faults in kernel address space.
460 		 */
461 		if (usermode) {
462 			ksig.sig = SIGSEGV;
463 			ksig.code = SEGV_ACCERR;
464 			goto nogo;
465 		}
466 
467 		map = kernel_map;
468 	} else {
469 		/*
470 		 * This is a fault on non-kernel virtual memory. If curproc
471 		 * is NULL or curproc->p_vmspace is NULL the fault is fatal.
472 		 */
473 		vm = (p != NULL) ? p->p_vmspace : NULL;
474 		if (vm == NULL) {
475 			ksig.sig = SIGSEGV;
476 			ksig.code = 0;
477 			goto nogo;
478 		}
479 
480 		map = &vm->vm_map;
481 		if (!usermode && (td->td_intr_nesting_level != 0 ||
482 		    pcb->pcb_onfault == NULL)) {
483 			abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
484 			return;
485 		}
486 	}
487 
488 	ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ;
489 	if (prefetch)
490 		ftype |= VM_PROT_EXECUTE;
491 
492 #ifdef INVARIANTS
493 	onfault = pcb->pcb_onfault;
494 	pcb->pcb_onfault = NULL;
495 #endif
496 
497 	/* Fault in the page. */
498 	rv = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &ksig.sig,
499 	    &ucode);
500 	ksig.code = ucode;
501 
502 #ifdef INVARIANTS
503 	pcb->pcb_onfault = onfault;
504 #endif
505 
506 	if (__predict_true(rv == KERN_SUCCESS))
507 		goto out;
508 nogo:
509 	if (!usermode) {
510 		if (td->td_intr_nesting_level == 0 &&
511 		    pcb->pcb_onfault != NULL) {
512 			tf->tf_r0 = rv;
513 			tf->tf_pc = (int)pcb->pcb_onfault;
514 			return;
515 		}
516 		CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv);
517 		abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
518 		return;
519 	}
520 
521 	ksig.addr = far;
522 
523 do_trapsignal:
524 	call_trapsignal(td, ksig.sig, ksig.code, ksig.addr, idx);
525 out:
526 	if (usermode)
527 		userret(td, tf);
528 }
529 
530 /*
531  * abort_fatal() handles the following data aborts:
532  *
533  *  FAULT_DEBUG		- Debug Event
534  *  FAULT_ACCESS_xx	- Acces Bit
535  *  FAULT_EA_PREC	- Precise External Abort
536  *  FAULT_DOMAIN_xx	- Domain Fault
537  *  FAULT_EA_TRAN_xx	- External Translation Abort
538  *  FAULT_EA_IMPREC	- Imprecise External Abort
539  *  + all undefined codes for ABORT
540  *
541  * We should never see these on a properly functioning system.
542  *
543  * This function is also called by the other handlers if they
544  * detect a fatal problem.
545  *
546  * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
547  */
548 static int
549 abort_fatal(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
550     u_int prefetch, struct thread *td, struct ksig *ksig)
551 {
552 	bool usermode;
553 	const char *mode;
554 	const char *rw_mode;
555 #ifdef KDB
556 	bool handled;
557 #endif
558 
559 	usermode = TRAPF_USERMODE(tf);
560 #ifdef KDTRACE_HOOKS
561 	if (!usermode) {
562 		if (dtrace_trap_func != NULL && (*dtrace_trap_func)(tf, far))
563 			return (0);
564 	}
565 #endif
566 
567 	mode = usermode ? "user" : "kernel";
568 	rw_mode  = fsr & FSR_WNR ? "write" : "read";
569 	disable_interrupts(PSR_I|PSR_F);
570 
571 	if (td != NULL) {
572 		printf("Fatal %s mode data abort: '%s' on %s\n", mode,
573 		    aborts[idx].desc, rw_mode);
574 		printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
575 		if (idx != FAULT_EA_IMPREC)
576 			printf("%08x, ", far);
577 		else
578 			printf("Invalid,  ");
579 		printf("spsr=%08x\n", tf->tf_spsr);
580 	} else {
581 		printf("Fatal %s mode prefetch abort at 0x%08x\n",
582 		    mode, tf->tf_pc);
583 		printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
584 	}
585 
586 	printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
587 	    tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
588 	printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
589 	    tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
590 	printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
591 	    tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
592 	printf("r12=%08x, ", tf->tf_r12);
593 
594 	if (usermode)
595 		printf("usp=%08x, ulr=%08x",
596 		    tf->tf_usr_sp, tf->tf_usr_lr);
597 	else
598 		printf("ssp=%08x, slr=%08x",
599 		    tf->tf_svc_sp, tf->tf_svc_lr);
600 	printf(", pc =%08x\n\n", tf->tf_pc);
601 
602 #ifdef KDB
603 	if (debugger_on_trap) {
604 		kdb_why = KDB_WHY_TRAP;
605 		handled = kdb_trap(fsr, 0, tf);
606 		kdb_why = KDB_WHY_UNSET;
607 		if (handled)
608 			return (0);
609 	}
610 #endif
611 	panic("Fatal abort");
612 	/*NOTREACHED*/
613 }
614 
615 /*
616  * abort_align() handles the following data abort:
617  *
618  *  FAULT_ALIGN - Alignment fault
619  *
620  * Everything should be aligned in kernel with exception of user to kernel
621  * and vice versa data copying, so if pcb_onfault is not set, it's fatal.
622  * We generate signal in case of abort from user mode.
623  */
624 static int
625 abort_align(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
626     u_int prefetch, struct thread *td, struct ksig *ksig)
627 {
628 	bool usermode;
629 
630 	usermode = TRAPF_USERMODE(tf);
631 	if (!usermode) {
632 		if (td->td_intr_nesting_level == 0 && td != NULL &&
633 		    td->td_pcb->pcb_onfault != NULL) {
634 			tf->tf_r0 = EFAULT;
635 			tf->tf_pc = (int)td->td_pcb->pcb_onfault;
636 			return (0);
637 		}
638 		abort_fatal(tf, idx, fsr, far, prefetch, td, ksig);
639 	}
640 	/* Deliver a bus error signal to the process */
641 	ksig->code = BUS_ADRALN;
642 	ksig->sig = SIGBUS;
643 	ksig->addr = far;
644 	return (1);
645 }
646 
647 /*
648  * abort_icache() handles the following data abort:
649  *
650  * FAULT_ICACHE - Instruction cache maintenance
651  *
652  * According to manual, FAULT_ICACHE is translation fault during cache
653  * maintenance operation. In fact, no cache maintenance operation on
654  * not mapped virtual addresses should be called. As cache maintenance
655  * operation (except DMB, DSB, and Flush Prefetch Buffer) are privileged,
656  * the abort is concider as fatal for now. However, all the matter with
657  * cache maintenance operation on virtual addresses could be really complex
658  * and fuzzy in SMP case, so maybe in future standard fault mechanism
659  * should be held here including vm_fault() calling.
660  */
661 static int
662 abort_icache(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
663     u_int prefetch, struct thread *td, struct ksig *ksig)
664 {
665 
666 	abort_fatal(tf, idx, fsr, far, prefetch, td, ksig);
667 	return(0);
668 }
669