1 /* $NetBSD: fault.c,v 1.116 2021/02/01 19:31:34 skrll Exp $ */
2
3 /*
4 * Copyright 2003 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Steve C. Woodford for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37 /*
38 * Copyright (c) 1994-1997 Mark Brinicombe.
39 * Copyright (c) 1994 Brini.
40 * All rights reserved.
41 *
42 * This code is derived from software written for Brini by Mark Brinicombe
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. All advertising materials mentioning features or use of this software
53 * must display the following acknowledgement:
54 * This product includes software developed by Brini.
55 * 4. The name of the company nor the name of the author may be used to
56 * endorse or promote products derived from this software without specific
57 * prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * RiscBSD kernel project
72 *
73 * fault.c
74 *
75 * Fault handlers
76 *
77 * Created : 28/11/94
78 */
79
80 #include "opt_ddb.h"
81 #include "opt_kgdb.h"
82 #include "opt_multiprocessor.h"
83
84 #include <sys/types.h>
85 __KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.116 2021/02/01 19:31:34 skrll Exp $");
86
87 #include <sys/param.h>
88
89 #include <sys/cpu.h>
90 #include <sys/intr.h>
91 #include <sys/kauth.h>
92 #include <sys/kernel.h>
93 #include <sys/proc.h>
94 #include <sys/systm.h>
95
96 #include <uvm/uvm_extern.h>
97 #include <uvm/uvm_stat.h>
98 #ifdef UVMHIST
99 #include <uvm/uvm.h>
100 #endif
101
102 #include <arm/locore.h>
103
104 #include <machine/pcb.h>
105 #if defined(DDB) || defined(KGDB)
106 #include <machine/db_machdep.h>
107 #ifdef KGDB
108 #include <sys/kgdb.h>
109 #endif
110 #if !defined(DDB)
111 #define kdb_trap kgdb_trap
112 #endif
113 #endif
114
115 #include <arch/arm/arm/disassem.h>
116 #include <arm/arm32/machdep.h>
117
118 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
119 int last_fault_code; /* For the benefit of pmap_fault_fixup() */
120 #endif
121
122 #if defined(CPU_ARM6) || defined(CPU_ARM7) || defined(CPU_ARM7TDMI)
123 /* These CPUs may need data/prefetch abort fixups */
124 #define CPU_ABORT_FIXUP_REQUIRED
125 #endif
126
127 struct data_abort {
128 int (*func)(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
129 const char *desc;
130 };
131
132 static int dab_fatal(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
133 static int dab_align(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
134 static int dab_buserr(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
135
136 static const struct data_abort data_aborts[] = {
137 {dab_fatal, "Vector Exception"},
138 {dab_align, "Alignment Fault 1"},
139 {dab_fatal, "Terminal Exception"},
140 {dab_align, "Alignment Fault 3"},
141 {dab_buserr, "External Linefetch Abort (S)"},
142 {NULL, "Translation Fault (S)"},
143 {dab_buserr, "External Linefetch Abort (P)"},
144 {NULL, "Translation Fault (P)"},
145 {dab_buserr, "External Non-Linefetch Abort (S)"},
146 {NULL, "Domain Fault (S)"},
147 {dab_buserr, "External Non-Linefetch Abort (P)"},
148 {NULL, "Domain Fault (P)"},
149 {dab_buserr, "External Translation Abort (L1)"},
150 {NULL, "Permission Fault (S)"},
151 {dab_buserr, "External Translation Abort (L2)"},
152 {NULL, "Permission Fault (P)"}
153 };
154
155 /* Determine if 'x' is a permission fault */
156 #define IS_PERMISSION_FAULT(x) \
157 (((1 << ((x) & FAULT_TYPE_MASK)) & \
158 ((1 << FAULT_PERM_P) | (1 << FAULT_PERM_S))) != 0)
159
160 #if 0
161 /* maybe one day we'll do emulations */
162 #define TRAPSIGNAL(l,k) (*(l)->l_proc->p_emul->e_trapsignal)((l), (k))
163 #else
164 #define TRAPSIGNAL(l,k) trapsignal((l), (k))
165 #endif
166
167 static inline void
call_trapsignal(struct lwp * l,const struct trapframe * tf,ksiginfo_t * ksi)168 call_trapsignal(struct lwp *l, const struct trapframe *tf, ksiginfo_t *ksi)
169 {
170 if (l->l_proc->p_pid == 1 || cpu_printfataltraps) {
171 printf("%d.%d(%s): trap: signo=%d code=%d addr=%p trap=%#x\n",
172 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm,
173 ksi->ksi_signo, ksi->ksi_code, ksi->ksi_addr,
174 ksi->ksi_trap);
175 printf("r0=%08x r1=%08x r2=%08x r3=%08x\n",
176 tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
177 printf("r4=%08x r5=%08x r6=%08x r7=%08x\n",
178 tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
179 printf("r8=%08x r9=%08x rA=%08x rB=%08x\n",
180 tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
181 printf("ip=%08x sp=%08x lr=%08x pc=%08x spsr=%08x\n",
182 tf->tf_r12, tf->tf_usr_sp, tf->tf_usr_lr, tf->tf_pc,
183 tf->tf_spsr);
184 }
185
186 TRAPSIGNAL(l, ksi);
187 }
188
189 static inline int
data_abort_fixup(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l)190 data_abort_fixup(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l)
191 {
192 #ifdef CPU_ABORT_FIXUP_REQUIRED
193 int error;
194
195 /* Call the CPU specific data abort fixup routine */
196 error = cpu_dataabt_fixup(tf);
197 if (__predict_true(error != ABORT_FIXUP_FAILED))
198 return error;
199
200 /*
201 * Oops, couldn't fix up the instruction
202 */
203 printf("%s: fixup for %s mode data abort failed.\n", __func__,
204 TRAP_USERMODE(tf) ? "user" : "kernel");
205 #ifdef THUMB_CODE
206 if (tf->tf_spsr & PSR_T_bit) {
207 printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
208 tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
209 *((uint16 *)((tf->tf_pc + 2) & ~1)));
210 }
211 else
212 #endif
213 {
214 printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
215 *((u_int *)tf->tf_pc));
216 }
217 disassemble(tf->tf_pc);
218
219 /* Die now if this happened in kernel mode */
220 if (!TRAP_USERMODE(tf))
221 dab_fatal(tf, fsr, far, l, NULL);
222
223 return error;
224 #else
225 return ABORT_FIXUP_OK;
226 #endif /* CPU_ABORT_FIXUP_REQUIRED */
227 }
228
229 void
data_abort_handler(trapframe_t * tf)230 data_abort_handler(trapframe_t *tf)
231 {
232 struct vm_map *map;
233 struct lwp * const l = curlwp;
234 struct cpu_info * const ci = curcpu();
235 u_int far, fsr;
236 vm_prot_t ftype;
237 void *onfault;
238 vaddr_t va;
239 int error;
240 ksiginfo_t ksi;
241
242 UVMHIST_FUNC(__func__);
243 UVMHIST_CALLED(maphist);
244
245 /* Grab FAR/FSR before enabling interrupts */
246 far = cpu_faultaddress();
247 fsr = cpu_faultstatus();
248
249 /* Update vmmeter statistics */
250 ci->ci_data.cpu_ntrap++;
251
252 /* Re-enable interrupts if they were enabled previously */
253 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
254 #ifdef __NO_FIQ
255 if (__predict_true((tf->tf_spsr & I32_bit) != I32_bit))
256 restore_interrupts(tf->tf_spsr & IF32_bits);
257 #else
258 if (__predict_true((tf->tf_spsr & IF32_bits) != IF32_bits))
259 restore_interrupts(tf->tf_spsr & IF32_bits);
260 #endif
261
262 /* Get the current lwp structure */
263
264 UVMHIST_LOG(maphist, " (l=%#jx, far=%#jx, fsr=%#jx",
265 (uintptr_t)l, far, fsr, 0);
266 UVMHIST_LOG(maphist, " tf=%#jx, pc=%#jx)",
267 (uintptr_t)tf, (uintptr_t)tf->tf_pc, 0, 0);
268
269 /* Data abort came from user mode? */
270 bool user = (TRAP_USERMODE(tf) != 0);
271 if (user)
272 LWP_CACHE_CREDS(l, l->l_proc);
273
274 /* Grab the current pcb */
275 struct pcb * const pcb = lwp_getpcb(l);
276
277 curcpu()->ci_abt_evs[fsr & FAULT_TYPE_MASK].ev_count++;
278
279 /* Invoke the appropriate handler, if necessary */
280 if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
281 #ifdef DIAGNOSTIC
282 printf("%s: data_aborts fsr=0x%x far=0x%x\n",
283 __func__, fsr, far);
284 #endif
285 if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
286 l, &ksi))
287 goto do_trapsignal;
288 goto out;
289 }
290
291 /*
292 * At this point, we're dealing with one of the following data aborts:
293 *
294 * FAULT_TRANS_S - Translation -- Section
295 * FAULT_TRANS_P - Translation -- Page
296 * FAULT_DOMAIN_S - Domain -- Section
297 * FAULT_DOMAIN_P - Domain -- Page
298 * FAULT_PERM_S - Permission -- Section
299 * FAULT_PERM_P - Permission -- Page
300 *
301 * These are the main virtual memory-related faults signalled by
302 * the MMU.
303 */
304
305 KASSERTMSG(!user || tf == lwp_trapframe(l), "tf %p vs %p", tf,
306 lwp_trapframe(l));
307
308 /*
309 * Make sure the Program Counter is sane. We could fall foul of
310 * someone executing Thumb code, in which case the PC might not
311 * be word-aligned. This would cause a kernel alignment fault
312 * further down if we have to decode the current instruction.
313 */
314 #ifdef THUMB_CODE
315 /*
316 * XXX: It would be nice to be able to support Thumb in the kernel
317 * at some point.
318 */
319 if (__predict_false(!user && (tf->tf_pc & 3) != 0)) {
320 printf("\n%s: Misaligned Kernel-mode Program Counter\n",
321 __func__);
322 dab_fatal(tf, fsr, far, l, NULL);
323 }
324 #else
325 if (__predict_false((tf->tf_pc & 3) != 0)) {
326 if (user) {
327 /*
328 * Give the user an illegal instruction signal.
329 */
330 /* Deliver a SIGILL to the process */
331 KSI_INIT_TRAP(&ksi);
332 ksi.ksi_signo = SIGILL;
333 ksi.ksi_code = ILL_ILLOPC;
334 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
335 ksi.ksi_trap = fsr;
336 goto do_trapsignal;
337 }
338
339 /*
340 * The kernel never executes Thumb code.
341 */
342 printf("\n%s: Misaligned Kernel-mode Program Counter\n",
343 __func__);
344 dab_fatal(tf, fsr, far, l, NULL);
345 }
346 #endif
347
348 /* See if the CPU state needs to be fixed up */
349 switch (data_abort_fixup(tf, fsr, far, l)) {
350 case ABORT_FIXUP_RETURN:
351 return;
352 case ABORT_FIXUP_FAILED:
353 /* Deliver a SIGILL to the process */
354 KSI_INIT_TRAP(&ksi);
355 ksi.ksi_signo = SIGILL;
356 ksi.ksi_code = ILL_ILLOPC;
357 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
358 ksi.ksi_trap = fsr;
359 goto do_trapsignal;
360 default:
361 break;
362 }
363
364 va = trunc_page((vaddr_t)far);
365
366 /*
367 * It is only a kernel address space fault iff:
368 * 1. user == 0 and
369 * 2. pcb_onfault not set or
370 * 3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
371 */
372 if (!user && (va >= VM_MIN_KERNEL_ADDRESS ||
373 (va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
374 __predict_true((pcb->pcb_onfault == NULL ||
375 (read_insn(tf->tf_pc, false) & 0x05200000) != 0x04200000))) {
376 map = kernel_map;
377
378 /* Was the fault due to the FPE ? */
379 if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
380 KSI_INIT_TRAP(&ksi);
381 ksi.ksi_signo = SIGSEGV;
382 ksi.ksi_code = SEGV_ACCERR;
383 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
384 ksi.ksi_trap = fsr;
385
386 /*
387 * Force exit via userret()
388 * This is necessary as the FPE is an extension to
389 * userland that actually runs in a priveledged mode
390 * but uses USR mode permissions for its accesses.
391 */
392 user = true;
393 goto do_trapsignal;
394 }
395 } else {
396 map = &l->l_proc->p_vmspace->vm_map;
397 }
398
399 /*
400 * We need to know whether the page should be mapped as R or R/W.
401 * Before ARMv6, the MMU did not give us the info as to whether the
402 * fault was caused by a read or a write.
403 *
404 * However, we know that a permission fault can only be the result of
405 * a write to a read-only location, so we can deal with those quickly.
406 *
407 * Otherwise we need to disassemble the instruction responsible to
408 * determine if it was a write.
409 */
410 if (CPU_IS_ARMV6_P() || CPU_IS_ARMV7_P()) {
411 ftype = (fsr & FAULT_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
412 } else if (IS_PERMISSION_FAULT(fsr)) {
413 ftype = VM_PROT_WRITE;
414 } else {
415 #ifdef THUMB_CODE
416 /* Fast track the ARM case. */
417 if (__predict_false(tf->tf_spsr & PSR_T_bit)) {
418 u_int insn = read_thumb_insn(tf->tf_pc, user);
419 u_int insn_f8 = insn & 0xf800;
420 u_int insn_fe = insn & 0xfe00;
421
422 if (insn_f8 == 0x6000 || /* STR(1) */
423 insn_f8 == 0x7000 || /* STRB(1) */
424 insn_f8 == 0x8000 || /* STRH(1) */
425 insn_f8 == 0x9000 || /* STR(3) */
426 insn_f8 == 0xc000 || /* STM */
427 insn_fe == 0x5000 || /* STR(2) */
428 insn_fe == 0x5200 || /* STRH(2) */
429 insn_fe == 0x5400) /* STRB(2) */
430 ftype = VM_PROT_WRITE;
431 else
432 ftype = VM_PROT_READ;
433 }
434 else
435 #endif
436 {
437 u_int insn = read_insn(tf->tf_pc, user);
438
439 if (((insn & 0x0c100000) == 0x04000000) || /* STR[B] */
440 ((insn & 0x0e1000b0) == 0x000000b0) || /* STR[HD]*/
441 ((insn & 0x0a100000) == 0x08000000) || /* STM/CDT*/
442 ((insn & 0x0f9000f0) == 0x01800090)) /* STREX[BDH] */
443 ftype = VM_PROT_WRITE;
444 else if ((insn & 0x0fb00ff0) == 0x01000090)/* SWP */
445 ftype = VM_PROT_READ | VM_PROT_WRITE;
446 else
447 ftype = VM_PROT_READ;
448 }
449 }
450
451 /*
452 * See if the fault is as a result of ref/mod emulation,
453 * or domain mismatch.
454 */
455 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
456 last_fault_code = fsr;
457 #endif
458 if (pmap_fault_fixup(map->pmap, va, ftype, user)) {
459 UVMHIST_LOG(maphist, " <- ref/mod emul", 0, 0, 0, 0);
460 goto out;
461 }
462
463 if (__predict_false(curcpu()->ci_intr_depth > 0)) {
464 if (pcb->pcb_onfault) {
465 tf->tf_r0 = EINVAL;
466 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
467 return;
468 }
469 printf("\nNon-emulated page fault with intr_depth > 0\n");
470 dab_fatal(tf, fsr, far, l, NULL);
471 }
472
473 #ifdef PMAP_FAULTINFO
474 struct pcb_faultinfo * const pfi = &pcb->pcb_faultinfo;
475 struct proc * const p = curproc;
476
477 if (p->p_pid == pfi->pfi_lastpid && va == pfi->pfi_faultaddr) {
478 if (++pfi->pfi_repeats > 4) {
479 tlb_asid_t asid = tlb_get_asid();
480 pt_entry_t *ptep = pfi->pfi_faultptep;
481
482 printf("%s: fault #%u (%x/%s) for %#" PRIxVADDR
483 "(%#x) at pc %#" PRIxREGISTER " curpid=%u/%u "
484 "ptep@%p=%#" PRIxPTE ")\n", __func__,
485 pfi->pfi_repeats, fsr & FAULT_TYPE_MASK,
486 data_aborts[fsr & FAULT_TYPE_MASK].desc, va,
487 far, tf->tf_pc, map->pmap->pm_pai[0].pai_asid,
488 asid, ptep, ptep ? *ptep : 0);
489 cpu_Debugger();
490 }
491 } else {
492 pfi->pfi_lastpid = p->p_pid;
493 pfi->pfi_faultaddr = va;
494 pfi->pfi_repeats = 0;
495 pfi->pfi_faultptep = NULL;
496 pfi->pfi_faulttype = fsr & FAULT_TYPE_MASK;
497 }
498 #endif /* PMAP_FAULTINFO */
499
500 onfault = pcb->pcb_onfault;
501 pcb->pcb_onfault = NULL;
502 error = uvm_fault(map, va, ftype);
503 pcb->pcb_onfault = onfault;
504
505 if (__predict_true(error == 0)) {
506 if (user)
507 uvm_grow(l->l_proc, va); /* Record any stack growth */
508 UVMHIST_LOG(maphist, " <- uvm", 0, 0, 0, 0);
509 goto out;
510 }
511
512 if (user == 0) {
513 if (pcb->pcb_onfault) {
514 tf->tf_r0 = error;
515 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
516 return;
517 }
518
519 printf("\nuvm_fault(%p, %lx, %x) -> %x\n", map, va, ftype,
520 error);
521 dab_fatal(tf, fsr, far, l, NULL);
522 }
523
524 KSI_INIT_TRAP(&ksi);
525
526 switch (error) {
527 case ENOMEM:
528 printf("UVM: pid %d (%s), uid %d killed: "
529 "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
530 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
531 ksi.ksi_signo = SIGKILL;
532 break;
533 case EACCES:
534 ksi.ksi_signo = SIGSEGV;
535 ksi.ksi_code = SEGV_ACCERR;
536 break;
537 case EINVAL:
538 ksi.ksi_signo = SIGBUS;
539 ksi.ksi_code = BUS_ADRERR;
540 break;
541 default:
542 ksi.ksi_signo = SIGSEGV;
543 ksi.ksi_code = SEGV_MAPERR;
544 break;
545 }
546 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
547 ksi.ksi_trap = fsr;
548 UVMHIST_LOG(maphist, " <- error (%jd)", error, 0, 0, 0);
549
550 do_trapsignal:
551 call_trapsignal(l, tf, &ksi);
552 out:
553 /* If returning to user mode, make sure to invoke userret() */
554 if (user)
555 userret(l);
556 }
557
558 /*
559 * dab_fatal() handles the following data aborts:
560 *
561 * FAULT_WRTBUF_0 - Vector Exception
562 * FAULT_WRTBUF_1 - Terminal Exception
563 *
564 * We should never see these on a properly functioning system.
565 *
566 * This function is also called by the other handlers if they
567 * detect a fatal problem.
568 *
569 * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
570 */
571 static int
dab_fatal(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)572 dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
573 {
574 const char * const mode = TRAP_USERMODE(tf) ? "user" : "kernel";
575
576 if (l != NULL) {
577 printf("Fatal %s mode data abort: '%s'\n", mode,
578 data_aborts[fsr & FAULT_TYPE_MASK].desc);
579 printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
580 if ((fsr & FAULT_IMPRECISE) == 0)
581 printf("%08x, ", far);
582 else
583 printf("Invalid, ");
584 printf("spsr=%08x\n", tf->tf_spsr);
585 } else {
586 printf("Fatal %s mode prefetch abort at 0x%08x\n",
587 mode, tf->tf_pc);
588 printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
589 }
590
591 printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
592 tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
593 printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
594 tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
595 printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
596 tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
597 printf("r12=%08x, ", tf->tf_r12);
598
599 if (TRAP_USERMODE(tf))
600 printf("usp=%08x, ulr=%08x",
601 tf->tf_usr_sp, tf->tf_usr_lr);
602 else
603 printf("ssp=%08x, slr=%08x",
604 tf->tf_svc_sp, tf->tf_svc_lr);
605 printf(", pc =%08x\n\n", tf->tf_pc);
606
607 #if defined(DDB) || defined(KGDB)
608 kdb_trap(T_FAULT, tf);
609 #endif
610 panic("Fatal abort");
611 /*NOTREACHED*/
612 }
613
614 /*
615 * dab_align() handles the following data aborts:
616 *
617 * FAULT_ALIGN_0 - Alignment fault
618 * FAULT_ALIGN_0 - Alignment fault
619 *
620 * These faults are fatal if they happen in kernel mode. Otherwise, we
621 * deliver a bus error to the process.
622 */
623 static int
dab_align(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)624 dab_align(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
625 {
626 /* Alignment faults are always fatal if they occur in kernel mode */
627 if (!TRAP_USERMODE(tf))
628 dab_fatal(tf, fsr, far, l, NULL);
629
630 /* pcb_onfault *must* be NULL at this point */
631 KDASSERT(((struct pcb *)lwp_getpcb(l))->pcb_onfault == NULL);
632
633 /* See if the CPU state needs to be fixed up */
634 (void) data_abort_fixup(tf, fsr, far, l);
635
636 /* Deliver a bus error signal to the process */
637 KSI_INIT_TRAP(ksi);
638 ksi->ksi_signo = SIGBUS;
639 ksi->ksi_code = BUS_ADRALN;
640 ksi->ksi_addr = (uint32_t *)(intptr_t)far;
641 ksi->ksi_trap = fsr;
642
643 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
644
645 return 1;
646 }
647
648 /*
649 * dab_buserr() handles the following data aborts:
650 *
651 * FAULT_BUSERR_0 - External Abort on Linefetch -- Section
652 * FAULT_BUSERR_1 - External Abort on Linefetch -- Page
653 * FAULT_BUSERR_2 - External Abort on Non-linefetch -- Section
654 * FAULT_BUSERR_3 - External Abort on Non-linefetch -- Page
655 * FAULT_BUSTRNL1 - External abort on Translation -- Level 1
656 * FAULT_BUSTRNL2 - External abort on Translation -- Level 2
657 *
658 * If pcb_onfault is set, flag the fault and return to the handler.
659 * If the fault occurred in user mode, give the process a SIGBUS.
660 *
661 * Note: On XScale, FAULT_BUSERR_0, FAULT_BUSERR_1, and FAULT_BUSERR_2
662 * can be flagged as imprecise in the FSR. This causes a real headache
663 * since some of the machine state is lost. In this case, tf->tf_pc
664 * may not actually point to the offending instruction. In fact, if
665 * we've taken a double abort fault, it generally points somewhere near
666 * the top of "data_abort_entry" in exception.S.
667 *
668 * In all other cases, these data aborts are considered fatal.
669 */
670 static int
dab_buserr(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)671 dab_buserr(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l,
672 ksiginfo_t *ksi)
673 {
674 struct pcb *pcb = lwp_getpcb(l);
675
676 #ifdef __XSCALE__
677 if ((fsr & FAULT_IMPRECISE) != 0 &&
678 (tf->tf_spsr & PSR_MODE) == PSR_ABT32_MODE) {
679 /*
680 * Oops, an imprecise, double abort fault. We've lost the
681 * r14_abt/spsr_abt values corresponding to the original
682 * abort, and the spsr saved in the trapframe indicates
683 * ABT mode.
684 */
685 tf->tf_spsr &= ~PSR_MODE;
686
687 /*
688 * We use a simple heuristic to determine if the double abort
689 * happened as a result of a kernel or user mode access.
690 * If the current trapframe is at the top of the kernel stack,
691 * the fault _must_ have come from user mode.
692 */
693 if (tf != ((trapframe_t *)pcb->pcb_ksp) - 1) {
694 /*
695 * Kernel mode. We're either about to die a
696 * spectacular death, or pcb_onfault will come
697 * to our rescue. Either way, the current value
698 * of tf->tf_pc is irrelevant.
699 */
700 tf->tf_spsr |= PSR_SVC32_MODE;
701 if (pcb->pcb_onfault == NULL)
702 printf("\nKernel mode double abort!\n");
703 } else {
704 /*
705 * User mode. We've lost the program counter at the
706 * time of the fault (not that it was accurate anyway;
707 * it's not called an imprecise fault for nothing).
708 * About all we can do is copy r14_usr to tf_pc and
709 * hope for the best. The process is about to get a
710 * SIGBUS, so it's probably history anyway.
711 */
712 tf->tf_spsr |= PSR_USR32_MODE;
713 tf->tf_pc = tf->tf_usr_lr;
714 #ifdef THUMB_CODE
715 tf->tf_spsr &= ~PSR_T_bit;
716 if (tf->tf_usr_lr & 1)
717 tf->tf_spsr |= PSR_T_bit;
718 #endif
719 }
720 }
721
722 /* FAR is invalid for imprecise exceptions */
723 if ((fsr & FAULT_IMPRECISE) != 0)
724 far = 0;
725 #endif /* __XSCALE__ */
726
727 if (pcb->pcb_onfault) {
728 KDASSERT(TRAP_USERMODE(tf) == 0);
729 tf->tf_r0 = EFAULT;
730 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
731 return 0;
732 }
733
734 /* See if the CPU state needs to be fixed up */
735 (void) data_abort_fixup(tf, fsr, far, l);
736
737 /*
738 * At this point, if the fault happened in kernel mode, we're toast
739 */
740 if (!TRAP_USERMODE(tf))
741 dab_fatal(tf, fsr, far, l, NULL);
742
743 /* Deliver a bus error signal to the process */
744 KSI_INIT_TRAP(ksi);
745 ksi->ksi_signo = SIGBUS;
746 ksi->ksi_code = BUS_ADRERR;
747 ksi->ksi_addr = (uint32_t *)(intptr_t)far;
748 ksi->ksi_trap = fsr;
749
750 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
751
752 return 1;
753 }
754
755 static inline int
prefetch_abort_fixup(trapframe_t * tf)756 prefetch_abort_fixup(trapframe_t *tf)
757 {
758 #ifdef CPU_ABORT_FIXUP_REQUIRED
759 int error;
760
761 /* Call the CPU specific prefetch abort fixup routine */
762 error = cpu_prefetchabt_fixup(tf);
763 if (__predict_true(error != ABORT_FIXUP_FAILED))
764 return error;
765
766 /*
767 * Oops, couldn't fix up the instruction
768 */
769 printf("%s: fixup for %s mode prefetch abort failed.\n", __func__,
770 TRAP_USERMODE(tf) ? "user" : "kernel");
771 #ifdef THUMB_CODE
772 if (tf->tf_spsr & PSR_T_bit) {
773 printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
774 tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
775 *((uint16 *)((tf->tf_pc + 2) & ~1)));
776 }
777 else
778 #endif
779 {
780 printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
781 *((u_int *)tf->tf_pc));
782 }
783 disassemble(tf->tf_pc);
784
785 /* Die now if this happened in kernel mode */
786 if (!TRAP_USERMODE(tf))
787 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
788
789 return error;
790 #else
791 return ABORT_FIXUP_OK;
792 #endif /* CPU_ABORT_FIXUP_REQUIRED */
793 }
794
795 /*
796 * void prefetch_abort_handler(trapframe_t *tf)
797 *
798 * Abort handler called when instruction execution occurs at
799 * a non existent or restricted (access permissions) memory page.
800 * If the address is invalid and we were in SVC mode then panic as
801 * the kernel should never prefetch abort.
802 * If the address is invalid and the page is mapped then the user process
803 * does not have read permission so send it a signal.
804 * Otherwise fault the page in and try again.
805 */
806 void
prefetch_abort_handler(trapframe_t * tf)807 prefetch_abort_handler(trapframe_t *tf)
808 {
809 struct lwp *l;
810 struct pcb *pcb __diagused;
811 struct vm_map *map;
812 vaddr_t fault_pc, va;
813 ksiginfo_t ksi;
814 int error, user;
815
816 UVMHIST_FUNC(__func__);
817 UVMHIST_CALLED(maphist);
818
819 /* Update vmmeter statistics */
820 curcpu()->ci_data.cpu_ntrap++;
821
822 l = curlwp;
823 pcb = lwp_getpcb(l);
824
825 if ((user = TRAP_USERMODE(tf)) != 0)
826 LWP_CACHE_CREDS(l, l->l_proc);
827
828 /*
829 * Enable IRQ's (disabled by the abort) This always comes
830 * from user mode so we know interrupts were not disabled.
831 * But we check anyway.
832 */
833 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
834 #ifdef __NO_FIQ
835 if (__predict_true((tf->tf_spsr & I32_bit) != I32_bit))
836 restore_interrupts(tf->tf_spsr & IF32_bits);
837 #else
838 if (__predict_true((tf->tf_spsr & IF32_bits) != IF32_bits))
839 restore_interrupts(tf->tf_spsr & IF32_bits);
840 #endif
841
842 /* See if the CPU state needs to be fixed up */
843 switch (prefetch_abort_fixup(tf)) {
844 case ABORT_FIXUP_RETURN:
845 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
846 return;
847 case ABORT_FIXUP_FAILED:
848 /* Deliver a SIGILL to the process */
849 KSI_INIT_TRAP(&ksi);
850 ksi.ksi_signo = SIGILL;
851 ksi.ksi_code = ILL_ILLOPC;
852 ksi.ksi_addr = (uint32_t *)(intptr_t) tf->tf_pc;
853 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf,
854 lwp_trapframe(l));
855 goto do_trapsignal;
856 default:
857 break;
858 }
859
860 /* Prefetch aborts cannot happen in kernel mode */
861 if (__predict_false(!user))
862 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
863
864 /* Get fault address */
865 fault_pc = tf->tf_pc;
866 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
867 UVMHIST_LOG(maphist, " (pc=%#jx, l=%#jx, tf=%#jx)",
868 fault_pc, (uintptr_t)l, (uintptr_t)tf, 0);
869
870 #ifdef THUMB_CODE
871 recheck:
872 #endif
873 /* Ok validate the address, can only execute in USER space */
874 if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS ||
875 (fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) {
876 KSI_INIT_TRAP(&ksi);
877 ksi.ksi_signo = SIGSEGV;
878 ksi.ksi_code = SEGV_ACCERR;
879 ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
880 ksi.ksi_trap = fault_pc;
881 goto do_trapsignal;
882 }
883
884 map = &l->l_proc->p_vmspace->vm_map;
885 va = trunc_page(fault_pc);
886
887 /*
888 * See if the pmap can handle this fault on its own...
889 */
890 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
891 last_fault_code = -1;
892 #endif
893 if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ|VM_PROT_EXECUTE, 1)) {
894 UVMHIST_LOG (maphist, " <- emulated", 0, 0, 0, 0);
895 goto out;
896 }
897
898 #ifdef DIAGNOSTIC
899 if (__predict_false(curcpu()->ci_intr_depth > 0)) {
900 printf("\nNon-emulated prefetch abort with intr_depth > 0\n");
901 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
902 }
903 #endif
904
905 KASSERT(pcb->pcb_onfault == NULL);
906 error = uvm_fault(map, va, VM_PROT_READ|VM_PROT_EXECUTE);
907
908 if (__predict_true(error == 0)) {
909 UVMHIST_LOG (maphist, " <- uvm", 0, 0, 0, 0);
910 goto out;
911 }
912 KSI_INIT_TRAP(&ksi);
913
914 UVMHIST_LOG (maphist, " <- fatal (%jd)", error, 0, 0, 0);
915
916 if (error == ENOMEM) {
917 printf("UVM: pid %d (%s), uid %d killed: "
918 "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
919 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
920 ksi.ksi_signo = SIGKILL;
921 } else
922 ksi.ksi_signo = SIGSEGV;
923
924 ksi.ksi_code = SEGV_MAPERR;
925 ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
926 ksi.ksi_trap = fault_pc;
927
928 do_trapsignal:
929 call_trapsignal(l, tf, &ksi);
930
931 out:
932
933 #ifdef THUMB_CODE
934 #define THUMB_32BIT(hi) (((hi) & 0xe000) == 0xe000 && ((hi) & 0x1800))
935 /* thumb-32 instruction was located on page boundary? */
936 if ((tf->tf_spsr & PSR_T_bit) &&
937 ((fault_pc & PAGE_MASK) == (PAGE_SIZE - THUMB_INSN_SIZE)) &&
938 THUMB_32BIT(*(uint16_t *)tf->tf_pc)) {
939 fault_pc = tf->tf_pc + THUMB_INSN_SIZE;
940 goto recheck;
941 }
942 #endif /* THUMB_CODE */
943
944 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
945 userret(l);
946 }
947
948 /*
949 * Tentatively read an 8, 16, or 32-bit value from 'addr'.
950 * If the read succeeds, the value is written to 'rptr' and zero is returned.
951 * Else, return EFAULT.
952 */
953 int
badaddr_read(void * addr,size_t size,void * rptr)954 badaddr_read(void *addr, size_t size, void *rptr)
955 {
956 extern int badaddr_read_1(const uint8_t *, uint8_t *);
957 extern int badaddr_read_2(const uint16_t *, uint16_t *);
958 extern int badaddr_read_4(const uint32_t *, uint32_t *);
959 union {
960 uint8_t v1;
961 uint16_t v2;
962 uint32_t v4;
963 } u;
964 int rv, s;
965
966 cpu_drain_writebuf();
967
968 s = splhigh();
969
970 /* Read from the test address. */
971 switch (size) {
972 case sizeof(uint8_t):
973 rv = badaddr_read_1(addr, &u.v1);
974 if (rv == 0 && rptr)
975 *(uint8_t *) rptr = u.v1;
976 break;
977
978 case sizeof(uint16_t):
979 rv = badaddr_read_2(addr, &u.v2);
980 if (rv == 0 && rptr)
981 *(uint16_t *) rptr = u.v2;
982 break;
983
984 case sizeof(uint32_t):
985 rv = badaddr_read_4(addr, &u.v4);
986 if (rv == 0 && rptr)
987 *(uint32_t *) rptr = u.v4;
988 break;
989
990 default:
991 panic("%s: invalid size (%zu)", __func__, size);
992 }
993
994 splx(s);
995
996 /* Return EFAULT if the address was invalid, else zero */
997 return rv;
998 }
999