1 /*- 2 * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/bus.h> 45 #include <sys/proc.h> 46 #include <sys/ptrace.h> 47 #include <sys/syscall.h> 48 #include <sys/sysent.h> 49 #ifdef KDB 50 #include <sys/kdb.h> 51 #endif 52 53 #include <vm/vm.h> 54 #include <vm/pmap.h> 55 #include <vm/vm_kern.h> 56 #include <vm/vm_map.h> 57 #include <vm/vm_param.h> 58 #include <vm/vm_extern.h> 59 60 #include <machine/fpe.h> 61 #include <machine/frame.h> 62 #include <machine/pcb.h> 63 #include <machine/pcpu.h> 64 65 #include <machine/resource.h> 66 #include <machine/intr.h> 67 68 #ifdef KDTRACE_HOOKS 69 #include <sys/dtrace_bsd.h> 70 #endif 71 72 int (*dtrace_invop_jump_addr)(struct trapframe *); 73 74 /* Called from exception.S */ 75 void do_trap_supervisor(struct trapframe *); 76 void do_trap_user(struct trapframe *); 77 78 static __inline void 79 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno) 80 { 81 ksiginfo_t ksi; 82 83 ksiginfo_init_trap(&ksi); 84 ksi.ksi_signo = sig; 85 ksi.ksi_code = code; 86 ksi.ksi_addr = addr; 87 ksi.ksi_trapno = trapno; 88 trapsignal(td, &ksi); 89 } 90 91 int 92 cpu_fetch_syscall_args(struct thread *td) 93 { 94 struct proc *p; 95 syscallarg_t *ap, *dst_ap; 96 struct syscall_args *sa; 97 98 p = td->td_proc; 99 sa = &td->td_sa; 100 ap = &td->td_frame->tf_a[0]; 101 dst_ap = &sa->args[0]; 102 103 sa->code = td->td_frame->tf_t[0]; 104 sa->original_code = sa->code; 105 106 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) { 107 sa->code = *ap++; 108 } else { 109 *dst_ap++ = *ap++; 110 } 111 112 if (__predict_false(sa->code >= p->p_sysent->sv_size)) 113 sa->callp = &p->p_sysent->sv_table[0]; 114 else 115 sa->callp = &p->p_sysent->sv_table[sa->code]; 116 117 KASSERT(sa->callp->sy_narg <= nitems(sa->args), 118 ("Syscall %d takes too many arguments", sa->code)); 119 120 memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap)); 121 122 td->td_retval[0] = 0; 123 td->td_retval[1] = 0; 124 125 return (0); 126 } 127 128 #include "../../kern/subr_syscall.c" 129 130 static void 131 dump_regs(struct trapframe *frame) 132 { 133 int n; 134 int i; 135 136 n = nitems(frame->tf_t); 137 for (i = 0; i < n; i++) 138 printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]); 139 140 n = nitems(frame->tf_s); 141 for (i = 0; i < n; i++) 142 printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]); 143 144 n = nitems(frame->tf_a); 145 for (i = 0; i < n; i++) 146 printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]); 147 148 printf("ra == 0x%016lx\n", frame->tf_ra); 149 printf("sp == 0x%016lx\n", frame->tf_sp); 150 printf("gp == 0x%016lx\n", frame->tf_gp); 151 printf("tp == 0x%016lx\n", frame->tf_tp); 152 153 printf("sepc == 0x%016lx\n", frame->tf_sepc); 154 printf("sstatus == 0x%016lx\n", frame->tf_sstatus); 155 } 156 157 static void 158 ecall_handler(void) 159 { 160 struct thread *td; 161 162 td = curthread; 163 164 syscallenter(td); 165 syscallret(td); 166 } 167 168 static void 169 page_fault_handler(struct trapframe *frame, int usermode) 170 { 171 struct vm_map *map; 172 uint64_t stval; 173 struct thread *td; 174 struct pcb *pcb; 175 vm_prot_t ftype; 176 vm_offset_t va; 177 struct proc *p; 178 int error, sig, ucode; 179 #ifdef KDB 180 bool handled; 181 #endif 182 183 #ifdef KDB 184 if (kdb_active) { 185 kdb_reenter(); 186 return; 187 } 188 #endif 189 190 td = curthread; 191 p = td->td_proc; 192 pcb = td->td_pcb; 193 stval = frame->tf_stval; 194 195 if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 || 196 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 197 "Kernel page fault") != 0) 198 goto fatal; 199 200 if (usermode) { 201 if (!VIRT_IS_VALID(stval)) { 202 call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval, 203 frame->tf_scause & SCAUSE_CODE); 204 goto done; 205 } 206 map = &p->p_vmspace->vm_map; 207 } else { 208 /* 209 * Enable interrupts for the duration of the page fault. For 210 * user faults this was done already in do_trap_user(). 211 */ 212 intr_enable(); 213 214 if (stval >= VM_MIN_KERNEL_ADDRESS) { 215 map = kernel_map; 216 } else { 217 if (pcb->pcb_onfault == 0) 218 goto fatal; 219 map = &p->p_vmspace->vm_map; 220 } 221 } 222 223 va = trunc_page(stval); 224 225 if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) { 226 ftype = VM_PROT_WRITE; 227 } else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) { 228 ftype = VM_PROT_EXECUTE; 229 } else { 230 ftype = VM_PROT_READ; 231 } 232 233 if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype)) 234 goto done; 235 236 error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode); 237 if (error != KERN_SUCCESS) { 238 if (usermode) { 239 call_trapsignal(td, sig, ucode, (void *)stval, 240 frame->tf_scause & SCAUSE_CODE); 241 } else { 242 if (pcb->pcb_onfault != 0) { 243 frame->tf_a[0] = error; 244 frame->tf_sepc = pcb->pcb_onfault; 245 return; 246 } 247 goto fatal; 248 } 249 } 250 251 done: 252 if (usermode) 253 userret(td, frame); 254 return; 255 256 fatal: 257 dump_regs(frame); 258 #ifdef KDB 259 if (debugger_on_trap) { 260 kdb_why = KDB_WHY_TRAP; 261 handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame); 262 kdb_why = KDB_WHY_UNSET; 263 if (handled) 264 return; 265 } 266 #endif 267 panic("Fatal page fault at %#lx: %#016lx", frame->tf_sepc, stval); 268 } 269 270 void 271 do_trap_supervisor(struct trapframe *frame) 272 { 273 uint64_t exception; 274 275 /* Ensure we came from supervisor mode, interrupts disabled */ 276 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 277 SSTATUS_SPP, ("Came from S mode with interrupts enabled")); 278 279 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 280 ("Came from S mode with SUM enabled")); 281 282 exception = frame->tf_scause & SCAUSE_CODE; 283 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 284 /* Interrupt */ 285 riscv_cpu_intr(frame); 286 return; 287 } 288 289 #ifdef KDTRACE_HOOKS 290 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 291 return; 292 #endif 293 294 CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p", 295 curthread, frame->tf_sepc, frame); 296 297 switch (exception) { 298 case SCAUSE_LOAD_ACCESS_FAULT: 299 case SCAUSE_STORE_ACCESS_FAULT: 300 case SCAUSE_INST_ACCESS_FAULT: 301 dump_regs(frame); 302 panic("Memory access exception at 0x%016lx\n", frame->tf_sepc); 303 break; 304 case SCAUSE_LOAD_MISALIGNED: 305 case SCAUSE_STORE_MISALIGNED: 306 case SCAUSE_INST_MISALIGNED: 307 dump_regs(frame); 308 panic("Misaligned address exception at %#016lx: %#016lx\n", 309 frame->tf_sepc, frame->tf_stval); 310 break; 311 case SCAUSE_STORE_PAGE_FAULT: 312 case SCAUSE_LOAD_PAGE_FAULT: 313 case SCAUSE_INST_PAGE_FAULT: 314 page_fault_handler(frame, 0); 315 break; 316 case SCAUSE_BREAKPOINT: 317 #ifdef KDTRACE_HOOKS 318 if (dtrace_invop_jump_addr != NULL && 319 dtrace_invop_jump_addr(frame) == 0) 320 break; 321 #endif 322 #ifdef KDB 323 kdb_trap(exception, 0, frame); 324 #else 325 dump_regs(frame); 326 panic("No debugger in kernel.\n"); 327 #endif 328 break; 329 case SCAUSE_ILLEGAL_INSTRUCTION: 330 dump_regs(frame); 331 panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc); 332 break; 333 default: 334 dump_regs(frame); 335 panic("Unknown kernel exception %lx trap value %lx\n", 336 exception, frame->tf_stval); 337 } 338 } 339 340 void 341 do_trap_user(struct trapframe *frame) 342 { 343 uint64_t exception; 344 struct thread *td; 345 struct pcb *pcb; 346 347 td = curthread; 348 pcb = td->td_pcb; 349 350 KASSERT(td->td_frame == frame, 351 ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame)); 352 353 /* Ensure we came from usermode, interrupts disabled */ 354 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0, 355 ("Came from U mode with interrupts enabled")); 356 357 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 358 ("Came from U mode with SUM enabled")); 359 360 exception = frame->tf_scause & SCAUSE_CODE; 361 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 362 /* Interrupt */ 363 riscv_cpu_intr(frame); 364 return; 365 } 366 intr_enable(); 367 368 CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p", 369 curthread, frame->tf_sepc, frame); 370 371 switch (exception) { 372 case SCAUSE_LOAD_ACCESS_FAULT: 373 case SCAUSE_STORE_ACCESS_FAULT: 374 case SCAUSE_INST_ACCESS_FAULT: 375 call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc, 376 exception); 377 userret(td, frame); 378 break; 379 case SCAUSE_LOAD_MISALIGNED: 380 case SCAUSE_STORE_MISALIGNED: 381 case SCAUSE_INST_MISALIGNED: 382 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sepc, 383 exception); 384 userret(td, frame); 385 break; 386 case SCAUSE_STORE_PAGE_FAULT: 387 case SCAUSE_LOAD_PAGE_FAULT: 388 case SCAUSE_INST_PAGE_FAULT: 389 page_fault_handler(frame, 1); 390 break; 391 case SCAUSE_ECALL_USER: 392 frame->tf_sepc += 4; /* Next instruction */ 393 ecall_handler(); 394 break; 395 case SCAUSE_ILLEGAL_INSTRUCTION: 396 if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) { 397 /* 398 * May be a FPE trap. Enable FPE usage 399 * for this thread and try again. 400 */ 401 fpe_state_clear(); 402 frame->tf_sstatus &= ~SSTATUS_FS_MASK; 403 frame->tf_sstatus |= SSTATUS_FS_CLEAN; 404 pcb->pcb_fpflags |= PCB_FP_STARTED; 405 break; 406 } 407 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, 408 exception); 409 userret(td, frame); 410 break; 411 case SCAUSE_BREAKPOINT: 412 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc, 413 exception); 414 userret(td, frame); 415 break; 416 default: 417 dump_regs(frame); 418 panic("Unknown userland exception %lx, trap value %lx\n", 419 exception, frame->tf_stval); 420 } 421 } 422