1 /*- 2 * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/bus.h> 45 #include <sys/proc.h> 46 #include <sys/ptrace.h> 47 #include <sys/syscall.h> 48 #include <sys/sysent.h> 49 #ifdef KDB 50 #include <sys/kdb.h> 51 #endif 52 53 #include <vm/vm.h> 54 #include <vm/pmap.h> 55 #include <vm/vm_kern.h> 56 #include <vm/vm_map.h> 57 #include <vm/vm_param.h> 58 #include <vm/vm_extern.h> 59 60 #ifdef FPE 61 #include <machine/fpe.h> 62 #endif 63 #include <machine/frame.h> 64 #include <machine/pcb.h> 65 #include <machine/pcpu.h> 66 67 #include <machine/resource.h> 68 #include <machine/intr.h> 69 70 #ifdef KDTRACE_HOOKS 71 #include <sys/dtrace_bsd.h> 72 #endif 73 74 int (*dtrace_invop_jump_addr)(struct trapframe *); 75 76 /* Called from exception.S */ 77 void do_trap_supervisor(struct trapframe *); 78 void do_trap_user(struct trapframe *); 79 80 static __inline void 81 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno) 82 { 83 ksiginfo_t ksi; 84 85 ksiginfo_init_trap(&ksi); 86 ksi.ksi_signo = sig; 87 ksi.ksi_code = code; 88 ksi.ksi_addr = addr; 89 ksi.ksi_trapno = trapno; 90 trapsignal(td, &ksi); 91 } 92 93 int 94 cpu_fetch_syscall_args(struct thread *td) 95 { 96 struct proc *p; 97 syscallarg_t *ap, *dst_ap; 98 struct syscall_args *sa; 99 100 p = td->td_proc; 101 sa = &td->td_sa; 102 ap = &td->td_frame->tf_a[0]; 103 dst_ap = &sa->args[0]; 104 105 sa->code = td->td_frame->tf_t[0]; 106 sa->original_code = sa->code; 107 108 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) { 109 sa->code = *ap++; 110 } else { 111 *dst_ap++ = *ap++; 112 } 113 114 if (__predict_false(sa->code >= p->p_sysent->sv_size)) 115 sa->callp = &p->p_sysent->sv_table[0]; 116 else 117 sa->callp = &p->p_sysent->sv_table[sa->code]; 118 119 KASSERT(sa->callp->sy_narg <= nitems(sa->args), 120 ("Syscall %d takes too many arguments", sa->code)); 121 122 memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap)); 123 124 td->td_retval[0] = 0; 125 td->td_retval[1] = 0; 126 127 return (0); 128 } 129 130 #include "../../kern/subr_syscall.c" 131 132 static void 133 dump_regs(struct trapframe *frame) 134 { 135 int n; 136 int i; 137 138 n = nitems(frame->tf_t); 139 for (i = 0; i < n; i++) 140 printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]); 141 142 n = nitems(frame->tf_s); 143 for (i = 0; i < n; i++) 144 printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]); 145 146 n = nitems(frame->tf_a); 147 for (i = 0; i < n; i++) 148 printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]); 149 150 printf("ra == 0x%016lx\n", frame->tf_ra); 151 printf("sp == 0x%016lx\n", frame->tf_sp); 152 printf("gp == 0x%016lx\n", frame->tf_gp); 153 printf("tp == 0x%016lx\n", frame->tf_tp); 154 155 printf("sepc == 0x%016lx\n", frame->tf_sepc); 156 printf("sstatus == 0x%016lx\n", frame->tf_sstatus); 157 } 158 159 static void 160 ecall_handler(void) 161 { 162 struct thread *td; 163 164 td = curthread; 165 166 syscallenter(td); 167 syscallret(td); 168 } 169 170 static void 171 page_fault_handler(struct trapframe *frame, int usermode) 172 { 173 struct vm_map *map; 174 uint64_t stval; 175 struct thread *td; 176 struct pcb *pcb; 177 vm_prot_t ftype; 178 vm_offset_t va; 179 struct proc *p; 180 int error, sig, ucode; 181 #ifdef KDB 182 bool handled; 183 #endif 184 185 #ifdef KDB 186 if (kdb_active) { 187 kdb_reenter(); 188 return; 189 } 190 #endif 191 192 td = curthread; 193 p = td->td_proc; 194 pcb = td->td_pcb; 195 stval = frame->tf_stval; 196 197 if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 || 198 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 199 "Kernel page fault") != 0) 200 goto fatal; 201 202 if (usermode) { 203 if (!VIRT_IS_VALID(stval)) { 204 call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval, 205 frame->tf_scause & SCAUSE_CODE); 206 goto done; 207 } 208 map = &p->p_vmspace->vm_map; 209 } else { 210 /* 211 * Enable interrupts for the duration of the page fault. For 212 * user faults this was done already in do_trap_user(). 213 */ 214 intr_enable(); 215 216 if (stval >= VM_MIN_KERNEL_ADDRESS) { 217 map = kernel_map; 218 } else { 219 if (pcb->pcb_onfault == 0) 220 goto fatal; 221 map = &p->p_vmspace->vm_map; 222 } 223 } 224 225 va = trunc_page(stval); 226 227 if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) { 228 ftype = VM_PROT_WRITE; 229 } else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) { 230 ftype = VM_PROT_EXECUTE; 231 } else { 232 ftype = VM_PROT_READ; 233 } 234 235 if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype)) 236 goto done; 237 238 error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode); 239 if (error != KERN_SUCCESS) { 240 if (usermode) { 241 call_trapsignal(td, sig, ucode, (void *)stval, 242 frame->tf_scause & SCAUSE_CODE); 243 } else { 244 if (pcb->pcb_onfault != 0) { 245 frame->tf_a[0] = error; 246 frame->tf_sepc = pcb->pcb_onfault; 247 return; 248 } 249 goto fatal; 250 } 251 } 252 253 done: 254 if (usermode) 255 userret(td, frame); 256 return; 257 258 fatal: 259 dump_regs(frame); 260 #ifdef KDB 261 if (debugger_on_trap) { 262 kdb_why = KDB_WHY_TRAP; 263 handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame); 264 kdb_why = KDB_WHY_UNSET; 265 if (handled) 266 return; 267 } 268 #endif 269 panic("Fatal page fault at %#lx: %#016lx", frame->tf_sepc, stval); 270 } 271 272 void 273 do_trap_supervisor(struct trapframe *frame) 274 { 275 uint64_t exception; 276 277 /* Ensure we came from supervisor mode, interrupts disabled */ 278 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 279 SSTATUS_SPP, ("Came from S mode with interrupts enabled")); 280 281 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 282 ("Came from S mode with SUM enabled")); 283 284 exception = frame->tf_scause & SCAUSE_CODE; 285 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 286 /* Interrupt */ 287 riscv_cpu_intr(frame); 288 return; 289 } 290 291 #ifdef KDTRACE_HOOKS 292 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 293 return; 294 #endif 295 296 CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p", 297 curthread, frame->tf_sepc, frame); 298 299 switch (exception) { 300 case SCAUSE_LOAD_ACCESS_FAULT: 301 case SCAUSE_STORE_ACCESS_FAULT: 302 case SCAUSE_INST_ACCESS_FAULT: 303 dump_regs(frame); 304 panic("Memory access exception at 0x%016lx\n", frame->tf_sepc); 305 break; 306 case SCAUSE_LOAD_MISALIGNED: 307 case SCAUSE_STORE_MISALIGNED: 308 case SCAUSE_INST_MISALIGNED: 309 dump_regs(frame); 310 panic("Misaligned address exception at %#016lx: %#016lx\n", 311 frame->tf_sepc, frame->tf_stval); 312 break; 313 case SCAUSE_STORE_PAGE_FAULT: 314 case SCAUSE_LOAD_PAGE_FAULT: 315 case SCAUSE_INST_PAGE_FAULT: 316 page_fault_handler(frame, 0); 317 break; 318 case SCAUSE_BREAKPOINT: 319 #ifdef KDTRACE_HOOKS 320 if (dtrace_invop_jump_addr != NULL && 321 dtrace_invop_jump_addr(frame) == 0) 322 break; 323 #endif 324 #ifdef KDB 325 kdb_trap(exception, 0, frame); 326 #else 327 dump_regs(frame); 328 panic("No debugger in kernel.\n"); 329 #endif 330 break; 331 case SCAUSE_ILLEGAL_INSTRUCTION: 332 dump_regs(frame); 333 panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc); 334 break; 335 default: 336 dump_regs(frame); 337 panic("Unknown kernel exception %lx trap value %lx\n", 338 exception, frame->tf_stval); 339 } 340 } 341 342 void 343 do_trap_user(struct trapframe *frame) 344 { 345 uint64_t exception; 346 struct thread *td; 347 struct pcb *pcb; 348 349 td = curthread; 350 pcb = td->td_pcb; 351 352 KASSERT(td->td_frame == frame, 353 ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame)); 354 355 /* Ensure we came from usermode, interrupts disabled */ 356 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0, 357 ("Came from U mode with interrupts enabled")); 358 359 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 360 ("Came from U mode with SUM enabled")); 361 362 exception = frame->tf_scause & SCAUSE_CODE; 363 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 364 /* Interrupt */ 365 riscv_cpu_intr(frame); 366 return; 367 } 368 intr_enable(); 369 370 CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p", 371 curthread, frame->tf_sepc, frame); 372 373 switch (exception) { 374 case SCAUSE_LOAD_ACCESS_FAULT: 375 case SCAUSE_STORE_ACCESS_FAULT: 376 case SCAUSE_INST_ACCESS_FAULT: 377 call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc, 378 exception); 379 userret(td, frame); 380 break; 381 case SCAUSE_LOAD_MISALIGNED: 382 case SCAUSE_STORE_MISALIGNED: 383 case SCAUSE_INST_MISALIGNED: 384 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sepc, 385 exception); 386 userret(td, frame); 387 break; 388 case SCAUSE_STORE_PAGE_FAULT: 389 case SCAUSE_LOAD_PAGE_FAULT: 390 case SCAUSE_INST_PAGE_FAULT: 391 page_fault_handler(frame, 1); 392 break; 393 case SCAUSE_ECALL_USER: 394 frame->tf_sepc += 4; /* Next instruction */ 395 ecall_handler(); 396 break; 397 case SCAUSE_ILLEGAL_INSTRUCTION: 398 #ifdef FPE 399 if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) { 400 /* 401 * May be a FPE trap. Enable FPE usage 402 * for this thread and try again. 403 */ 404 fpe_state_clear(); 405 frame->tf_sstatus &= ~SSTATUS_FS_MASK; 406 frame->tf_sstatus |= SSTATUS_FS_CLEAN; 407 pcb->pcb_fpflags |= PCB_FP_STARTED; 408 break; 409 } 410 #endif 411 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, 412 exception); 413 userret(td, frame); 414 break; 415 case SCAUSE_BREAKPOINT: 416 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc, 417 exception); 418 userret(td, frame); 419 break; 420 default: 421 dump_regs(frame); 422 panic("Unknown userland exception %lx, trap value %lx\n", 423 exception, frame->tf_stval); 424 } 425 } 426