1 /*- 2 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 3 * Copyright (C) 1995, 1996 TooLs GmbH. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by TooLs GmbH. 17 * 4. The name of TooLs GmbH may not be used to endorse or promote products 18 * derived from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $NetBSD: trap.c,v 1.58 2002/03/04 04:07:35 dbj Exp $ 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kdb.h> 39 #include <sys/proc.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/ptrace.h> 44 #include <sys/reboot.h> 45 #include <sys/syscall.h> 46 #include <sys/sysent.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/uio.h> 50 #include <sys/signalvar.h> 51 #include <sys/vmmeter.h> 52 53 #include <security/audit/audit.h> 54 55 #include <vm/vm.h> 56 #include <vm/pmap.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_param.h> 59 #include <vm/vm_kern.h> 60 #include <vm/vm_map.h> 61 #include <vm/vm_page.h> 62 63 #include <machine/_inttypes.h> 64 #include <machine/altivec.h> 65 #include <machine/cpu.h> 66 #include <machine/db_machdep.h> 67 #include <machine/fpu.h> 68 #include <machine/frame.h> 69 #include <machine/pcb.h> 70 #include <machine/psl.h> 71 #include <machine/slb.h> 72 #include <machine/spr.h> 73 #include <machine/sr.h> 74 #include <machine/trap.h> 75 76 /* Below matches setjmp.S */ 77 #define FAULTBUF_LR 21 78 #define FAULTBUF_R1 1 79 #define FAULTBUF_R2 2 80 #define FAULTBUF_CR 22 81 #define FAULTBUF_R14 3 82 83 #define MOREARGS(sp) ((caddr_t)((uintptr_t)(sp) + \ 84 sizeof(struct callframe) - 3*sizeof(register_t))) /* more args go here */ 85 86 static void trap_fatal(struct trapframe *frame); 87 static void printtrap(u_int vector, struct trapframe *frame, int isfatal, 88 int user); 89 static bool trap_pfault(struct trapframe *frame, bool user, int *signo, 90 int *ucode); 91 static int fix_unaligned(struct thread *td, struct trapframe *frame); 92 static int handle_onfault(struct trapframe *frame); 93 static void syscall(struct trapframe *frame); 94 95 #if defined(__powerpc64__) && defined(AIM) 96 static void normalize_inputs(void); 97 #endif 98 99 extern vm_offset_t __startkernel; 100 101 extern int copy_fault(void); 102 extern int fusufault(void); 103 104 #ifdef KDB 105 int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */ 106 #endif 107 108 struct powerpc_exception { 109 u_int vector; 110 char *name; 111 }; 112 113 #ifdef KDTRACE_HOOKS 114 #include <sys/dtrace_bsd.h> 115 116 int (*dtrace_invop_jump_addr)(struct trapframe *); 117 #endif 118 119 static struct powerpc_exception powerpc_exceptions[] = { 120 { EXC_CRIT, "critical input" }, 121 { EXC_RST, "system reset" }, 122 { EXC_MCHK, "machine check" }, 123 { EXC_DSI, "data storage interrupt" }, 124 { EXC_DSE, "data segment exception" }, 125 { EXC_ISI, "instruction storage interrupt" }, 126 { EXC_ISE, "instruction segment exception" }, 127 { EXC_EXI, "external interrupt" }, 128 { EXC_ALI, "alignment" }, 129 { EXC_PGM, "program" }, 130 { EXC_HEA, "hypervisor emulation assistance" }, 131 { EXC_FPU, "floating-point unavailable" }, 132 { EXC_APU, "auxiliary proc unavailable" }, 133 { EXC_DECR, "decrementer" }, 134 { EXC_FIT, "fixed-interval timer" }, 135 { EXC_WDOG, "watchdog timer" }, 136 { EXC_SC, "system call" }, 137 { EXC_TRC, "trace" }, 138 { EXC_FPA, "floating-point assist" }, 139 { EXC_DEBUG, "debug" }, 140 { EXC_PERF, "performance monitoring" }, 141 { EXC_VEC, "altivec unavailable" }, 142 { EXC_VSX, "vsx unavailable" }, 143 { EXC_FAC, "facility unavailable" }, 144 { EXC_ITMISS, "instruction tlb miss" }, 145 { EXC_DLMISS, "data load tlb miss" }, 146 { EXC_DSMISS, "data store tlb miss" }, 147 { EXC_BPT, "instruction breakpoint" }, 148 { EXC_SMI, "system management" }, 149 { EXC_VECAST_G4, "altivec assist" }, 150 { EXC_THRM, "thermal management" }, 151 { EXC_RUNMODETRC, "run mode/trace" }, 152 { EXC_SOFT_PATCH, "soft patch exception" }, 153 { EXC_LAST, NULL } 154 }; 155 156 static int uprintf_signal; 157 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN, 158 &uprintf_signal, 0, 159 "Print debugging information on trap signal to ctty"); 160 161 #define ESR_BITMASK \ 162 "\20" \ 163 "\040b0\037b1\036b2\035b3\034PIL\033PRR\032PTR\031FP" \ 164 "\030ST\027b9\026DLK\025ILK\024b12\023b13\022BO\021PIE" \ 165 "\020b16\017b17\016b18\015b19\014b20\013b21\012b22\011b23" \ 166 "\010SPE\007EPID\006b26\005b27\004b28\003b29\002b30\001b31" 167 #define MCSR_BITMASK \ 168 "\20" \ 169 "\040MCP\037ICERR\036DCERR\035TLBPERR\034L2MMU_MHIT\033b5\032b6\031b7" \ 170 "\030b8\027b9\026b10\025NMI\024MAV\023MEA\022b14\021IF" \ 171 "\020LD\017ST\016LDG\015b19\014b20\013b21\012b22\011b23" \ 172 "\010b24\007b25\006b26\005b27\004b28\003b29\002TLBSYNC\001BSL2_ERR" 173 #define MSSSR_BITMASK \ 174 "\20" \ 175 "\040b0\037b1\036b2\035b3\034b4\033b5\032b6\031b7" \ 176 "\030b8\027b9\026b10\025b11\024b12\023L2TAG\022L2DAT\021L3TAG" \ 177 "\020L3DAT\017APE\016DPE\015TEA\014b20\013b21\012b22\011b23" \ 178 "\010b24\007b25\006b26\005b27\004b28\003b29\002b30\001b31" 179 180 static const char * 181 trapname(u_int vector) 182 { 183 struct powerpc_exception *pe; 184 185 for (pe = powerpc_exceptions; pe->vector != EXC_LAST; pe++) { 186 if (pe->vector == vector) 187 return (pe->name); 188 } 189 190 return ("unknown"); 191 } 192 193 static inline bool 194 frame_is_trap_inst(struct trapframe *frame) 195 { 196 #ifdef AIM 197 return (frame->exc == EXC_PGM && frame->srr1 & EXC_PGM_TRAP); 198 #else 199 return ((frame->cpu.booke.esr & ESR_PTR) != 0); 200 #endif 201 } 202 203 void 204 trap(struct trapframe *frame) 205 { 206 struct thread *td; 207 struct proc *p; 208 #ifdef KDTRACE_HOOKS 209 uint32_t inst; 210 #endif 211 int sig, type, user; 212 u_int ucode; 213 ksiginfo_t ksi; 214 register_t addr, fscr; 215 216 VM_CNT_INC(v_trap); 217 218 #ifdef KDB 219 if (kdb_active) { 220 kdb_reenter(); 221 return; 222 } 223 #endif 224 225 td = curthread; 226 p = td->td_proc; 227 228 type = ucode = frame->exc; 229 sig = 0; 230 user = frame->srr1 & PSL_PR; 231 addr = 0; 232 233 CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name, 234 trapname(type), user ? "user" : "kernel"); 235 236 #ifdef KDTRACE_HOOKS 237 /* 238 * A trap can occur while DTrace executes a probe. Before 239 * executing the probe, DTrace blocks re-scheduling and sets 240 * a flag in its per-cpu flags to indicate that it doesn't 241 * want to fault. On returning from the probe, the no-fault 242 * flag is cleared and finally re-scheduling is enabled. 243 * 244 * If the DTrace kernel module has registered a trap handler, 245 * call it and if it returns non-zero, assume that it has 246 * handled the trap and modified the trap frame so that this 247 * function can return normally. 248 */ 249 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type) != 0) 250 return; 251 #endif 252 253 if (user) { 254 td->td_pticks = 0; 255 td->td_frame = frame; 256 addr = frame->srr0; 257 if (td->td_cowgen != atomic_load_int(&p->p_cowgen)) 258 thread_cow_update(td); 259 260 /* User Mode Traps */ 261 switch (type) { 262 case EXC_RUNMODETRC: 263 case EXC_TRC: 264 frame->srr1 &= ~PSL_SE; 265 sig = SIGTRAP; 266 ucode = TRAP_TRACE; 267 break; 268 269 #if defined(__powerpc64__) && defined(AIM) 270 case EXC_DSE: 271 addr = frame->dar; 272 /* FALLTHROUGH */ 273 case EXC_ISE: 274 /* DSE/ISE are automatically fatal with radix pmap. */ 275 if (radix_mmu || 276 handle_user_slb_spill(&p->p_vmspace->vm_pmap, 277 addr) != 0){ 278 sig = SIGSEGV; 279 ucode = SEGV_MAPERR; 280 } 281 break; 282 #endif 283 case EXC_DSI: 284 addr = frame->dar; 285 /* FALLTHROUGH */ 286 case EXC_ISI: 287 if (trap_pfault(frame, true, &sig, &ucode)) 288 sig = 0; 289 break; 290 291 case EXC_SC: 292 syscall(frame); 293 break; 294 295 case EXC_FPU: 296 KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU, 297 ("FPU already enabled for thread")); 298 enable_fpu(td); 299 break; 300 301 case EXC_VEC: 302 KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC, 303 ("Altivec already enabled for thread")); 304 enable_vec(td); 305 break; 306 307 case EXC_VSX: 308 KASSERT((td->td_pcb->pcb_flags & PCB_VSX) != PCB_VSX, 309 ("VSX already enabled for thread")); 310 if (!(td->td_pcb->pcb_flags & PCB_VEC)) 311 enable_vec(td); 312 if (td->td_pcb->pcb_flags & PCB_FPU) 313 save_fpu(td); 314 td->td_pcb->pcb_flags |= PCB_VSX; 315 enable_fpu(td); 316 break; 317 318 case EXC_FAC: 319 fscr = mfspr(SPR_FSCR); 320 switch (fscr & FSCR_IC_MASK) { 321 case FSCR_IC_HTM: 322 CTR0(KTR_TRAP, 323 "Hardware Transactional Memory subsystem disabled"); 324 sig = SIGILL; 325 ucode = ILL_ILLOPC; 326 break; 327 case FSCR_IC_DSCR: 328 td->td_pcb->pcb_flags |= PCB_CFSCR | PCB_CDSCR; 329 fscr |= FSCR_DSCR; 330 mtspr(SPR_DSCR, 0); 331 break; 332 case FSCR_IC_EBB: 333 td->td_pcb->pcb_flags |= PCB_CFSCR; 334 fscr |= FSCR_EBB; 335 mtspr(SPR_EBBHR, 0); 336 mtspr(SPR_EBBRR, 0); 337 mtspr(SPR_BESCR, 0); 338 break; 339 case FSCR_IC_TAR: 340 td->td_pcb->pcb_flags |= PCB_CFSCR; 341 fscr |= FSCR_TAR; 342 mtspr(SPR_TAR, 0); 343 break; 344 case FSCR_IC_LM: 345 td->td_pcb->pcb_flags |= PCB_CFSCR; 346 fscr |= FSCR_LM; 347 mtspr(SPR_LMRR, 0); 348 mtspr(SPR_LMSER, 0); 349 break; 350 default: 351 sig = SIGILL; 352 ucode = ILL_ILLOPC; 353 } 354 mtspr(SPR_FSCR, fscr & ~FSCR_IC_MASK); 355 break; 356 case EXC_HEA: 357 sig = SIGILL; 358 ucode = ILL_ILLOPC; 359 break; 360 361 case EXC_VECAST_E: 362 case EXC_VECAST_G4: 363 case EXC_VECAST_G5: 364 /* 365 * We get a VPU assist exception for IEEE mode 366 * vector operations on denormalized floats. 367 * Emulating this is a giant pain, so for now, 368 * just switch off IEEE mode and treat them as 369 * zero. 370 */ 371 372 save_vec(td); 373 td->td_pcb->pcb_vec.vscr |= ALTIVEC_VSCR_NJ; 374 enable_vec(td); 375 break; 376 377 case EXC_ALI: 378 if (fix_unaligned(td, frame) != 0) { 379 sig = SIGBUS; 380 ucode = BUS_ADRALN; 381 addr = frame->dar; 382 } 383 else 384 frame->srr0 += 4; 385 break; 386 387 case EXC_DEBUG: /* Single stepping */ 388 mtspr(SPR_DBSR, mfspr(SPR_DBSR)); 389 frame->srr1 &= ~PSL_DE; 390 frame->cpu.booke.dbcr0 &= ~(DBCR0_IDM | DBCR0_IC); 391 sig = SIGTRAP; 392 ucode = TRAP_TRACE; 393 break; 394 395 case EXC_PGM: 396 /* Identify the trap reason */ 397 if (frame_is_trap_inst(frame)) { 398 #ifdef KDTRACE_HOOKS 399 inst = fuword32((const void *)frame->srr0); 400 if (inst == 0x0FFFDDDD && 401 dtrace_pid_probe_ptr != NULL) { 402 (*dtrace_pid_probe_ptr)(frame); 403 break; 404 } 405 #endif 406 sig = SIGTRAP; 407 ucode = TRAP_BRKPT; 408 break; 409 } 410 411 if ((frame->srr1 & EXC_PGM_FPENABLED) && 412 (td->td_pcb->pcb_flags & PCB_FPU)) 413 sig = SIGFPE; 414 else 415 sig = ppc_instr_emulate(frame, td); 416 417 if (sig == SIGILL) { 418 if (frame->srr1 & EXC_PGM_PRIV) 419 ucode = ILL_PRVOPC; 420 else if (frame->srr1 & EXC_PGM_ILLEGAL) 421 ucode = ILL_ILLOPC; 422 } else if (sig == SIGFPE) { 423 ucode = get_fpu_exception(td); 424 } 425 426 break; 427 428 case EXC_MCHK: 429 sig = cpu_machine_check(td, frame, &ucode); 430 printtrap(frame->exc, frame, 0, (frame->srr1 & PSL_PR)); 431 break; 432 433 #if defined(__powerpc64__) && defined(AIM) 434 case EXC_SOFT_PATCH: 435 /* 436 * Point to the instruction that generated the exception to execute it again, 437 * and normalize the register values. 438 */ 439 frame->srr0 -= 4; 440 normalize_inputs(); 441 break; 442 #endif 443 444 default: 445 trap_fatal(frame); 446 } 447 } else { 448 /* Kernel Mode Traps */ 449 450 KASSERT(cold || td->td_ucred != NULL, 451 ("kernel trap doesn't have ucred")); 452 switch (type) { 453 case EXC_PGM: 454 #ifdef KDTRACE_HOOKS 455 if (frame_is_trap_inst(frame)) { 456 if (*(uint32_t *)frame->srr0 == EXC_DTRACE) { 457 if (dtrace_invop_jump_addr != NULL) { 458 dtrace_invop_jump_addr(frame); 459 return; 460 } 461 } 462 } 463 #endif 464 #ifdef KDB 465 if (db_trap_glue(frame)) 466 return; 467 #endif 468 break; 469 #if defined(__powerpc64__) && defined(AIM) 470 case EXC_DSE: 471 /* DSE on radix mmu is automatically fatal. */ 472 if (radix_mmu) 473 break; 474 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0 && 475 (frame->dar & SEGMENT_MASK) == USER_ADDR) { 476 __asm __volatile ("slbmte %0, %1" :: 477 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), 478 "r"(USER_SLB_SLBE)); 479 return; 480 } 481 break; 482 #endif 483 case EXC_DSI: 484 if (trap_pfault(frame, false, NULL, NULL)) 485 return; 486 break; 487 case EXC_MCHK: 488 if (handle_onfault(frame)) 489 return; 490 break; 491 default: 492 break; 493 } 494 trap_fatal(frame); 495 } 496 497 if (sig != 0) { 498 ksiginfo_init_trap(&ksi); 499 ksi.ksi_signo = sig; 500 ksi.ksi_code = (int) ucode; /* XXX, not POSIX */ 501 ksi.ksi_addr = (void *)addr; 502 ksi.ksi_trapno = type; 503 if (uprintf_signal) { 504 uprintf("pid %d comm %s: signal %d code %d type 0x%x " 505 "addr 0x%lx r1 0x%lx srr0 0x%lx srr1 0x%lx\n", 506 p->p_pid, p->p_comm, sig, ucode, type, 507 (u_long)addr, (u_long)frame->fixreg[1], 508 (u_long)frame->srr0, (u_long)frame->srr1); 509 } 510 511 trapsignal(td, &ksi); 512 } 513 514 userret(td, frame); 515 } 516 517 static void 518 trap_fatal(struct trapframe *frame) 519 { 520 #ifdef KDB 521 bool handled; 522 #endif 523 524 printtrap(frame->exc, frame, 1, (frame->srr1 & PSL_PR)); 525 #ifdef KDB 526 if (debugger_on_trap) { 527 kdb_why = KDB_WHY_TRAP; 528 handled = kdb_trap(frame->exc, 0, frame); 529 kdb_why = KDB_WHY_UNSET; 530 if (handled) 531 return; 532 } 533 #endif 534 panic("%s trap", trapname(frame->exc)); 535 } 536 537 static void 538 cpu_printtrap(u_int vector, struct trapframe *frame, int isfatal, int user) 539 { 540 #ifdef AIM 541 uint16_t ver; 542 543 switch (vector) { 544 case EXC_MCHK: 545 ver = mfpvr() >> 16; 546 if (MPC745X_P(ver)) 547 printf(" msssr0 = 0x%b\n", 548 (int)mfspr(SPR_MSSSR0), MSSSR_BITMASK); 549 case EXC_DSE: 550 case EXC_DSI: 551 case EXC_DTMISS: 552 printf(" dsisr = 0x%lx\n", 553 (u_long)frame->cpu.aim.dsisr); 554 break; 555 } 556 #elif defined(BOOKE) 557 vm_paddr_t pa; 558 559 switch (vector) { 560 case EXC_MCHK: 561 pa = mfspr(SPR_MCARU); 562 pa = (pa << 32) | (u_register_t)mfspr(SPR_MCAR); 563 printf(" mcsr = 0x%b\n", 564 (int)mfspr(SPR_MCSR), MCSR_BITMASK); 565 printf(" mcar = 0x%jx\n", (uintmax_t)pa); 566 } 567 printf(" esr = 0x%b\n", 568 (int)frame->cpu.booke.esr, ESR_BITMASK); 569 #endif 570 } 571 572 static void 573 printtrap(u_int vector, struct trapframe *frame, int isfatal, int user) 574 { 575 576 printf("\n"); 577 printf("%s %s trap:\n", isfatal ? "fatal" : "handled", 578 user ? "user" : "kernel"); 579 printf("\n"); 580 printf(" exception = 0x%x (%s)\n", vector, trapname(vector)); 581 switch (vector) { 582 case EXC_DSE: 583 case EXC_DSI: 584 case EXC_DTMISS: 585 case EXC_ALI: 586 case EXC_MCHK: 587 printf(" virtual address = 0x%" PRIxPTR "\n", frame->dar); 588 break; 589 case EXC_ISE: 590 case EXC_ISI: 591 case EXC_ITMISS: 592 printf(" virtual address = 0x%" PRIxPTR "\n", frame->srr0); 593 break; 594 } 595 cpu_printtrap(vector, frame, isfatal, user); 596 printf(" srr0 = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n", 597 frame->srr0, frame->srr0 - (register_t)(__startkernel - KERNBASE)); 598 printf(" srr1 = 0x%lx\n", (u_long)frame->srr1); 599 printf(" current msr = 0x%" PRIxPTR "\n", mfmsr()); 600 printf(" lr = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n", 601 frame->lr, frame->lr - (register_t)(__startkernel - KERNBASE)); 602 printf(" frame = %p\n", frame); 603 printf(" curthread = %p\n", curthread); 604 if (curthread != NULL) 605 printf(" pid = %d, comm = %s\n", 606 curthread->td_proc->p_pid, curthread->td_name); 607 printf("\n"); 608 } 609 610 /* 611 * Handles a fatal fault when we have onfault state to recover. Returns 612 * non-zero if there was onfault recovery state available. 613 */ 614 static int 615 handle_onfault(struct trapframe *frame) 616 { 617 struct thread *td; 618 jmp_buf *fb; 619 620 td = curthread; 621 #if defined(__powerpc64__) || defined(BOOKE) 622 uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault; 623 624 if (dispatch == 0) 625 return (0); 626 /* Short-circuit radix and Book-E paths. */ 627 switch (dispatch) { 628 case COPYFAULT: 629 frame->srr0 = (uintptr_t)copy_fault; 630 return (1); 631 case FUSUFAULT: 632 frame->srr0 = (uintptr_t)fusufault; 633 return (1); 634 default: 635 break; 636 } 637 #endif 638 fb = td->td_pcb->pcb_onfault; 639 if (fb != NULL) { 640 frame->srr0 = (*fb)->_jb[FAULTBUF_LR]; 641 frame->fixreg[1] = (*fb)->_jb[FAULTBUF_R1]; 642 frame->fixreg[2] = (*fb)->_jb[FAULTBUF_R2]; 643 frame->fixreg[3] = 1; 644 frame->cr = (*fb)->_jb[FAULTBUF_CR]; 645 bcopy(&(*fb)->_jb[FAULTBUF_R14], &frame->fixreg[14], 646 18 * sizeof(register_t)); 647 td->td_pcb->pcb_onfault = NULL; /* Returns twice, not thrice */ 648 return (1); 649 } 650 return (0); 651 } 652 653 int 654 cpu_fetch_syscall_args(struct thread *td) 655 { 656 struct proc *p; 657 struct trapframe *frame; 658 struct syscall_args *sa; 659 caddr_t params; 660 size_t argsz; 661 int error, n, narg, i; 662 663 p = td->td_proc; 664 frame = td->td_frame; 665 sa = &td->td_sa; 666 667 sa->code = frame->fixreg[0]; 668 sa->original_code = sa->code; 669 params = (caddr_t)(frame->fixreg + FIRSTARG); 670 n = NARGREG; 671 672 if (sa->code == SYS_syscall) { 673 /* 674 * code is first argument, 675 * followed by actual args. 676 */ 677 sa->code = *(register_t *) params; 678 params += sizeof(register_t); 679 n -= 1; 680 } else if (sa->code == SYS___syscall) { 681 /* 682 * Like syscall, but code is a quad, 683 * so as to maintain quad alignment 684 * for the rest of the args. 685 */ 686 if (SV_PROC_FLAG(p, SV_ILP32)) { 687 params += sizeof(register_t); 688 sa->code = *(register_t *) params; 689 params += sizeof(register_t); 690 n -= 2; 691 } else { 692 sa->code = *(register_t *) params; 693 params += sizeof(register_t); 694 n -= 1; 695 } 696 } 697 698 if (sa->code >= p->p_sysent->sv_size) 699 sa->callp = &p->p_sysent->sv_table[0]; 700 else 701 sa->callp = &p->p_sysent->sv_table[sa->code]; 702 703 narg = sa->callp->sy_narg; 704 705 if (SV_PROC_FLAG(p, SV_ILP32)) { 706 argsz = sizeof(uint32_t); 707 708 for (i = 0; i < n; i++) 709 sa->args[i] = ((u_register_t *)(params))[i] & 710 0xffffffff; 711 } else { 712 argsz = sizeof(uint64_t); 713 714 for (i = 0; i < n; i++) 715 sa->args[i] = ((u_register_t *)(params))[i]; 716 } 717 718 if (narg > n) 719 error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n, 720 (narg - n) * argsz); 721 else 722 error = 0; 723 724 #ifdef __powerpc64__ 725 if (SV_PROC_FLAG(p, SV_ILP32) && narg > n) { 726 /* Expand the size of arguments copied from the stack */ 727 728 for (i = narg; i >= n; i--) 729 sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n]; 730 } 731 #endif 732 733 if (error == 0) { 734 td->td_retval[0] = 0; 735 td->td_retval[1] = frame->fixreg[FIRSTARG + 1]; 736 } 737 return (error); 738 } 739 740 #include "../../kern/subr_syscall.c" 741 742 void 743 syscall(struct trapframe *frame) 744 { 745 struct thread *td; 746 747 td = curthread; 748 td->td_frame = frame; 749 750 #if defined(__powerpc64__) && defined(AIM) 751 /* 752 * Speculatively restore last user SLB segment, which we know is 753 * invalid already, since we are likely to do copyin()/copyout(). 754 */ 755 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0) 756 __asm __volatile ("slbmte %0, %1; isync" :: 757 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE)); 758 #endif 759 760 syscallenter(td); 761 syscallret(td); 762 } 763 764 static bool 765 trap_pfault(struct trapframe *frame, bool user, int *signo, int *ucode) 766 { 767 vm_offset_t eva; 768 struct thread *td; 769 struct proc *p; 770 vm_map_t map; 771 vm_prot_t ftype; 772 int rv, is_user; 773 774 td = curthread; 775 p = td->td_proc; 776 if (frame->exc == EXC_ISI) { 777 eva = frame->srr0; 778 ftype = VM_PROT_EXECUTE; 779 if (frame->srr1 & SRR1_ISI_PFAULT) 780 ftype |= VM_PROT_READ; 781 } else { 782 eva = frame->dar; 783 #ifdef BOOKE 784 if (frame->cpu.booke.esr & ESR_ST) 785 #else 786 if (frame->cpu.aim.dsisr & DSISR_STORE) 787 #endif 788 ftype = VM_PROT_WRITE; 789 else 790 ftype = VM_PROT_READ; 791 } 792 #if defined(__powerpc64__) && defined(AIM) 793 if (radix_mmu && pmap_nofault(&p->p_vmspace->vm_pmap, eva, ftype) == 0) 794 return (true); 795 #endif 796 797 if (__predict_false((td->td_pflags & TDP_NOFAULTING) == 0)) { 798 /* 799 * If we get a page fault while in a critical section, then 800 * it is most likely a fatal kernel page fault. The kernel 801 * is already going to panic trying to get a sleep lock to 802 * do the VM lookup, so just consider it a fatal trap so the 803 * kernel can print out a useful trap message and even get 804 * to the debugger. 805 * 806 * If we get a page fault while holding a non-sleepable 807 * lock, then it is most likely a fatal kernel page fault. 808 * If WITNESS is enabled, then it's going to whine about 809 * bogus LORs with various VM locks, so just skip to the 810 * fatal trap handling directly. 811 */ 812 if (td->td_critnest != 0 || 813 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 814 "Kernel page fault") != 0) { 815 trap_fatal(frame); 816 return (false); 817 } 818 } 819 if (user) { 820 KASSERT(p->p_vmspace != NULL, ("trap_pfault: vmspace NULL")); 821 map = &p->p_vmspace->vm_map; 822 } else { 823 rv = pmap_decode_kernel_ptr(eva, &is_user, &eva); 824 if (rv != 0) 825 return (false); 826 827 if (is_user) 828 map = &p->p_vmspace->vm_map; 829 else 830 map = kernel_map; 831 } 832 833 /* Fault in the page. */ 834 rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode); 835 /* 836 * XXXDTRACE: add dtrace_doubletrap_func here? 837 */ 838 839 if (rv == KERN_SUCCESS) 840 return (true); 841 842 if (!user && handle_onfault(frame)) 843 return (true); 844 845 return (false); 846 } 847 848 /* 849 * For now, this only deals with the particular unaligned access case 850 * that gcc tends to generate. Eventually it should handle all of the 851 * possibilities that can happen on a 32-bit PowerPC in big-endian mode. 852 */ 853 854 static int 855 fix_unaligned(struct thread *td, struct trapframe *frame) 856 { 857 struct thread *fputhread; 858 #ifdef BOOKE 859 uint32_t inst; 860 #endif 861 int indicator, reg; 862 double *fpr; 863 864 #ifdef __SPE__ 865 indicator = (frame->cpu.booke.esr & (ESR_ST|ESR_SPE)); 866 if (indicator & ESR_SPE) { 867 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0) 868 return (-1); 869 reg = EXC_ALI_INST_RST(inst); 870 fpr = (double *)td->td_pcb->pcb_vec.vr[reg]; 871 fputhread = PCPU_GET(vecthread); 872 873 /* Juggle the SPE to ensure that we've initialized 874 * the registers, and that their current state is in 875 * the PCB. 876 */ 877 if (fputhread != td) { 878 if (fputhread) 879 save_vec(fputhread); 880 enable_vec(td); 881 } 882 save_vec(td); 883 884 if (!(indicator & ESR_ST)) { 885 if (copyin((void *)frame->dar, fpr, 886 sizeof(double)) != 0) 887 return (-1); 888 frame->fixreg[reg] = td->td_pcb->pcb_vec.vr[reg][1]; 889 enable_vec(td); 890 } else { 891 td->td_pcb->pcb_vec.vr[reg][1] = frame->fixreg[reg]; 892 if (copyout(fpr, (void *)frame->dar, 893 sizeof(double)) != 0) 894 return (-1); 895 } 896 return (0); 897 } 898 #else 899 #ifdef BOOKE 900 indicator = (frame->cpu.booke.esr & ESR_ST) ? EXC_ALI_STFD : EXC_ALI_LFD; 901 #else 902 indicator = EXC_ALI_OPCODE_INDICATOR(frame->cpu.aim.dsisr); 903 #endif 904 905 switch (indicator) { 906 case EXC_ALI_LFD: 907 case EXC_ALI_STFD: 908 #ifdef BOOKE 909 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0) 910 return (-1); 911 reg = EXC_ALI_INST_RST(inst); 912 #else 913 reg = EXC_ALI_RST(frame->cpu.aim.dsisr); 914 #endif 915 fpr = &td->td_pcb->pcb_fpu.fpr[reg].fpr; 916 fputhread = PCPU_GET(fputhread); 917 918 /* Juggle the FPU to ensure that we've initialized 919 * the FPRs, and that their current state is in 920 * the PCB. 921 */ 922 if (fputhread != td) { 923 if (fputhread) 924 save_fpu(fputhread); 925 enable_fpu(td); 926 } 927 save_fpu(td); 928 929 if (indicator == EXC_ALI_LFD) { 930 if (copyin((void *)frame->dar, fpr, 931 sizeof(double)) != 0) 932 return (-1); 933 enable_fpu(td); 934 } else { 935 if (copyout(fpr, (void *)frame->dar, 936 sizeof(double)) != 0) 937 return (-1); 938 } 939 return (0); 940 break; 941 } 942 #endif 943 944 return (-1); 945 } 946 947 #if defined(__powerpc64__) && defined(AIM) 948 #define MSKNSHL(x, m, n) "(((" #x ") & " #m ") << " #n ")" 949 #define MSKNSHR(x, m, n) "(((" #x ") & " #m ") >> " #n ")" 950 951 /* xvcpsgndp instruction, built in opcode format. 952 * This can be changed to use mnemonic after a toolchain update. 953 */ 954 #define XVCPSGNDP(xt, xa, xb) \ 955 __asm __volatile(".long (" \ 956 MSKNSHL(60, 0x3f, 26) " | " \ 957 MSKNSHL(xt, 0x1f, 21) " | " \ 958 MSKNSHL(xa, 0x1f, 16) " | " \ 959 MSKNSHL(xb, 0x1f, 11) " | " \ 960 MSKNSHL(240, 0xff, 3) " | " \ 961 MSKNSHR(xa, 0x20, 3) " | " \ 962 MSKNSHR(xa, 0x20, 4) " | " \ 963 MSKNSHR(xa, 0x20, 5) ")") 964 965 /* Macros to normalize 1 or 10 VSX registers */ 966 #define NORM(x) XVCPSGNDP(x, x, x) 967 #define NORM10(x) \ 968 NORM(x ## 0); NORM(x ## 1); NORM(x ## 2); NORM(x ## 3); NORM(x ## 4); \ 969 NORM(x ## 5); NORM(x ## 6); NORM(x ## 7); NORM(x ## 8); NORM(x ## 9) 970 971 static void 972 normalize_inputs(void) 973 { 974 register_t msr; 975 976 /* enable VSX */ 977 msr = mfmsr(); 978 mtmsr(msr | PSL_VSX); 979 980 NORM(0); NORM(1); NORM(2); NORM(3); NORM(4); 981 NORM(5); NORM(6); NORM(7); NORM(8); NORM(9); 982 NORM10(1); NORM10(2); NORM10(3); NORM10(4); NORM10(5); 983 NORM(60); NORM(61); NORM(62); NORM(63); 984 985 /* restore MSR */ 986 mtmsr(msr); 987 } 988 #endif 989 990 #ifdef KDB 991 int 992 db_trap_glue(struct trapframe *frame) 993 { 994 995 if (!(frame->srr1 & PSL_PR) 996 && (frame->exc == EXC_TRC || frame->exc == EXC_RUNMODETRC 997 || frame_is_trap_inst(frame) 998 || frame->exc == EXC_BPT 999 || frame->exc == EXC_DEBUG 1000 || frame->exc == EXC_DSI)) { 1001 int type = frame->exc; 1002 1003 /* Ignore DTrace traps. */ 1004 if (*(uint32_t *)frame->srr0 == EXC_DTRACE) 1005 return (0); 1006 if (frame_is_trap_inst(frame)) { 1007 type = T_BREAKPOINT; 1008 } 1009 return (kdb_trap(type, 0, frame)); 1010 } 1011 1012 return (0); 1013 } 1014 #endif 1015