1 /*- 2 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 3 * Copyright (C) 1995, 1996 TooLs GmbH. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by TooLs GmbH. 17 * 4. The name of TooLs GmbH may not be used to endorse or promote products 18 * derived from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $NetBSD: trap.c,v 1.58 2002/03/04 04:07:35 dbj Exp $ 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kdb.h> 39 #include <sys/proc.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/ptrace.h> 44 #include <sys/reboot.h> 45 #include <sys/syscall.h> 46 #include <sys/sysent.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/uio.h> 50 #include <sys/signalvar.h> 51 #include <sys/vmmeter.h> 52 53 #include <security/audit/audit.h> 54 55 #include <vm/vm.h> 56 #include <vm/pmap.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_param.h> 59 #include <vm/vm_kern.h> 60 #include <vm/vm_map.h> 61 #include <vm/vm_page.h> 62 63 #include <machine/_inttypes.h> 64 #include <machine/altivec.h> 65 #include <machine/cpu.h> 66 #include <machine/db_machdep.h> 67 #include <machine/fpu.h> 68 #include <machine/frame.h> 69 #include <machine/pcb.h> 70 #include <machine/psl.h> 71 #include <machine/slb.h> 72 #include <machine/spr.h> 73 #include <machine/sr.h> 74 #include <machine/trap.h> 75 76 /* Below matches setjmp.S */ 77 #define FAULTBUF_LR 21 78 #define FAULTBUF_R1 1 79 #define FAULTBUF_R2 2 80 #define FAULTBUF_CR 22 81 #define FAULTBUF_R14 3 82 83 #define MOREARGS(sp) ((caddr_t)((uintptr_t)(sp) + \ 84 sizeof(struct callframe) - 3*sizeof(register_t))) /* more args go here */ 85 86 static void trap_fatal(struct trapframe *frame); 87 static void printtrap(u_int vector, struct trapframe *frame, int isfatal, 88 int user); 89 static bool trap_pfault(struct trapframe *frame, bool user, int *signo, 90 int *ucode); 91 static int fix_unaligned(struct thread *td, struct trapframe *frame); 92 static int handle_onfault(struct trapframe *frame); 93 static void syscall(struct trapframe *frame); 94 95 #if defined(__powerpc64__) && defined(AIM) 96 static void normalize_inputs(void); 97 #endif 98 99 extern vm_offset_t __startkernel; 100 101 extern int copy_fault(void); 102 extern int fusufault(void); 103 104 #ifdef KDB 105 int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */ 106 #endif 107 108 struct powerpc_exception { 109 u_int vector; 110 char *name; 111 }; 112 113 #ifdef KDTRACE_HOOKS 114 #include <sys/dtrace_bsd.h> 115 116 int (*dtrace_invop_jump_addr)(struct trapframe *); 117 #endif 118 119 static struct powerpc_exception powerpc_exceptions[] = { 120 { EXC_CRIT, "critical input" }, 121 { EXC_RST, "system reset" }, 122 { EXC_MCHK, "machine check" }, 123 { EXC_DSI, "data storage interrupt" }, 124 { EXC_DSE, "data segment exception" }, 125 { EXC_ISI, "instruction storage interrupt" }, 126 { EXC_ISE, "instruction segment exception" }, 127 { EXC_EXI, "external interrupt" }, 128 { EXC_ALI, "alignment" }, 129 { EXC_PGM, "program" }, 130 { EXC_HEA, "hypervisor emulation assistance" }, 131 { EXC_FPU, "floating-point unavailable" }, 132 { EXC_APU, "auxiliary proc unavailable" }, 133 { EXC_DECR, "decrementer" }, 134 { EXC_FIT, "fixed-interval timer" }, 135 { EXC_WDOG, "watchdog timer" }, 136 { EXC_SC, "system call" }, 137 { EXC_TRC, "trace" }, 138 { EXC_FPA, "floating-point assist" }, 139 { EXC_DEBUG, "debug" }, 140 { EXC_PERF, "performance monitoring" }, 141 { EXC_VEC, "altivec unavailable" }, 142 { EXC_VSX, "vsx unavailable" }, 143 { EXC_FAC, "facility unavailable" }, 144 { EXC_ITMISS, "instruction tlb miss" }, 145 { EXC_DLMISS, "data load tlb miss" }, 146 { EXC_DSMISS, "data store tlb miss" }, 147 { EXC_BPT, "instruction breakpoint" }, 148 { EXC_SMI, "system management" }, 149 { EXC_VECAST_G4, "altivec assist" }, 150 { EXC_THRM, "thermal management" }, 151 { EXC_RUNMODETRC, "run mode/trace" }, 152 { EXC_SOFT_PATCH, "soft patch exception" }, 153 { EXC_LAST, NULL } 154 }; 155 156 #define ESR_BITMASK \ 157 "\20" \ 158 "\040b0\037b1\036b2\035b3\034PIL\033PRR\032PTR\031FP" \ 159 "\030ST\027b9\026DLK\025ILK\024b12\023b13\022BO\021PIE" \ 160 "\020b16\017b17\016b18\015b19\014b20\013b21\012b22\011b23" \ 161 "\010SPE\007EPID\006b26\005b27\004b28\003b29\002b30\001b31" 162 #define MCSR_BITMASK \ 163 "\20" \ 164 "\040MCP\037ICERR\036DCERR\035TLBPERR\034L2MMU_MHIT\033b5\032b6\031b7" \ 165 "\030b8\027b9\026b10\025NMI\024MAV\023MEA\022b14\021IF" \ 166 "\020LD\017ST\016LDG\015b19\014b20\013b21\012b22\011b23" \ 167 "\010b24\007b25\006b26\005b27\004b28\003b29\002TLBSYNC\001BSL2_ERR" 168 #define MSSSR_BITMASK \ 169 "\20" \ 170 "\040b0\037b1\036b2\035b3\034b4\033b5\032b6\031b7" \ 171 "\030b8\027b9\026b10\025b11\024b12\023L2TAG\022L2DAT\021L3TAG" \ 172 "\020L3DAT\017APE\016DPE\015TEA\014b20\013b21\012b22\011b23" \ 173 "\010b24\007b25\006b26\005b27\004b28\003b29\002b30\001b31" 174 175 static const char * 176 trapname(u_int vector) 177 { 178 struct powerpc_exception *pe; 179 180 for (pe = powerpc_exceptions; pe->vector != EXC_LAST; pe++) { 181 if (pe->vector == vector) 182 return (pe->name); 183 } 184 185 return ("unknown"); 186 } 187 188 static inline bool 189 frame_is_trap_inst(struct trapframe *frame) 190 { 191 #ifdef AIM 192 return (frame->exc == EXC_PGM && frame->srr1 & EXC_PGM_TRAP); 193 #else 194 return ((frame->cpu.booke.esr & ESR_PTR) != 0); 195 #endif 196 } 197 198 void 199 trap(struct trapframe *frame) 200 { 201 struct thread *td; 202 struct proc *p; 203 #ifdef KDTRACE_HOOKS 204 uint32_t inst; 205 #endif 206 int sig, type, user; 207 u_int ucode; 208 ksiginfo_t ksi; 209 register_t addr, fscr; 210 211 VM_CNT_INC(v_trap); 212 213 #ifdef KDB 214 if (kdb_active) { 215 kdb_reenter(); 216 return; 217 } 218 #endif 219 220 td = curthread; 221 p = td->td_proc; 222 223 type = ucode = frame->exc; 224 sig = 0; 225 user = frame->srr1 & PSL_PR; 226 addr = 0; 227 228 CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name, 229 trapname(type), user ? "user" : "kernel"); 230 231 #ifdef KDTRACE_HOOKS 232 /* 233 * A trap can occur while DTrace executes a probe. Before 234 * executing the probe, DTrace blocks re-scheduling and sets 235 * a flag in its per-cpu flags to indicate that it doesn't 236 * want to fault. On returning from the probe, the no-fault 237 * flag is cleared and finally re-scheduling is enabled. 238 * 239 * If the DTrace kernel module has registered a trap handler, 240 * call it and if it returns non-zero, assume that it has 241 * handled the trap and modified the trap frame so that this 242 * function can return normally. 243 */ 244 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type) != 0) 245 return; 246 #endif 247 248 if (user) { 249 td->td_pticks = 0; 250 td->td_frame = frame; 251 addr = frame->srr0; 252 if (td->td_cowgen != p->p_cowgen) 253 thread_cow_update(td); 254 255 /* User Mode Traps */ 256 switch (type) { 257 case EXC_RUNMODETRC: 258 case EXC_TRC: 259 frame->srr1 &= ~PSL_SE; 260 sig = SIGTRAP; 261 ucode = TRAP_TRACE; 262 break; 263 264 #if defined(__powerpc64__) && defined(AIM) 265 case EXC_DSE: 266 addr = frame->dar; 267 /* FALLTHROUGH */ 268 case EXC_ISE: 269 /* DSE/ISE are automatically fatal with radix pmap. */ 270 if (radix_mmu || 271 handle_user_slb_spill(&p->p_vmspace->vm_pmap, 272 addr) != 0){ 273 sig = SIGSEGV; 274 ucode = SEGV_MAPERR; 275 } 276 break; 277 #endif 278 case EXC_DSI: 279 addr = frame->dar; 280 /* FALLTHROUGH */ 281 case EXC_ISI: 282 if (trap_pfault(frame, true, &sig, &ucode)) 283 sig = 0; 284 break; 285 286 case EXC_SC: 287 syscall(frame); 288 break; 289 290 case EXC_FPU: 291 KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU, 292 ("FPU already enabled for thread")); 293 enable_fpu(td); 294 break; 295 296 case EXC_VEC: 297 KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC, 298 ("Altivec already enabled for thread")); 299 enable_vec(td); 300 break; 301 302 case EXC_VSX: 303 KASSERT((td->td_pcb->pcb_flags & PCB_VSX) != PCB_VSX, 304 ("VSX already enabled for thread")); 305 if (!(td->td_pcb->pcb_flags & PCB_VEC)) 306 enable_vec(td); 307 if (td->td_pcb->pcb_flags & PCB_FPU) 308 save_fpu(td); 309 td->td_pcb->pcb_flags |= PCB_VSX; 310 enable_fpu(td); 311 break; 312 313 case EXC_FAC: 314 fscr = mfspr(SPR_FSCR); 315 switch (fscr & FSCR_IC_MASK) { 316 case FSCR_IC_HTM: 317 CTR0(KTR_TRAP, 318 "Hardware Transactional Memory subsystem disabled"); 319 sig = SIGILL; 320 ucode = ILL_ILLOPC; 321 break; 322 case FSCR_IC_DSCR: 323 td->td_pcb->pcb_flags |= PCB_CFSCR | PCB_CDSCR; 324 fscr |= FSCR_DSCR; 325 mtspr(SPR_DSCR, 0); 326 break; 327 case FSCR_IC_EBB: 328 td->td_pcb->pcb_flags |= PCB_CFSCR; 329 fscr |= FSCR_EBB; 330 mtspr(SPR_EBBHR, 0); 331 mtspr(SPR_EBBRR, 0); 332 mtspr(SPR_BESCR, 0); 333 break; 334 case FSCR_IC_TAR: 335 td->td_pcb->pcb_flags |= PCB_CFSCR; 336 fscr |= FSCR_TAR; 337 mtspr(SPR_TAR, 0); 338 break; 339 case FSCR_IC_LM: 340 td->td_pcb->pcb_flags |= PCB_CFSCR; 341 fscr |= FSCR_LM; 342 mtspr(SPR_LMRR, 0); 343 mtspr(SPR_LMSER, 0); 344 break; 345 default: 346 sig = SIGILL; 347 ucode = ILL_ILLOPC; 348 } 349 mtspr(SPR_FSCR, fscr & ~FSCR_IC_MASK); 350 break; 351 case EXC_HEA: 352 sig = SIGILL; 353 ucode = ILL_ILLOPC; 354 break; 355 356 case EXC_VECAST_E: 357 case EXC_VECAST_G4: 358 case EXC_VECAST_G5: 359 /* 360 * We get a VPU assist exception for IEEE mode 361 * vector operations on denormalized floats. 362 * Emulating this is a giant pain, so for now, 363 * just switch off IEEE mode and treat them as 364 * zero. 365 */ 366 367 save_vec(td); 368 td->td_pcb->pcb_vec.vscr |= ALTIVEC_VSCR_NJ; 369 enable_vec(td); 370 break; 371 372 case EXC_ALI: 373 if (fix_unaligned(td, frame) != 0) { 374 sig = SIGBUS; 375 ucode = BUS_ADRALN; 376 addr = frame->dar; 377 } 378 else 379 frame->srr0 += 4; 380 break; 381 382 case EXC_DEBUG: /* Single stepping */ 383 mtspr(SPR_DBSR, mfspr(SPR_DBSR)); 384 frame->srr1 &= ~PSL_DE; 385 frame->cpu.booke.dbcr0 &= ~(DBCR0_IDM | DBCR0_IC); 386 sig = SIGTRAP; 387 ucode = TRAP_TRACE; 388 break; 389 390 case EXC_PGM: 391 /* Identify the trap reason */ 392 if (frame_is_trap_inst(frame)) { 393 #ifdef KDTRACE_HOOKS 394 inst = fuword32((const void *)frame->srr0); 395 if (inst == 0x0FFFDDDD && 396 dtrace_pid_probe_ptr != NULL) { 397 (*dtrace_pid_probe_ptr)(frame); 398 break; 399 } 400 #endif 401 sig = SIGTRAP; 402 ucode = TRAP_BRKPT; 403 } else { 404 sig = ppc_instr_emulate(frame, td); 405 if (sig == SIGILL) { 406 if (frame->srr1 & EXC_PGM_PRIV) 407 ucode = ILL_PRVOPC; 408 else if (frame->srr1 & EXC_PGM_ILLEGAL) 409 ucode = ILL_ILLOPC; 410 } else if (sig == SIGFPE) 411 ucode = FPE_FLTINV; /* Punt for now, invalid operation. */ 412 } 413 break; 414 415 case EXC_MCHK: 416 sig = cpu_machine_check(td, frame, &ucode); 417 printtrap(frame->exc, frame, 0, (frame->srr1 & PSL_PR)); 418 break; 419 420 #if defined(__powerpc64__) && defined(AIM) 421 case EXC_SOFT_PATCH: 422 /* 423 * Point to the instruction that generated the exception to execute it again, 424 * and normalize the register values. 425 */ 426 frame->srr0 -= 4; 427 normalize_inputs(); 428 break; 429 #endif 430 431 default: 432 trap_fatal(frame); 433 } 434 } else { 435 /* Kernel Mode Traps */ 436 437 KASSERT(cold || td->td_ucred != NULL, 438 ("kernel trap doesn't have ucred")); 439 switch (type) { 440 case EXC_PGM: 441 #ifdef KDTRACE_HOOKS 442 if (frame_is_trap_inst(frame)) { 443 if (*(uint32_t *)frame->srr0 == EXC_DTRACE) { 444 if (dtrace_invop_jump_addr != NULL) { 445 dtrace_invop_jump_addr(frame); 446 return; 447 } 448 } 449 } 450 #endif 451 #ifdef KDB 452 if (db_trap_glue(frame)) 453 return; 454 #endif 455 break; 456 #if defined(__powerpc64__) && defined(AIM) 457 case EXC_DSE: 458 /* DSE on radix mmu is automatically fatal. */ 459 if (radix_mmu) 460 break; 461 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0 && 462 (frame->dar & SEGMENT_MASK) == USER_ADDR) { 463 __asm __volatile ("slbmte %0, %1" :: 464 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), 465 "r"(USER_SLB_SLBE)); 466 return; 467 } 468 break; 469 #endif 470 case EXC_DSI: 471 if (trap_pfault(frame, false, NULL, NULL)) 472 return; 473 break; 474 case EXC_MCHK: 475 if (handle_onfault(frame)) 476 return; 477 break; 478 default: 479 break; 480 } 481 trap_fatal(frame); 482 } 483 484 if (sig != 0) { 485 if (p->p_sysent->sv_transtrap != NULL) 486 sig = (p->p_sysent->sv_transtrap)(sig, type); 487 ksiginfo_init_trap(&ksi); 488 ksi.ksi_signo = sig; 489 ksi.ksi_code = (int) ucode; /* XXX, not POSIX */ 490 ksi.ksi_addr = (void *)addr; 491 ksi.ksi_trapno = type; 492 trapsignal(td, &ksi); 493 } 494 495 userret(td, frame); 496 } 497 498 static void 499 trap_fatal(struct trapframe *frame) 500 { 501 #ifdef KDB 502 bool handled; 503 #endif 504 505 printtrap(frame->exc, frame, 1, (frame->srr1 & PSL_PR)); 506 #ifdef KDB 507 if (debugger_on_trap) { 508 kdb_why = KDB_WHY_TRAP; 509 handled = kdb_trap(frame->exc, 0, frame); 510 kdb_why = KDB_WHY_UNSET; 511 if (handled) 512 return; 513 } 514 #endif 515 panic("%s trap", trapname(frame->exc)); 516 } 517 518 static void 519 cpu_printtrap(u_int vector, struct trapframe *frame, int isfatal, int user) 520 { 521 #ifdef AIM 522 uint16_t ver; 523 524 switch (vector) { 525 case EXC_MCHK: 526 ver = mfpvr() >> 16; 527 if (MPC745X_P(ver)) 528 printf(" msssr0 = 0x%b\n", 529 (int)mfspr(SPR_MSSSR0), MSSSR_BITMASK); 530 case EXC_DSE: 531 case EXC_DSI: 532 case EXC_DTMISS: 533 printf(" dsisr = 0x%lx\n", 534 (u_long)frame->cpu.aim.dsisr); 535 break; 536 } 537 #elif defined(BOOKE) 538 vm_paddr_t pa; 539 540 switch (vector) { 541 case EXC_MCHK: 542 pa = mfspr(SPR_MCARU); 543 pa = (pa << 32) | (u_register_t)mfspr(SPR_MCAR); 544 printf(" mcsr = 0x%b\n", 545 (int)mfspr(SPR_MCSR), MCSR_BITMASK); 546 printf(" mcar = 0x%jx\n", (uintmax_t)pa); 547 } 548 printf(" esr = 0x%b\n", 549 (int)frame->cpu.booke.esr, ESR_BITMASK); 550 #endif 551 } 552 553 static void 554 printtrap(u_int vector, struct trapframe *frame, int isfatal, int user) 555 { 556 557 printf("\n"); 558 printf("%s %s trap:\n", isfatal ? "fatal" : "handled", 559 user ? "user" : "kernel"); 560 printf("\n"); 561 printf(" exception = 0x%x (%s)\n", vector, trapname(vector)); 562 switch (vector) { 563 case EXC_DSE: 564 case EXC_DSI: 565 case EXC_DTMISS: 566 case EXC_ALI: 567 case EXC_MCHK: 568 printf(" virtual address = 0x%" PRIxPTR "\n", frame->dar); 569 break; 570 case EXC_ISE: 571 case EXC_ISI: 572 case EXC_ITMISS: 573 printf(" virtual address = 0x%" PRIxPTR "\n", frame->srr0); 574 break; 575 } 576 cpu_printtrap(vector, frame, isfatal, user); 577 printf(" srr0 = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n", 578 frame->srr0, frame->srr0 - (register_t)(__startkernel - KERNBASE)); 579 printf(" srr1 = 0x%lx\n", (u_long)frame->srr1); 580 printf(" current msr = 0x%" PRIxPTR "\n", mfmsr()); 581 printf(" lr = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n", 582 frame->lr, frame->lr - (register_t)(__startkernel - KERNBASE)); 583 printf(" frame = %p\n", frame); 584 printf(" curthread = %p\n", curthread); 585 if (curthread != NULL) 586 printf(" pid = %d, comm = %s\n", 587 curthread->td_proc->p_pid, curthread->td_name); 588 printf("\n"); 589 } 590 591 /* 592 * Handles a fatal fault when we have onfault state to recover. Returns 593 * non-zero if there was onfault recovery state available. 594 */ 595 static int 596 handle_onfault(struct trapframe *frame) 597 { 598 struct thread *td; 599 jmp_buf *fb; 600 601 td = curthread; 602 #if defined(__powerpc64__) || defined(BOOKE) 603 uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault; 604 605 if (dispatch == 0) 606 return (0); 607 /* Short-circuit radix and Book-E paths. */ 608 switch (dispatch) { 609 case COPYFAULT: 610 frame->srr0 = (uintptr_t)copy_fault; 611 return (1); 612 case FUSUFAULT: 613 frame->srr0 = (uintptr_t)fusufault; 614 return (1); 615 default: 616 break; 617 } 618 #endif 619 fb = td->td_pcb->pcb_onfault; 620 if (fb != NULL) { 621 frame->srr0 = (*fb)->_jb[FAULTBUF_LR]; 622 frame->fixreg[1] = (*fb)->_jb[FAULTBUF_R1]; 623 frame->fixreg[2] = (*fb)->_jb[FAULTBUF_R2]; 624 frame->fixreg[3] = 1; 625 frame->cr = (*fb)->_jb[FAULTBUF_CR]; 626 bcopy(&(*fb)->_jb[FAULTBUF_R14], &frame->fixreg[14], 627 18 * sizeof(register_t)); 628 td->td_pcb->pcb_onfault = NULL; /* Returns twice, not thrice */ 629 return (1); 630 } 631 return (0); 632 } 633 634 int 635 cpu_fetch_syscall_args(struct thread *td) 636 { 637 struct proc *p; 638 struct trapframe *frame; 639 struct syscall_args *sa; 640 caddr_t params; 641 size_t argsz; 642 int error, n, narg, i; 643 644 p = td->td_proc; 645 frame = td->td_frame; 646 sa = &td->td_sa; 647 648 sa->code = frame->fixreg[0]; 649 params = (caddr_t)(frame->fixreg + FIRSTARG); 650 n = NARGREG; 651 652 if (sa->code == SYS_syscall) { 653 /* 654 * code is first argument, 655 * followed by actual args. 656 */ 657 sa->code = *(register_t *) params; 658 params += sizeof(register_t); 659 n -= 1; 660 } else if (sa->code == SYS___syscall) { 661 /* 662 * Like syscall, but code is a quad, 663 * so as to maintain quad alignment 664 * for the rest of the args. 665 */ 666 if (SV_PROC_FLAG(p, SV_ILP32)) { 667 params += sizeof(register_t); 668 sa->code = *(register_t *) params; 669 params += sizeof(register_t); 670 n -= 2; 671 } else { 672 sa->code = *(register_t *) params; 673 params += sizeof(register_t); 674 n -= 1; 675 } 676 } 677 678 if (sa->code >= p->p_sysent->sv_size) 679 sa->callp = &p->p_sysent->sv_table[0]; 680 else 681 sa->callp = &p->p_sysent->sv_table[sa->code]; 682 683 narg = sa->callp->sy_narg; 684 685 if (SV_PROC_FLAG(p, SV_ILP32)) { 686 argsz = sizeof(uint32_t); 687 688 for (i = 0; i < n; i++) 689 sa->args[i] = ((u_register_t *)(params))[i] & 690 0xffffffff; 691 } else { 692 argsz = sizeof(uint64_t); 693 694 for (i = 0; i < n; i++) 695 sa->args[i] = ((u_register_t *)(params))[i]; 696 } 697 698 if (narg > n) 699 error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n, 700 (narg - n) * argsz); 701 else 702 error = 0; 703 704 #ifdef __powerpc64__ 705 if (SV_PROC_FLAG(p, SV_ILP32) && narg > n) { 706 /* Expand the size of arguments copied from the stack */ 707 708 for (i = narg; i >= n; i--) 709 sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n]; 710 } 711 #endif 712 713 if (error == 0) { 714 td->td_retval[0] = 0; 715 td->td_retval[1] = frame->fixreg[FIRSTARG + 1]; 716 } 717 return (error); 718 } 719 720 #include "../../kern/subr_syscall.c" 721 722 void 723 syscall(struct trapframe *frame) 724 { 725 struct thread *td; 726 727 td = curthread; 728 td->td_frame = frame; 729 730 #if defined(__powerpc64__) && defined(AIM) 731 /* 732 * Speculatively restore last user SLB segment, which we know is 733 * invalid already, since we are likely to do copyin()/copyout(). 734 */ 735 if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0) 736 __asm __volatile ("slbmte %0, %1; isync" :: 737 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE)); 738 #endif 739 740 syscallenter(td); 741 syscallret(td); 742 } 743 744 static bool 745 trap_pfault(struct trapframe *frame, bool user, int *signo, int *ucode) 746 { 747 vm_offset_t eva; 748 struct thread *td; 749 struct proc *p; 750 vm_map_t map; 751 vm_prot_t ftype; 752 int rv, is_user; 753 754 td = curthread; 755 p = td->td_proc; 756 if (frame->exc == EXC_ISI) { 757 eva = frame->srr0; 758 ftype = VM_PROT_EXECUTE; 759 if (frame->srr1 & SRR1_ISI_PFAULT) 760 ftype |= VM_PROT_READ; 761 } else { 762 eva = frame->dar; 763 #ifdef BOOKE 764 if (frame->cpu.booke.esr & ESR_ST) 765 #else 766 if (frame->cpu.aim.dsisr & DSISR_STORE) 767 #endif 768 ftype = VM_PROT_WRITE; 769 else 770 ftype = VM_PROT_READ; 771 } 772 #if defined(__powerpc64__) && defined(AIM) 773 if (radix_mmu && pmap_nofault(&p->p_vmspace->vm_pmap, eva, ftype) == 0) 774 return (true); 775 #endif 776 777 if (__predict_false((td->td_pflags & TDP_NOFAULTING) == 0)) { 778 /* 779 * If we get a page fault while in a critical section, then 780 * it is most likely a fatal kernel page fault. The kernel 781 * is already going to panic trying to get a sleep lock to 782 * do the VM lookup, so just consider it a fatal trap so the 783 * kernel can print out a useful trap message and even get 784 * to the debugger. 785 * 786 * If we get a page fault while holding a non-sleepable 787 * lock, then it is most likely a fatal kernel page fault. 788 * If WITNESS is enabled, then it's going to whine about 789 * bogus LORs with various VM locks, so just skip to the 790 * fatal trap handling directly. 791 */ 792 if (td->td_critnest != 0 || 793 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 794 "Kernel page fault") != 0) { 795 trap_fatal(frame); 796 return (false); 797 } 798 } 799 if (user) { 800 KASSERT(p->p_vmspace != NULL, ("trap_pfault: vmspace NULL")); 801 map = &p->p_vmspace->vm_map; 802 } else { 803 rv = pmap_decode_kernel_ptr(eva, &is_user, &eva); 804 if (rv != 0) 805 return (false); 806 807 if (is_user) 808 map = &p->p_vmspace->vm_map; 809 else 810 map = kernel_map; 811 } 812 813 /* Fault in the page. */ 814 rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode); 815 /* 816 * XXXDTRACE: add dtrace_doubletrap_func here? 817 */ 818 819 if (rv == KERN_SUCCESS) 820 return (true); 821 822 if (!user && handle_onfault(frame)) 823 return (true); 824 825 return (false); 826 } 827 828 /* 829 * For now, this only deals with the particular unaligned access case 830 * that gcc tends to generate. Eventually it should handle all of the 831 * possibilities that can happen on a 32-bit PowerPC in big-endian mode. 832 */ 833 834 static int 835 fix_unaligned(struct thread *td, struct trapframe *frame) 836 { 837 struct thread *fputhread; 838 #ifdef BOOKE 839 uint32_t inst; 840 #endif 841 int indicator, reg; 842 double *fpr; 843 844 #ifdef __SPE__ 845 indicator = (frame->cpu.booke.esr & (ESR_ST|ESR_SPE)); 846 if (indicator & ESR_SPE) { 847 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0) 848 return (-1); 849 reg = EXC_ALI_INST_RST(inst); 850 fpr = (double *)td->td_pcb->pcb_vec.vr[reg]; 851 fputhread = PCPU_GET(vecthread); 852 853 /* Juggle the SPE to ensure that we've initialized 854 * the registers, and that their current state is in 855 * the PCB. 856 */ 857 if (fputhread != td) { 858 if (fputhread) 859 save_vec(fputhread); 860 enable_vec(td); 861 } 862 save_vec(td); 863 864 if (!(indicator & ESR_ST)) { 865 if (copyin((void *)frame->dar, fpr, 866 sizeof(double)) != 0) 867 return (-1); 868 frame->fixreg[reg] = td->td_pcb->pcb_vec.vr[reg][1]; 869 enable_vec(td); 870 } else { 871 td->td_pcb->pcb_vec.vr[reg][1] = frame->fixreg[reg]; 872 if (copyout(fpr, (void *)frame->dar, 873 sizeof(double)) != 0) 874 return (-1); 875 } 876 return (0); 877 } 878 #else 879 #ifdef BOOKE 880 indicator = (frame->cpu.booke.esr & ESR_ST) ? EXC_ALI_STFD : EXC_ALI_LFD; 881 #else 882 indicator = EXC_ALI_OPCODE_INDICATOR(frame->cpu.aim.dsisr); 883 #endif 884 885 switch (indicator) { 886 case EXC_ALI_LFD: 887 case EXC_ALI_STFD: 888 #ifdef BOOKE 889 if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0) 890 return (-1); 891 reg = EXC_ALI_INST_RST(inst); 892 #else 893 reg = EXC_ALI_RST(frame->cpu.aim.dsisr); 894 #endif 895 fpr = &td->td_pcb->pcb_fpu.fpr[reg].fpr; 896 fputhread = PCPU_GET(fputhread); 897 898 /* Juggle the FPU to ensure that we've initialized 899 * the FPRs, and that their current state is in 900 * the PCB. 901 */ 902 if (fputhread != td) { 903 if (fputhread) 904 save_fpu(fputhread); 905 enable_fpu(td); 906 } 907 save_fpu(td); 908 909 if (indicator == EXC_ALI_LFD) { 910 if (copyin((void *)frame->dar, fpr, 911 sizeof(double)) != 0) 912 return (-1); 913 enable_fpu(td); 914 } else { 915 if (copyout(fpr, (void *)frame->dar, 916 sizeof(double)) != 0) 917 return (-1); 918 } 919 return (0); 920 break; 921 } 922 #endif 923 924 return (-1); 925 } 926 927 #if defined(__powerpc64__) && defined(AIM) 928 #define MSKNSHL(x, m, n) "(((" #x ") & " #m ") << " #n ")" 929 #define MSKNSHR(x, m, n) "(((" #x ") & " #m ") >> " #n ")" 930 931 /* xvcpsgndp instruction, built in opcode format. 932 * This can be changed to use mnemonic after a toolchain update. 933 */ 934 #define XVCPSGNDP(xt, xa, xb) \ 935 __asm __volatile(".long (" \ 936 MSKNSHL(60, 0x3f, 26) " | " \ 937 MSKNSHL(xt, 0x1f, 21) " | " \ 938 MSKNSHL(xa, 0x1f, 16) " | " \ 939 MSKNSHL(xb, 0x1f, 11) " | " \ 940 MSKNSHL(240, 0xff, 3) " | " \ 941 MSKNSHR(xa, 0x20, 3) " | " \ 942 MSKNSHR(xa, 0x20, 4) " | " \ 943 MSKNSHR(xa, 0x20, 5) ")") 944 945 /* Macros to normalize 1 or 10 VSX registers */ 946 #define NORM(x) XVCPSGNDP(x, x, x) 947 #define NORM10(x) \ 948 NORM(x ## 0); NORM(x ## 1); NORM(x ## 2); NORM(x ## 3); NORM(x ## 4); \ 949 NORM(x ## 5); NORM(x ## 6); NORM(x ## 7); NORM(x ## 8); NORM(x ## 9) 950 951 static void 952 normalize_inputs(void) 953 { 954 unsigned long msr; 955 956 /* enable VSX */ 957 msr = mfmsr(); 958 mtmsr(msr | PSL_VSX); 959 960 NORM(0); NORM(1); NORM(2); NORM(3); NORM(4); 961 NORM(5); NORM(6); NORM(7); NORM(8); NORM(9); 962 NORM10(1); NORM10(2); NORM10(3); NORM10(4); NORM10(5); 963 NORM(60); NORM(61); NORM(62); NORM(63); 964 965 /* restore MSR */ 966 mtmsr(msr); 967 } 968 #endif 969 970 #ifdef KDB 971 int 972 db_trap_glue(struct trapframe *frame) 973 { 974 975 if (!(frame->srr1 & PSL_PR) 976 && (frame->exc == EXC_TRC || frame->exc == EXC_RUNMODETRC 977 || frame_is_trap_inst(frame) 978 || frame->exc == EXC_BPT 979 || frame->exc == EXC_DEBUG 980 || frame->exc == EXC_DSI)) { 981 int type = frame->exc; 982 983 /* Ignore DTrace traps. */ 984 if (*(uint32_t *)frame->srr0 == EXC_DTRACE) 985 return (0); 986 if (frame_is_trap_inst(frame)) { 987 type = T_BREAKPOINT; 988 } 989 return (kdb_trap(type, 0, frame)); 990 } 991 992 return (0); 993 } 994 #endif 995