1e5acd89cSAndrew Turner /*- 2e5acd89cSAndrew Turner * Copyright (c) 2014 Andrew Turner 3e5acd89cSAndrew Turner * All rights reserved. 4e5acd89cSAndrew Turner * 5e5acd89cSAndrew Turner * Redistribution and use in source and binary forms, with or without 6e5acd89cSAndrew Turner * modification, are permitted provided that the following conditions 7e5acd89cSAndrew Turner * are met: 8e5acd89cSAndrew Turner * 1. Redistributions of source code must retain the above copyright 9e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer. 10e5acd89cSAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright 11e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer in the 12e5acd89cSAndrew Turner * documentation and/or other materials provided with the distribution. 13e5acd89cSAndrew Turner * 14e5acd89cSAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e5acd89cSAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e5acd89cSAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e5acd89cSAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e5acd89cSAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e5acd89cSAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e5acd89cSAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e5acd89cSAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e5acd89cSAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e5acd89cSAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e5acd89cSAndrew Turner * SUCH DAMAGE. 25e5acd89cSAndrew Turner * 26e5acd89cSAndrew Turner */ 27e5acd89cSAndrew Turner 28e5acd89cSAndrew Turner #include <sys/cdefs.h> 29e5acd89cSAndrew Turner __FBSDID("$FreeBSD$"); 30e5acd89cSAndrew Turner 31e5acd89cSAndrew Turner #include <sys/param.h> 32e5acd89cSAndrew Turner #include <sys/systm.h> 33e5acd89cSAndrew Turner #include <sys/kernel.h> 34daec9284SConrad Meyer #include <sys/ktr.h> 35e5acd89cSAndrew Turner #include <sys/lock.h> 36e5acd89cSAndrew Turner #include <sys/mutex.h> 37e5acd89cSAndrew Turner #include <sys/pioctl.h> 38e5acd89cSAndrew Turner #include <sys/proc.h> 39e5acd89cSAndrew Turner #include <sys/ptrace.h> 40e5acd89cSAndrew Turner #include <sys/syscall.h> 41e5acd89cSAndrew Turner #include <sys/sysent.h> 42e5acd89cSAndrew Turner #ifdef KDB 43e5acd89cSAndrew Turner #include <sys/kdb.h> 44e5acd89cSAndrew Turner #endif 45e5acd89cSAndrew Turner 46e5acd89cSAndrew Turner #include <vm/vm.h> 47e5acd89cSAndrew Turner #include <vm/pmap.h> 48e5acd89cSAndrew Turner #include <vm/vm_kern.h> 49e5acd89cSAndrew Turner #include <vm/vm_map.h> 50cb02f6b9SAndrew Turner #include <vm/vm_param.h> 51e5acd89cSAndrew Turner #include <vm/vm_extern.h> 52e5acd89cSAndrew Turner 53e5acd89cSAndrew Turner #include <machine/frame.h> 54e5acd89cSAndrew Turner #include <machine/pcb.h> 55e5acd89cSAndrew Turner #include <machine/pcpu.h> 56bcf2b954SAndrew Turner #include <machine/undefined.h> 57e5acd89cSAndrew Turner 58b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 59b78ee15eSRuslan Bukin #include <sys/dtrace_bsd.h> 60b78ee15eSRuslan Bukin #endif 61b78ee15eSRuslan Bukin 62e5acd89cSAndrew Turner #ifdef VFP 63e5acd89cSAndrew Turner #include <machine/vfp.h> 64e5acd89cSAndrew Turner #endif 65e5acd89cSAndrew Turner 66e5acd89cSAndrew Turner #ifdef KDB 67e5acd89cSAndrew Turner #include <machine/db_machdep.h> 68e5acd89cSAndrew Turner #endif 69e5acd89cSAndrew Turner 70e5acd89cSAndrew Turner #ifdef DDB 71e5acd89cSAndrew Turner #include <ddb/db_output.h> 72e5acd89cSAndrew Turner #endif 73e5acd89cSAndrew Turner 749d77aa2aSAndrew Turner extern register_t fsu_intr_fault; 75e5acd89cSAndrew Turner 76e5acd89cSAndrew Turner /* Called from exception.S */ 77d7635c7aSAndrew Turner void do_el1h_sync(struct thread *, struct trapframe *); 78d7635c7aSAndrew Turner void do_el0_sync(struct thread *, struct trapframe *); 79e5acd89cSAndrew Turner void do_el0_error(struct trapframe *); 80dc9b99a8SAndrew Turner void do_serror(struct trapframe *); 81dc9b99a8SAndrew Turner void unhandled_exception(struct trapframe *); 82dc9b99a8SAndrew Turner 830510aedcSKonstantin Belousov static void print_registers(struct trapframe *frame); 84e5acd89cSAndrew Turner 85b78ee15eSRuslan Bukin int (*dtrace_invop_jump_addr)(struct trapframe *); 86b78ee15eSRuslan Bukin 87acd6f4beSAndrew Turner typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t, 88acd6f4beSAndrew Turner uint64_t, int); 89acd6f4beSAndrew Turner 90c7bb1909SJustin Hibbits static abort_handler align_abort; 91acd6f4beSAndrew Turner static abort_handler data_abort; 92acd6f4beSAndrew Turner 93acd6f4beSAndrew Turner static abort_handler *abort_handlers[] = { 94acd6f4beSAndrew Turner [ISS_DATA_DFSC_TF_L0] = data_abort, 95acd6f4beSAndrew Turner [ISS_DATA_DFSC_TF_L1] = data_abort, 96acd6f4beSAndrew Turner [ISS_DATA_DFSC_TF_L2] = data_abort, 97acd6f4beSAndrew Turner [ISS_DATA_DFSC_TF_L3] = data_abort, 98acd6f4beSAndrew Turner [ISS_DATA_DFSC_AFF_L1] = data_abort, 99acd6f4beSAndrew Turner [ISS_DATA_DFSC_AFF_L2] = data_abort, 100acd6f4beSAndrew Turner [ISS_DATA_DFSC_AFF_L3] = data_abort, 101acd6f4beSAndrew Turner [ISS_DATA_DFSC_PF_L1] = data_abort, 102acd6f4beSAndrew Turner [ISS_DATA_DFSC_PF_L2] = data_abort, 103acd6f4beSAndrew Turner [ISS_DATA_DFSC_PF_L3] = data_abort, 104c7bb1909SJustin Hibbits [ISS_DATA_DFSC_ALIGN] = align_abort, 105acd6f4beSAndrew Turner }; 106acd6f4beSAndrew Turner 107e5acd89cSAndrew Turner static __inline void 108cb02f6b9SAndrew Turner call_trapsignal(struct thread *td, int sig, int code, void *addr) 109e5acd89cSAndrew Turner { 110e5acd89cSAndrew Turner ksiginfo_t ksi; 111e5acd89cSAndrew Turner 112e5acd89cSAndrew Turner ksiginfo_init_trap(&ksi); 113e5acd89cSAndrew Turner ksi.ksi_signo = sig; 114cb02f6b9SAndrew Turner ksi.ksi_code = code; 115cb02f6b9SAndrew Turner ksi.ksi_addr = addr; 116e5acd89cSAndrew Turner trapsignal(td, &ksi); 117e5acd89cSAndrew Turner } 118e5acd89cSAndrew Turner 119e5acd89cSAndrew Turner int 1202d88da2fSKonstantin Belousov cpu_fetch_syscall_args(struct thread *td) 121e5acd89cSAndrew Turner { 122e5acd89cSAndrew Turner struct proc *p; 123e5acd89cSAndrew Turner register_t *ap; 1242d88da2fSKonstantin Belousov struct syscall_args *sa; 125e5acd89cSAndrew Turner int nap; 126e5acd89cSAndrew Turner 127e5acd89cSAndrew Turner nap = 8; 128e5acd89cSAndrew Turner p = td->td_proc; 129e5acd89cSAndrew Turner ap = td->td_frame->tf_x; 1302d88da2fSKonstantin Belousov sa = &td->td_sa; 131e5acd89cSAndrew Turner 132e5acd89cSAndrew Turner sa->code = td->td_frame->tf_x[8]; 133e5acd89cSAndrew Turner 134e5acd89cSAndrew Turner if (sa->code == SYS_syscall || sa->code == SYS___syscall) { 135e5acd89cSAndrew Turner sa->code = *ap++; 136e5acd89cSAndrew Turner nap--; 137e5acd89cSAndrew Turner } 138e5acd89cSAndrew Turner 139e5acd89cSAndrew Turner if (sa->code >= p->p_sysent->sv_size) 140e5acd89cSAndrew Turner sa->callp = &p->p_sysent->sv_table[0]; 141e5acd89cSAndrew Turner else 142e5acd89cSAndrew Turner sa->callp = &p->p_sysent->sv_table[sa->code]; 143e5acd89cSAndrew Turner 144e5acd89cSAndrew Turner sa->narg = sa->callp->sy_narg; 145e5acd89cSAndrew Turner memcpy(sa->args, ap, nap * sizeof(register_t)); 146e5acd89cSAndrew Turner if (sa->narg > nap) 1476bc3fe5fSPedro F. Giffuni panic("ARM64TODO: Could we have more than 8 args?"); 148e5acd89cSAndrew Turner 149e5acd89cSAndrew Turner td->td_retval[0] = 0; 150e5acd89cSAndrew Turner td->td_retval[1] = 0; 151e5acd89cSAndrew Turner 152e5acd89cSAndrew Turner return (0); 153e5acd89cSAndrew Turner } 154e5acd89cSAndrew Turner 155e5acd89cSAndrew Turner #include "../../kern/subr_syscall.c" 156e5acd89cSAndrew Turner 157e5acd89cSAndrew Turner static void 158d7635c7aSAndrew Turner svc_handler(struct thread *td, struct trapframe *frame) 159e5acd89cSAndrew Turner { 160e5acd89cSAndrew Turner 16152a680fbSAndrew Turner if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) { 162c18ca749SJohn Baldwin syscallenter(td); 163c18ca749SJohn Baldwin syscallret(td); 16452a680fbSAndrew Turner } else { 16552a680fbSAndrew Turner call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr); 16652a680fbSAndrew Turner userret(td, frame); 16752a680fbSAndrew Turner } 168e5acd89cSAndrew Turner } 169e5acd89cSAndrew Turner 170e5acd89cSAndrew Turner static void 171c7bb1909SJustin Hibbits align_abort(struct thread *td, struct trapframe *frame, uint64_t esr, 172c7bb1909SJustin Hibbits uint64_t far, int lower) 173c7bb1909SJustin Hibbits { 174c7bb1909SJustin Hibbits if (!lower) 175c7bb1909SJustin Hibbits panic("Misaligned access from kernel space!"); 176c7bb1909SJustin Hibbits 177c7bb1909SJustin Hibbits call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr); 178c7bb1909SJustin Hibbits userret(td, frame); 179c7bb1909SJustin Hibbits } 180c7bb1909SJustin Hibbits 181c7bb1909SJustin Hibbits static void 182d7635c7aSAndrew Turner data_abort(struct thread *td, struct trapframe *frame, uint64_t esr, 183acd6f4beSAndrew Turner uint64_t far, int lower) 184e5acd89cSAndrew Turner { 185e5acd89cSAndrew Turner struct vm_map *map; 186e5acd89cSAndrew Turner struct proc *p; 187e5acd89cSAndrew Turner struct pcb *pcb; 188e5acd89cSAndrew Turner vm_prot_t ftype; 189cb02f6b9SAndrew Turner int error, sig, ucode; 190c9c4d38aSAndriy Gapon #ifdef KDB 191c9c4d38aSAndriy Gapon bool handled; 192c9c4d38aSAndriy Gapon #endif 193e5acd89cSAndrew Turner 1946ced3789SKonstantin Belousov /* 1956ced3789SKonstantin Belousov * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive 1966ced3789SKonstantin Belousov * and Store-Exclusive instruction usage restrictions", state 1976ced3789SKonstantin Belousov * of the exclusive monitors after data abort exception is unknown. 1986ced3789SKonstantin Belousov */ 1996ced3789SKonstantin Belousov clrex(); 2006ced3789SKonstantin Belousov 2018f746773SAndrew Turner #ifdef KDB 2028f746773SAndrew Turner if (kdb_active) { 2038f746773SAndrew Turner kdb_reenter(); 2048f746773SAndrew Turner return; 2058f746773SAndrew Turner } 2068f746773SAndrew Turner #endif 2078f746773SAndrew Turner 208e5acd89cSAndrew Turner pcb = td->td_pcb; 2090510aedcSKonstantin Belousov p = td->td_proc; 210e5acd89cSAndrew Turner if (lower) 2110510aedcSKonstantin Belousov map = &p->p_vmspace->vm_map; 212e5acd89cSAndrew Turner else { 2139d0a6b83SAndrew Turner intr_enable(); 2149d0a6b83SAndrew Turner 215e5acd89cSAndrew Turner /* The top bit tells us which range to use */ 21671cb533eSAndrew Turner if (far >= VM_MAXUSER_ADDRESS) { 217e5acd89cSAndrew Turner map = kernel_map; 2181d479540SAndrew Turner } else { 2190510aedcSKonstantin Belousov map = &p->p_vmspace->vm_map; 2201d479540SAndrew Turner if (map == NULL) 2211d479540SAndrew Turner map = kernel_map; 2221d479540SAndrew Turner } 223e5acd89cSAndrew Turner } 224e5acd89cSAndrew Turner 22578921ae8SAndrew Turner /* 226ca2cae0bSMark Johnston * Try to handle translation, access flag, and permission faults. 227ca2cae0bSMark Johnston * Translation faults may occur as a result of the required 228ca2cae0bSMark Johnston * break-before-make sequence used when promoting or demoting 229ca2cae0bSMark Johnston * superpages. Such faults must not occur while holding the pmap lock, 230ca2cae0bSMark Johnston * or pmap_fault() will recurse on that lock. 23178921ae8SAndrew Turner */ 232ca2cae0bSMark Johnston if ((lower || map == kernel_map || pcb->pcb_onfault != 0) && 233ca2cae0bSMark Johnston pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) 234e2b8bf0aSAndrew Turner return; 235e2b8bf0aSAndrew Turner 236510a3f1bSAndrew Turner KASSERT(td->td_md.md_spinlock_count == 0, 237510a3f1bSAndrew Turner ("data abort with spinlock held")); 238510a3f1bSAndrew Turner if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | 239510a3f1bSAndrew Turner WARN_GIANTOK, NULL, "Kernel page fault") != 0) { 240510a3f1bSAndrew Turner print_registers(frame); 241510a3f1bSAndrew Turner printf(" far: %16lx\n", far); 242510a3f1bSAndrew Turner printf(" esr: %.8lx\n", esr); 243510a3f1bSAndrew Turner panic("data abort in critical section or under mutex"); 244510a3f1bSAndrew Turner } 245510a3f1bSAndrew Turner 246acd6f4beSAndrew Turner switch (ESR_ELx_EXCEPTION(esr)) { 247acd6f4beSAndrew Turner case EXCP_INSN_ABORT: 248acd6f4beSAndrew Turner case EXCP_INSN_ABORT_L: 249ca2cae0bSMark Johnston ftype = VM_PROT_EXECUTE; 250acd6f4beSAndrew Turner break; 251acd6f4beSAndrew Turner default: 252ca2cae0bSMark Johnston ftype = (esr & ISS_DATA_WnR) == 0 ? VM_PROT_READ : 253ca2cae0bSMark Johnston VM_PROT_READ | VM_PROT_WRITE; 254acd6f4beSAndrew Turner break; 255acd6f4beSAndrew Turner } 256e5acd89cSAndrew Turner 2571fa67124SKonstantin Belousov /* Fault in the page. */ 258df08823dSKonstantin Belousov error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode); 259cb02f6b9SAndrew Turner if (error != KERN_SUCCESS) { 260e5acd89cSAndrew Turner if (lower) { 261cb02f6b9SAndrew Turner call_trapsignal(td, sig, ucode, (void *)far); 262e5acd89cSAndrew Turner } else { 263e5acd89cSAndrew Turner if (td->td_intr_nesting_level == 0 && 264e5acd89cSAndrew Turner pcb->pcb_onfault != 0) { 265e5acd89cSAndrew Turner frame->tf_x[0] = error; 266e5acd89cSAndrew Turner frame->tf_elr = pcb->pcb_onfault; 267e5acd89cSAndrew Turner return; 268e5acd89cSAndrew Turner } 2691e888d78SAndrew Turner 2701e888d78SAndrew Turner printf("Fatal data abort:\n"); 2711e888d78SAndrew Turner print_registers(frame); 2721e888d78SAndrew Turner printf(" far: %16lx\n", far); 2731e888d78SAndrew Turner printf(" esr: %.8lx\n", esr); 2741e888d78SAndrew Turner 2758f746773SAndrew Turner #ifdef KDB 276b317cfd4SJohn Baldwin if (debugger_on_trap) { 277c9c4d38aSAndriy Gapon kdb_why = KDB_WHY_TRAP; 278c9c4d38aSAndriy Gapon handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0, 279c9c4d38aSAndriy Gapon frame); 280c9c4d38aSAndriy Gapon kdb_why = KDB_WHY_UNSET; 281c9c4d38aSAndriy Gapon if (handled) 2828f746773SAndrew Turner return; 283c9c4d38aSAndriy Gapon } 2848f746773SAndrew Turner #endif 285e5acd89cSAndrew Turner panic("vm_fault failed: %lx", frame->tf_elr); 286e5acd89cSAndrew Turner } 287e5acd89cSAndrew Turner } 288e5acd89cSAndrew Turner 289e5acd89cSAndrew Turner if (lower) 290e5acd89cSAndrew Turner userret(td, frame); 291e5acd89cSAndrew Turner } 292e5acd89cSAndrew Turner 293ccd285e7SEd Maste static void 294ccd285e7SEd Maste print_registers(struct trapframe *frame) 295ccd285e7SEd Maste { 296ccd285e7SEd Maste u_int reg; 297ccd285e7SEd Maste 2982e620e70SAndrew Turner for (reg = 0; reg < nitems(frame->tf_x); reg++) { 299ccd285e7SEd Maste printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg, 300ccd285e7SEd Maste frame->tf_x[reg]); 301ccd285e7SEd Maste } 302ccd285e7SEd Maste printf(" sp: %16lx\n", frame->tf_sp); 303ccd285e7SEd Maste printf(" lr: %16lx\n", frame->tf_lr); 304ccd285e7SEd Maste printf(" elr: %16lx\n", frame->tf_elr); 305f17e4f07SAndrew Turner printf("spsr: %8x\n", frame->tf_spsr); 306ccd285e7SEd Maste } 307ccd285e7SEd Maste 308e5acd89cSAndrew Turner void 309d7635c7aSAndrew Turner do_el1h_sync(struct thread *td, struct trapframe *frame) 310e5acd89cSAndrew Turner { 311ce793a52SAndrew Turner struct trapframe *oframe; 312e5acd89cSAndrew Turner uint32_t exception; 3130510aedcSKonstantin Belousov uint64_t esr, far; 314acd6f4beSAndrew Turner int dfsc; 315e5acd89cSAndrew Turner 316e5acd89cSAndrew Turner /* Read the esr register to get the exception details */ 317f17e4f07SAndrew Turner esr = frame->tf_esr; 318e5acd89cSAndrew Turner exception = ESR_ELx_EXCEPTION(esr); 319e5acd89cSAndrew Turner 320b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 321b78ee15eSRuslan Bukin if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 322b78ee15eSRuslan Bukin return; 323b78ee15eSRuslan Bukin #endif 324b78ee15eSRuslan Bukin 3253ad7e84eSAndrew Turner CTR4(KTR_TRAP, 326d7635c7aSAndrew Turner "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, 327d7635c7aSAndrew Turner esr, frame->tf_elr, frame); 3283ad7e84eSAndrew Turner 329ce793a52SAndrew Turner oframe = td->td_frame; 330ce793a52SAndrew Turner 331ce793a52SAndrew Turner switch (exception) { 332ce793a52SAndrew Turner case EXCP_BRK: 333ce793a52SAndrew Turner case EXCP_WATCHPT_EL1: 334ce793a52SAndrew Turner case EXCP_SOFTSTP_EL1: 335ce793a52SAndrew Turner break; 336ce793a52SAndrew Turner default: 337ce793a52SAndrew Turner td->td_frame = frame; 338ce793a52SAndrew Turner break; 339ce793a52SAndrew Turner } 340ce793a52SAndrew Turner 341e5acd89cSAndrew Turner switch(exception) { 342e5acd89cSAndrew Turner case EXCP_FP_SIMD: 343e5acd89cSAndrew Turner case EXCP_TRAP_FP: 3446ed982a2SAndrew Turner #ifdef VFP 345d7635c7aSAndrew Turner if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) { 3466ed982a2SAndrew Turner vfp_restore_state(); 3476ed982a2SAndrew Turner } else 3486ed982a2SAndrew Turner #endif 3496ed982a2SAndrew Turner { 350ccd285e7SEd Maste print_registers(frame); 3511e888d78SAndrew Turner printf(" esr: %.8lx\n", esr); 352e5acd89cSAndrew Turner panic("VFP exception in the kernel"); 3536ed982a2SAndrew Turner } 3546ed982a2SAndrew Turner break; 355d953ec32SAndrew Turner case EXCP_INSN_ABORT: 356e5acd89cSAndrew Turner case EXCP_DATA_ABORT: 3570510aedcSKonstantin Belousov far = READ_SPECIALREG(far_el1); 358acd6f4beSAndrew Turner dfsc = esr & ISS_DATA_DFSC_MASK; 359acd6f4beSAndrew Turner if (dfsc < nitems(abort_handlers) && 360acd6f4beSAndrew Turner abort_handlers[dfsc] != NULL) 361acd6f4beSAndrew Turner abort_handlers[dfsc](td, frame, esr, far, 0); 362acd6f4beSAndrew Turner else 363acd6f4beSAndrew Turner panic("Unhandled EL1 %s abort: %x", 364acd6f4beSAndrew Turner exception == EXCP_INSN_ABORT ? "instruction" : 365acd6f4beSAndrew Turner "data", dfsc); 366e5acd89cSAndrew Turner break; 367e5acd89cSAndrew Turner case EXCP_BRK: 368b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 369b78ee15eSRuslan Bukin if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ 370b78ee15eSRuslan Bukin dtrace_invop_jump_addr != 0) { 371b78ee15eSRuslan Bukin dtrace_invop_jump_addr(frame); 372b78ee15eSRuslan Bukin break; 373b78ee15eSRuslan Bukin } 374b78ee15eSRuslan Bukin #endif 375de14bffeSOleksandr Tymoshenko #ifdef KDB 376be84f91cSAndrew Turner kdb_trap(exception, 0, 377be84f91cSAndrew Turner (td->td_frame != NULL) ? td->td_frame : frame); 378de14bffeSOleksandr Tymoshenko #else 379de14bffeSOleksandr Tymoshenko panic("No debugger in kernel.\n"); 380de14bffeSOleksandr Tymoshenko #endif 381b4cc39e7SAndrew Turner frame->tf_elr += 4; 382be84f91cSAndrew Turner break; 383e5acd89cSAndrew Turner case EXCP_WATCHPT_EL1: 384e5acd89cSAndrew Turner case EXCP_SOFTSTP_EL1: 385e5acd89cSAndrew Turner #ifdef KDB 386ce793a52SAndrew Turner kdb_trap(exception, 0, 387ce793a52SAndrew Turner (td->td_frame != NULL) ? td->td_frame : frame); 388e5acd89cSAndrew Turner #else 389e5acd89cSAndrew Turner panic("No debugger in kernel.\n"); 390e5acd89cSAndrew Turner #endif 391e5acd89cSAndrew Turner break; 392bcf2b954SAndrew Turner case EXCP_UNKNOWN: 393bcf2b954SAndrew Turner if (undef_insn(1, frame)) 394bcf2b954SAndrew Turner break; 395bcf2b954SAndrew Turner /* FALLTHROUGH */ 396e5acd89cSAndrew Turner default: 397ccd285e7SEd Maste print_registers(frame); 398e5acd89cSAndrew Turner panic("Unknown kernel exception %x esr_el1 %lx\n", exception, 399e5acd89cSAndrew Turner esr); 400e5acd89cSAndrew Turner } 401ce793a52SAndrew Turner 402ce793a52SAndrew Turner td->td_frame = oframe; 403e5acd89cSAndrew Turner } 404e5acd89cSAndrew Turner 405e5acd89cSAndrew Turner void 406d7635c7aSAndrew Turner do_el0_sync(struct thread *td, struct trapframe *frame) 407e5acd89cSAndrew Turner { 4087023544aSAndrew Turner pcpu_bp_harden bp_harden; 409e5acd89cSAndrew Turner uint32_t exception; 4100510aedcSKonstantin Belousov uint64_t esr, far; 411acd6f4beSAndrew Turner int dfsc; 412e5acd89cSAndrew Turner 413e5acd89cSAndrew Turner /* Check we have a sane environment when entering from userland */ 414e5acd89cSAndrew Turner KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, 415e5acd89cSAndrew Turner ("Invalid pcpu address from userland: %p (tpidr %lx)", 416e5acd89cSAndrew Turner get_pcpu(), READ_SPECIALREG(tpidr_el1))); 417e5acd89cSAndrew Turner 418f17e4f07SAndrew Turner esr = frame->tf_esr; 419e5acd89cSAndrew Turner exception = ESR_ELx_EXCEPTION(esr); 4200510aedcSKonstantin Belousov switch (exception) { 4210510aedcSKonstantin Belousov case EXCP_INSN_ABORT_L: 4227023544aSAndrew Turner far = READ_SPECIALREG(far_el1); 4237023544aSAndrew Turner 4247023544aSAndrew Turner /* 4257023544aSAndrew Turner * Userspace may be trying to train the branch predictor to 4267023544aSAndrew Turner * attack the kernel. If we are on a CPU affected by this 4277023544aSAndrew Turner * call the handler to clear the branch predictor state. 4287023544aSAndrew Turner */ 4297023544aSAndrew Turner if (far > VM_MAXUSER_ADDRESS) { 4307023544aSAndrew Turner bp_harden = PCPU_GET(bp_harden); 4317023544aSAndrew Turner if (bp_harden != NULL) 4327023544aSAndrew Turner bp_harden(); 4337023544aSAndrew Turner } 4347023544aSAndrew Turner break; 4357023544aSAndrew Turner case EXCP_UNKNOWN: 4360510aedcSKonstantin Belousov case EXCP_DATA_ABORT_L: 4370510aedcSKonstantin Belousov case EXCP_DATA_ABORT: 4380510aedcSKonstantin Belousov far = READ_SPECIALREG(far_el1); 4397023544aSAndrew Turner break; 4400510aedcSKonstantin Belousov } 4410510aedcSKonstantin Belousov intr_enable(); 442e5acd89cSAndrew Turner 4433ad7e84eSAndrew Turner CTR4(KTR_TRAP, 444d7635c7aSAndrew Turner "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, 445d7635c7aSAndrew Turner frame->tf_elr, frame); 4463ad7e84eSAndrew Turner 447e5acd89cSAndrew Turner switch(exception) { 448e5acd89cSAndrew Turner case EXCP_FP_SIMD: 449e5acd89cSAndrew Turner case EXCP_TRAP_FP: 450e5acd89cSAndrew Turner #ifdef VFP 451e5acd89cSAndrew Turner vfp_restore_state(); 452e5acd89cSAndrew Turner #else 453e5acd89cSAndrew Turner panic("VFP exception in userland"); 454e5acd89cSAndrew Turner #endif 455e5acd89cSAndrew Turner break; 4567af24ff7SEd Schouten case EXCP_SVC32: 4577af24ff7SEd Schouten case EXCP_SVC64: 458d7635c7aSAndrew Turner svc_handler(td, frame); 459e5acd89cSAndrew Turner break; 460e5acd89cSAndrew Turner case EXCP_INSN_ABORT_L: 461e5acd89cSAndrew Turner case EXCP_DATA_ABORT_L: 4624cbca608SZbigniew Bodek case EXCP_DATA_ABORT: 463acd6f4beSAndrew Turner dfsc = esr & ISS_DATA_DFSC_MASK; 464acd6f4beSAndrew Turner if (dfsc < nitems(abort_handlers) && 465acd6f4beSAndrew Turner abort_handlers[dfsc] != NULL) 466acd6f4beSAndrew Turner abort_handlers[dfsc](td, frame, esr, far, 1); 467acd6f4beSAndrew Turner else 468acd6f4beSAndrew Turner panic("Unhandled EL0 %s abort: %x", 469acd6f4beSAndrew Turner exception == EXCP_INSN_ABORT_L ? "instruction" : 470acd6f4beSAndrew Turner "data", dfsc); 471e5acd89cSAndrew Turner break; 472ccd285e7SEd Maste case EXCP_UNKNOWN: 473bcf2b954SAndrew Turner if (!undef_insn(0, frame)) 474bcf2b954SAndrew Turner call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far); 475bcf2b954SAndrew Turner userret(td, frame); 476ccd285e7SEd Maste break; 477729ac0eeSAndrew Turner case EXCP_SP_ALIGN: 478729ac0eeSAndrew Turner call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp); 479729ac0eeSAndrew Turner userret(td, frame); 480729ac0eeSAndrew Turner break; 4818bdcc096SAndrew Turner case EXCP_PC_ALIGN: 4828bdcc096SAndrew Turner call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr); 4838bdcc096SAndrew Turner userret(td, frame); 4848bdcc096SAndrew Turner break; 48505f39d1aSAndrew Turner case EXCP_BRKPT_EL0: 4860987c184SAndrew Turner case EXCP_BRK: 4870987c184SAndrew Turner call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr); 4880987c184SAndrew Turner userret(td, frame); 4890987c184SAndrew Turner break; 49013db6962SAndrew Turner case EXCP_MSR: 49113db6962SAndrew Turner call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr); 49213db6962SAndrew Turner userret(td, frame); 49313db6962SAndrew Turner break; 49487e19994SAndrew Turner case EXCP_SOFTSTP_EL0: 49587e19994SAndrew Turner td->td_frame->tf_spsr &= ~PSR_SS; 49687e19994SAndrew Turner td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 497fc232b89SAndrew Turner WRITE_SPECIALREG(mdscr_el1, 498fc232b89SAndrew Turner READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_SS); 49987e19994SAndrew Turner call_trapsignal(td, SIGTRAP, TRAP_TRACE, 50087e19994SAndrew Turner (void *)frame->tf_elr); 50187e19994SAndrew Turner userret(td, frame); 50287e19994SAndrew Turner break; 503e5acd89cSAndrew Turner default: 504e0c6c1d1SAndrew Turner call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr); 505e0c6c1d1SAndrew Turner userret(td, frame); 506e0c6c1d1SAndrew Turner break; 507e5acd89cSAndrew Turner } 5084c247b97SAndrew Turner 509d7635c7aSAndrew Turner KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, 5104c247b97SAndrew Turner ("Kernel VFP flags set while entering userspace")); 5116ed982a2SAndrew Turner KASSERT( 512d7635c7aSAndrew Turner td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate, 5136ed982a2SAndrew Turner ("Kernel VFP state in use when entering userspace")); 514e5acd89cSAndrew Turner } 515e5acd89cSAndrew Turner 516dc9b99a8SAndrew Turner /* 517dc9b99a8SAndrew Turner * TODO: We will need to handle these later when we support ARMv8.2 RAS. 518dc9b99a8SAndrew Turner */ 519e5acd89cSAndrew Turner void 520dc9b99a8SAndrew Turner do_serror(struct trapframe *frame) 521e5acd89cSAndrew Turner { 522dc9b99a8SAndrew Turner uint64_t esr, far; 523e5acd89cSAndrew Turner 524dc9b99a8SAndrew Turner far = READ_SPECIALREG(far_el1); 525dc9b99a8SAndrew Turner esr = frame->tf_esr; 526dc9b99a8SAndrew Turner 527dc9b99a8SAndrew Turner print_registers(frame); 528dc9b99a8SAndrew Turner printf(" far: %16lx\n", far); 529dc9b99a8SAndrew Turner printf(" esr: %.8lx\n", esr); 530dc9b99a8SAndrew Turner panic("Unhandled System Error"); 531e5acd89cSAndrew Turner } 532e5acd89cSAndrew Turner 533dc9b99a8SAndrew Turner void 534dc9b99a8SAndrew Turner unhandled_exception(struct trapframe *frame) 535dc9b99a8SAndrew Turner { 536dc9b99a8SAndrew Turner uint64_t esr, far; 537dc9b99a8SAndrew Turner 538dc9b99a8SAndrew Turner far = READ_SPECIALREG(far_el1); 539dc9b99a8SAndrew Turner esr = frame->tf_esr; 540dc9b99a8SAndrew Turner 541dc9b99a8SAndrew Turner print_registers(frame); 542dc9b99a8SAndrew Turner printf(" far: %16lx\n", far); 543dc9b99a8SAndrew Turner printf(" esr: %.8lx\n", esr); 544dc9b99a8SAndrew Turner panic("Unhandled exception"); 545dc9b99a8SAndrew Turner } 546