1e5acd89cSAndrew Turner /*- 2e5acd89cSAndrew Turner * Copyright (c) 2014 Andrew Turner 3e5acd89cSAndrew Turner * All rights reserved. 4e5acd89cSAndrew Turner * 5e5acd89cSAndrew Turner * Redistribution and use in source and binary forms, with or without 6e5acd89cSAndrew Turner * modification, are permitted provided that the following conditions 7e5acd89cSAndrew Turner * are met: 8e5acd89cSAndrew Turner * 1. Redistributions of source code must retain the above copyright 9e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer. 10e5acd89cSAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright 11e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer in the 12e5acd89cSAndrew Turner * documentation and/or other materials provided with the distribution. 13e5acd89cSAndrew Turner * 14e5acd89cSAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e5acd89cSAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e5acd89cSAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e5acd89cSAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e5acd89cSAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e5acd89cSAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e5acd89cSAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e5acd89cSAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e5acd89cSAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e5acd89cSAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e5acd89cSAndrew Turner * SUCH DAMAGE. 25e5acd89cSAndrew Turner * 26e5acd89cSAndrew Turner */ 27e5acd89cSAndrew Turner 28e5acd89cSAndrew Turner #include <sys/cdefs.h> 29e5acd89cSAndrew Turner __FBSDID("$FreeBSD$"); 30e5acd89cSAndrew Turner 31e5acd89cSAndrew Turner #include <sys/param.h> 32e5acd89cSAndrew Turner #include <sys/systm.h> 33e5acd89cSAndrew Turner #include <sys/kernel.h> 34e5acd89cSAndrew Turner #include <sys/lock.h> 35e5acd89cSAndrew Turner #include <sys/mutex.h> 36e5acd89cSAndrew Turner #include <sys/pioctl.h> 37e5acd89cSAndrew Turner #include <sys/proc.h> 38e5acd89cSAndrew Turner #include <sys/ptrace.h> 39e5acd89cSAndrew Turner #include <sys/syscall.h> 40e5acd89cSAndrew Turner #include <sys/sysent.h> 41e5acd89cSAndrew Turner #ifdef KDB 42e5acd89cSAndrew Turner #include <sys/kdb.h> 43e5acd89cSAndrew Turner #endif 44e5acd89cSAndrew Turner 45e5acd89cSAndrew Turner #include <vm/vm.h> 46e5acd89cSAndrew Turner #include <vm/pmap.h> 47e5acd89cSAndrew Turner #include <vm/vm_kern.h> 48e5acd89cSAndrew Turner #include <vm/vm_map.h> 49cb02f6b9SAndrew Turner #include <vm/vm_param.h> 50e5acd89cSAndrew Turner #include <vm/vm_extern.h> 51e5acd89cSAndrew Turner 52e5acd89cSAndrew Turner #include <machine/frame.h> 53e5acd89cSAndrew Turner #include <machine/pcb.h> 54e5acd89cSAndrew Turner #include <machine/pcpu.h> 55e5acd89cSAndrew Turner #include <machine/vmparam.h> 56e5acd89cSAndrew Turner 57b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 58b78ee15eSRuslan Bukin #include <sys/dtrace_bsd.h> 59b78ee15eSRuslan Bukin #endif 60b78ee15eSRuslan Bukin 61e5acd89cSAndrew Turner #ifdef VFP 62e5acd89cSAndrew Turner #include <machine/vfp.h> 63e5acd89cSAndrew Turner #endif 64e5acd89cSAndrew Turner 65e5acd89cSAndrew Turner #ifdef KDB 66e5acd89cSAndrew Turner #include <machine/db_machdep.h> 67e5acd89cSAndrew Turner #endif 68e5acd89cSAndrew Turner 69e5acd89cSAndrew Turner #ifdef DDB 70e5acd89cSAndrew Turner #include <ddb/db_output.h> 71e5acd89cSAndrew Turner #endif 72e5acd89cSAndrew Turner 739d77aa2aSAndrew Turner extern register_t fsu_intr_fault; 74e5acd89cSAndrew Turner 75e5acd89cSAndrew Turner /* Called from exception.S */ 76e5acd89cSAndrew Turner void do_el1h_sync(struct trapframe *); 77e5acd89cSAndrew Turner void do_el0_sync(struct trapframe *); 78e5acd89cSAndrew Turner void do_el0_error(struct trapframe *); 79e5acd89cSAndrew Turner 80b78ee15eSRuslan Bukin int (*dtrace_invop_jump_addr)(struct trapframe *); 81b78ee15eSRuslan Bukin 82e5acd89cSAndrew Turner static __inline void 83cb02f6b9SAndrew Turner call_trapsignal(struct thread *td, int sig, int code, void *addr) 84e5acd89cSAndrew Turner { 85e5acd89cSAndrew Turner ksiginfo_t ksi; 86e5acd89cSAndrew Turner 87e5acd89cSAndrew Turner ksiginfo_init_trap(&ksi); 88e5acd89cSAndrew Turner ksi.ksi_signo = sig; 89cb02f6b9SAndrew Turner ksi.ksi_code = code; 90cb02f6b9SAndrew Turner ksi.ksi_addr = addr; 91e5acd89cSAndrew Turner trapsignal(td, &ksi); 92e5acd89cSAndrew Turner } 93e5acd89cSAndrew Turner 94e5acd89cSAndrew Turner int 95e5acd89cSAndrew Turner cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) 96e5acd89cSAndrew Turner { 97e5acd89cSAndrew Turner struct proc *p; 98e5acd89cSAndrew Turner register_t *ap; 99e5acd89cSAndrew Turner int nap; 100e5acd89cSAndrew Turner 101e5acd89cSAndrew Turner nap = 8; 102e5acd89cSAndrew Turner p = td->td_proc; 103e5acd89cSAndrew Turner ap = td->td_frame->tf_x; 104e5acd89cSAndrew Turner 105e5acd89cSAndrew Turner sa->code = td->td_frame->tf_x[8]; 106e5acd89cSAndrew Turner 107e5acd89cSAndrew Turner if (sa->code == SYS_syscall || sa->code == SYS___syscall) { 108e5acd89cSAndrew Turner sa->code = *ap++; 109e5acd89cSAndrew Turner nap--; 110e5acd89cSAndrew Turner } 111e5acd89cSAndrew Turner 112e5acd89cSAndrew Turner if (p->p_sysent->sv_mask) 113e5acd89cSAndrew Turner sa->code &= p->p_sysent->sv_mask; 114e5acd89cSAndrew Turner if (sa->code >= p->p_sysent->sv_size) 115e5acd89cSAndrew Turner sa->callp = &p->p_sysent->sv_table[0]; 116e5acd89cSAndrew Turner else 117e5acd89cSAndrew Turner sa->callp = &p->p_sysent->sv_table[sa->code]; 118e5acd89cSAndrew Turner 119e5acd89cSAndrew Turner sa->narg = sa->callp->sy_narg; 120e5acd89cSAndrew Turner memcpy(sa->args, ap, nap * sizeof(register_t)); 121e5acd89cSAndrew Turner if (sa->narg > nap) 122c547d650SEd Maste panic("ARM64TODO: Could we have more then 8 args?"); 123e5acd89cSAndrew Turner 124e5acd89cSAndrew Turner td->td_retval[0] = 0; 125e5acd89cSAndrew Turner td->td_retval[1] = 0; 126e5acd89cSAndrew Turner 127e5acd89cSAndrew Turner return (0); 128e5acd89cSAndrew Turner } 129e5acd89cSAndrew Turner 130e5acd89cSAndrew Turner #include "../../kern/subr_syscall.c" 131e5acd89cSAndrew Turner 132e5acd89cSAndrew Turner static void 133e5acd89cSAndrew Turner svc_handler(struct trapframe *frame) 134e5acd89cSAndrew Turner { 135e5acd89cSAndrew Turner struct syscall_args sa; 136e5acd89cSAndrew Turner struct thread *td; 137e5acd89cSAndrew Turner int error; 138e5acd89cSAndrew Turner 139e5acd89cSAndrew Turner td = curthread; 140e5acd89cSAndrew Turner td->td_frame = frame; 141e5acd89cSAndrew Turner 142e5acd89cSAndrew Turner error = syscallenter(td, &sa); 143e5acd89cSAndrew Turner syscallret(td, error, &sa); 144e5acd89cSAndrew Turner } 145e5acd89cSAndrew Turner 146e5acd89cSAndrew Turner static void 147e5acd89cSAndrew Turner data_abort(struct trapframe *frame, uint64_t esr, int lower) 148e5acd89cSAndrew Turner { 149e5acd89cSAndrew Turner struct vm_map *map; 150e5acd89cSAndrew Turner struct thread *td; 151e5acd89cSAndrew Turner struct proc *p; 152e5acd89cSAndrew Turner struct pcb *pcb; 153e5acd89cSAndrew Turner vm_prot_t ftype; 154e5acd89cSAndrew Turner vm_offset_t va; 155e5acd89cSAndrew Turner uint64_t far; 156cb02f6b9SAndrew Turner int error, sig, ucode; 157e5acd89cSAndrew Turner 158e5acd89cSAndrew Turner td = curthread; 159e5acd89cSAndrew Turner pcb = td->td_pcb; 160e5acd89cSAndrew Turner 161e5acd89cSAndrew Turner /* 162e5acd89cSAndrew Turner * Special case for fuswintr and suswintr. These can't sleep so 163e5acd89cSAndrew Turner * handle them early on in the trap handler. 164e5acd89cSAndrew Turner */ 1659d77aa2aSAndrew Turner if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) { 166e5acd89cSAndrew Turner frame->tf_elr = pcb->pcb_onfault; 167e5acd89cSAndrew Turner return; 168e5acd89cSAndrew Turner } 169e5acd89cSAndrew Turner 170e5acd89cSAndrew Turner far = READ_SPECIALREG(far_el1); 171e5acd89cSAndrew Turner p = td->td_proc; 172e5acd89cSAndrew Turner 173e5acd89cSAndrew Turner if (lower) 174e5acd89cSAndrew Turner map = &td->td_proc->p_vmspace->vm_map; 175e5acd89cSAndrew Turner else { 176e5acd89cSAndrew Turner /* The top bit tells us which range to use */ 177e5acd89cSAndrew Turner if ((far >> 63) == 1) 178e5acd89cSAndrew Turner map = kernel_map; 179e5acd89cSAndrew Turner else 180e5acd89cSAndrew Turner map = &td->td_proc->p_vmspace->vm_map; 181e5acd89cSAndrew Turner } 182e5acd89cSAndrew Turner 183e5acd89cSAndrew Turner va = trunc_page(far); 184e5acd89cSAndrew Turner ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 185e5acd89cSAndrew Turner 186e5acd89cSAndrew Turner if (map != kernel_map) { 187e5acd89cSAndrew Turner /* 188e5acd89cSAndrew Turner * Keep swapout from messing with us during this 189e5acd89cSAndrew Turner * critical time. 190e5acd89cSAndrew Turner */ 191e5acd89cSAndrew Turner PROC_LOCK(p); 192e5acd89cSAndrew Turner ++p->p_lock; 193e5acd89cSAndrew Turner PROC_UNLOCK(p); 194e5acd89cSAndrew Turner 195e5acd89cSAndrew Turner /* Fault in the user page: */ 196e5acd89cSAndrew Turner error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 197e5acd89cSAndrew Turner 198e5acd89cSAndrew Turner PROC_LOCK(p); 199e5acd89cSAndrew Turner --p->p_lock; 200e5acd89cSAndrew Turner PROC_UNLOCK(p); 201e5acd89cSAndrew Turner } else { 202e5acd89cSAndrew Turner /* 203e5acd89cSAndrew Turner * Don't have to worry about process locking or stacks in the 204e5acd89cSAndrew Turner * kernel. 205e5acd89cSAndrew Turner */ 206e5acd89cSAndrew Turner error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 207e5acd89cSAndrew Turner } 208e5acd89cSAndrew Turner 209cb02f6b9SAndrew Turner if (error != KERN_SUCCESS) { 210e5acd89cSAndrew Turner if (lower) { 211e5acd89cSAndrew Turner sig = SIGSEGV; 212cb02f6b9SAndrew Turner if (error == KERN_PROTECTION_FAILURE) 213cb02f6b9SAndrew Turner ucode = SEGV_ACCERR; 214cb02f6b9SAndrew Turner else 215cb02f6b9SAndrew Turner ucode = SEGV_MAPERR; 216cb02f6b9SAndrew Turner call_trapsignal(td, sig, ucode, (void *)far); 217e5acd89cSAndrew Turner } else { 218e5acd89cSAndrew Turner if (td->td_intr_nesting_level == 0 && 219e5acd89cSAndrew Turner pcb->pcb_onfault != 0) { 220e5acd89cSAndrew Turner frame->tf_x[0] = error; 221e5acd89cSAndrew Turner frame->tf_elr = pcb->pcb_onfault; 222e5acd89cSAndrew Turner return; 223e5acd89cSAndrew Turner } 224e5acd89cSAndrew Turner panic("vm_fault failed: %lx", frame->tf_elr); 225e5acd89cSAndrew Turner } 226e5acd89cSAndrew Turner } 227e5acd89cSAndrew Turner 228e5acd89cSAndrew Turner if (lower) 229e5acd89cSAndrew Turner userret(td, frame); 230e5acd89cSAndrew Turner } 231e5acd89cSAndrew Turner 232e5acd89cSAndrew Turner void 233e5acd89cSAndrew Turner do_el1h_sync(struct trapframe *frame) 234e5acd89cSAndrew Turner { 235e5acd89cSAndrew Turner uint32_t exception; 236e5acd89cSAndrew Turner uint64_t esr; 237e5acd89cSAndrew Turner 238e5acd89cSAndrew Turner /* Read the esr register to get the exception details */ 239e5acd89cSAndrew Turner esr = READ_SPECIALREG(esr_el1); 240e5acd89cSAndrew Turner exception = ESR_ELx_EXCEPTION(esr); 241e5acd89cSAndrew Turner 242b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 243b78ee15eSRuslan Bukin if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 244b78ee15eSRuslan Bukin return; 245b78ee15eSRuslan Bukin #endif 246b78ee15eSRuslan Bukin 247e5acd89cSAndrew Turner /* 248e5acd89cSAndrew Turner * Sanity check we are in an exception er can handle. The IL bit 249e5acd89cSAndrew Turner * is used to indicate the instruction length, except in a few 250e5acd89cSAndrew Turner * exceptions described in the ARMv8 ARM. 251e5acd89cSAndrew Turner * 252e5acd89cSAndrew Turner * It is unclear in some cases if the bit is implementation defined. 253e5acd89cSAndrew Turner * The Foundation Model and QEMU disagree on if the IL bit should 254e5acd89cSAndrew Turner * be set when we are in a data fault from the same EL and the ISV 255e5acd89cSAndrew Turner * bit (bit 24) is also set. 256e5acd89cSAndrew Turner */ 257e5acd89cSAndrew Turner KASSERT((esr & ESR_ELx_IL) == ESR_ELx_IL || 258e5acd89cSAndrew Turner (exception == EXCP_DATA_ABORT && ((esr & ISS_DATA_ISV) == 0)), 259e5acd89cSAndrew Turner ("Invalid instruction length in exception")); 260e5acd89cSAndrew Turner 2613ad7e84eSAndrew Turner CTR4(KTR_TRAP, 2623ad7e84eSAndrew Turner "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", 2633ad7e84eSAndrew Turner curthread, esr, frame->tf_elr, frame); 2643ad7e84eSAndrew Turner 265e5acd89cSAndrew Turner switch(exception) { 266e5acd89cSAndrew Turner case EXCP_FP_SIMD: 267e5acd89cSAndrew Turner case EXCP_TRAP_FP: 268e5acd89cSAndrew Turner panic("VFP exception in the kernel"); 269e5acd89cSAndrew Turner case EXCP_DATA_ABORT: 270e5acd89cSAndrew Turner data_abort(frame, esr, 0); 271e5acd89cSAndrew Turner break; 272e5acd89cSAndrew Turner case EXCP_BRK: 273b78ee15eSRuslan Bukin #ifdef KDTRACE_HOOKS 274b78ee15eSRuslan Bukin if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ 275b78ee15eSRuslan Bukin dtrace_invop_jump_addr != 0) { 276b78ee15eSRuslan Bukin dtrace_invop_jump_addr(frame); 277b78ee15eSRuslan Bukin break; 278b78ee15eSRuslan Bukin } 279b78ee15eSRuslan Bukin #endif 280e5acd89cSAndrew Turner case EXCP_WATCHPT_EL1: 281e5acd89cSAndrew Turner case EXCP_SOFTSTP_EL1: 282e5acd89cSAndrew Turner #ifdef KDB 283e5acd89cSAndrew Turner kdb_trap(exception, 0, frame); 284e5acd89cSAndrew Turner #else 285e5acd89cSAndrew Turner panic("No debugger in kernel.\n"); 286e5acd89cSAndrew Turner #endif 287e5acd89cSAndrew Turner break; 288e5acd89cSAndrew Turner default: 289e5acd89cSAndrew Turner panic("Unknown kernel exception %x esr_el1 %lx\n", exception, 290e5acd89cSAndrew Turner esr); 291e5acd89cSAndrew Turner } 292e5acd89cSAndrew Turner } 293e5acd89cSAndrew Turner 294e5acd89cSAndrew Turner void 295e5acd89cSAndrew Turner do_el0_sync(struct trapframe *frame) 296e5acd89cSAndrew Turner { 297e5acd89cSAndrew Turner uint32_t exception; 298e5acd89cSAndrew Turner uint64_t esr; 299e5acd89cSAndrew Turner 300e5acd89cSAndrew Turner /* Check we have a sane environment when entering from userland */ 301e5acd89cSAndrew Turner KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, 302e5acd89cSAndrew Turner ("Invalid pcpu address from userland: %p (tpidr %lx)", 303e5acd89cSAndrew Turner get_pcpu(), READ_SPECIALREG(tpidr_el1))); 304e5acd89cSAndrew Turner 305e5acd89cSAndrew Turner esr = READ_SPECIALREG(esr_el1); 306e5acd89cSAndrew Turner exception = ESR_ELx_EXCEPTION(esr); 307e5acd89cSAndrew Turner 3083ad7e84eSAndrew Turner CTR4(KTR_TRAP, 3093ad7e84eSAndrew Turner "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", 3103ad7e84eSAndrew Turner curthread, esr, frame->tf_elr, frame); 3113ad7e84eSAndrew Turner 312e5acd89cSAndrew Turner switch(exception) { 313e5acd89cSAndrew Turner case EXCP_FP_SIMD: 314e5acd89cSAndrew Turner case EXCP_TRAP_FP: 315e5acd89cSAndrew Turner #ifdef VFP 316e5acd89cSAndrew Turner vfp_restore_state(); 317e5acd89cSAndrew Turner #else 318e5acd89cSAndrew Turner panic("VFP exception in userland"); 319e5acd89cSAndrew Turner #endif 320e5acd89cSAndrew Turner break; 321e5acd89cSAndrew Turner case EXCP_SVC: 3229028b18fSZbigniew Bodek /* 3239028b18fSZbigniew Bodek * Ensure the svc_handler is being run with interrupts enabled. 3249028b18fSZbigniew Bodek * They will be automatically restored when returning from 3259028b18fSZbigniew Bodek * exception handler. 3269028b18fSZbigniew Bodek */ 3279028b18fSZbigniew Bodek intr_enable(); 328e5acd89cSAndrew Turner svc_handler(frame); 329e5acd89cSAndrew Turner break; 330e5acd89cSAndrew Turner case EXCP_INSN_ABORT_L: 331e5acd89cSAndrew Turner case EXCP_DATA_ABORT_L: 332e5acd89cSAndrew Turner data_abort(frame, esr, 1); 333e5acd89cSAndrew Turner break; 334e5acd89cSAndrew Turner default: 335e5acd89cSAndrew Turner panic("Unknown userland exception %x esr_el1 %lx\n", exception, 336e5acd89cSAndrew Turner esr); 337e5acd89cSAndrew Turner } 338e5acd89cSAndrew Turner } 339e5acd89cSAndrew Turner 340e5acd89cSAndrew Turner void 341e5acd89cSAndrew Turner do_el0_error(struct trapframe *frame) 342e5acd89cSAndrew Turner { 343e5acd89cSAndrew Turner 344c547d650SEd Maste panic("ARM64TODO: do_el0_error"); 345e5acd89cSAndrew Turner } 346e5acd89cSAndrew Turner 347