1e5acd89cSAndrew Turner /*- 2e5acd89cSAndrew Turner * Copyright (c) 2014 Andrew Turner 3e5acd89cSAndrew Turner * All rights reserved. 4e5acd89cSAndrew Turner * 5e5acd89cSAndrew Turner * Redistribution and use in source and binary forms, with or without 6e5acd89cSAndrew Turner * modification, are permitted provided that the following conditions 7e5acd89cSAndrew Turner * are met: 8e5acd89cSAndrew Turner * 1. Redistributions of source code must retain the above copyright 9e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer. 10e5acd89cSAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright 11e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer in the 12e5acd89cSAndrew Turner * documentation and/or other materials provided with the distribution. 13e5acd89cSAndrew Turner * 14e5acd89cSAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e5acd89cSAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e5acd89cSAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e5acd89cSAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e5acd89cSAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e5acd89cSAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e5acd89cSAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e5acd89cSAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e5acd89cSAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e5acd89cSAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e5acd89cSAndrew Turner * SUCH DAMAGE. 25e5acd89cSAndrew Turner * 26e5acd89cSAndrew Turner */ 27e5acd89cSAndrew Turner 28574a7c6cSAndrew Turner #include "opt_platform.h" 29574a7c6cSAndrew Turner 30e5acd89cSAndrew Turner #include <sys/cdefs.h> 31e5acd89cSAndrew Turner __FBSDID("$FreeBSD$"); 32e5acd89cSAndrew Turner 33e5acd89cSAndrew Turner #include <sys/param.h> 34e5acd89cSAndrew Turner #include <sys/systm.h> 35e5acd89cSAndrew Turner #include <sys/limits.h> 36e5acd89cSAndrew Turner #include <sys/proc.h> 37e5acd89cSAndrew Turner #include <sys/sf_buf.h> 38e5acd89cSAndrew Turner #include <sys/signal.h> 398e5d76e6SAndrew Turner #include <sys/sysent.h> 40e5acd89cSAndrew Turner #include <sys/unistd.h> 41e5acd89cSAndrew Turner 42e5acd89cSAndrew Turner #include <vm/vm.h> 43e5acd89cSAndrew Turner #include <vm/vm_page.h> 44e5acd89cSAndrew Turner #include <vm/vm_map.h> 45e5acd89cSAndrew Turner #include <vm/uma.h> 46e5acd89cSAndrew Turner #include <vm/uma_int.h> 47e5acd89cSAndrew Turner 48e5acd89cSAndrew Turner #include <machine/armreg.h> 49e5acd89cSAndrew Turner #include <machine/cpu.h> 509615213bSAndrew Turner #include <machine/md_var.h> 51e5acd89cSAndrew Turner #include <machine/pcb.h> 52e5acd89cSAndrew Turner #include <machine/frame.h> 53e5acd89cSAndrew Turner 542db317caSAndrew Turner #ifdef VFP 552db317caSAndrew Turner #include <machine/vfp.h> 562db317caSAndrew Turner #endif 572db317caSAndrew Turner 58574a7c6cSAndrew Turner #include <dev/psci/psci.h> 59574a7c6cSAndrew Turner 60e5acd89cSAndrew Turner /* 61e5acd89cSAndrew Turner * Finish a fork operation, with process p2 nearly set up. 62e5acd89cSAndrew Turner * Copy and update the pcb, set up the stack so that the child 63e5acd89cSAndrew Turner * ready to run and return to user mode. 64e5acd89cSAndrew Turner */ 65e5acd89cSAndrew Turner void 66e5acd89cSAndrew Turner cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) 67e5acd89cSAndrew Turner { 68e5acd89cSAndrew Turner struct pcb *pcb2; 69e5acd89cSAndrew Turner struct trapframe *tf; 70e5acd89cSAndrew Turner 71e5acd89cSAndrew Turner if ((flags & RFPROC) == 0) 72e5acd89cSAndrew Turner return; 73e5acd89cSAndrew Turner 742db317caSAndrew Turner if (td1 == curthread) { 752db317caSAndrew Turner /* 762db317caSAndrew Turner * Save the tpidr_el0 and the vfp state, these normally happen 772db317caSAndrew Turner * in cpu_switch, but if userland changes these then forks 782db317caSAndrew Turner * this may not have happened. 792db317caSAndrew Turner */ 802db317caSAndrew Turner td1->td_pcb->pcb_tpidr_el0 = READ_SPECIALREG(tpidr_el0); 818c9c3144SOlivier Houchard td1->td_pcb->pcb_tpidrro_el0 = READ_SPECIALREG(tpidrro_el0); 822db317caSAndrew Turner #ifdef VFP 832db317caSAndrew Turner if ((td1->td_pcb->pcb_fpflags & PCB_FP_STARTED) != 0) 84f692e325SAndrew Turner vfp_save_state(td1, td1->td_pcb); 852db317caSAndrew Turner #endif 862db317caSAndrew Turner } 872db317caSAndrew Turner 88e5acd89cSAndrew Turner pcb2 = (struct pcb *)(td2->td_kstack + 89e5acd89cSAndrew Turner td2->td_kstack_pages * PAGE_SIZE) - 1; 90e5acd89cSAndrew Turner 91e5acd89cSAndrew Turner td2->td_pcb = pcb2; 92e5acd89cSAndrew Turner bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); 93e5acd89cSAndrew Turner 941fd001dbSMitchell Horne /* Clear the debug register state. */ 951fd001dbSMitchell Horne bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs)); 961fd001dbSMitchell Horne 9785b7c566SAndrew Turner ptrauth_fork(td2, td1); 9885b7c566SAndrew Turner 99e5acd89cSAndrew Turner tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1); 100e5acd89cSAndrew Turner bcopy(td1->td_frame, tf, sizeof(*tf)); 101e5acd89cSAndrew Turner tf->tf_x[0] = 0; 102e5acd89cSAndrew Turner tf->tf_x[1] = 0; 103739e4482SAndrew Turner tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF); 104e5acd89cSAndrew Turner 105e5acd89cSAndrew Turner td2->td_frame = tf; 106e5acd89cSAndrew Turner 107e5acd89cSAndrew Turner /* Set the return value registers for fork() */ 1081c33a94aSAndrew Turner td2->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return; 1091c33a94aSAndrew Turner td2->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td2; 1101c1f31a5SAndrew Turner td2->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline; 111e5acd89cSAndrew Turner td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; 112baf8f20aSAndrew Turner 113baf8f20aSAndrew Turner vfp_new_thread(td2, td1, true); 114e5acd89cSAndrew Turner 115e5acd89cSAndrew Turner /* Setup to release spin count in fork_exit(). */ 116e5acd89cSAndrew Turner td2->td_md.md_spinlock_count = 1; 11717b6ee96SAndrew Turner td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT; 118ae92ace0SAndrew Turner 119ae92ace0SAndrew Turner #if defined(PERTHREAD_SSP) 120ae92ace0SAndrew Turner /* Set the new canary */ 121ae92ace0SAndrew Turner arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary)); 122ae92ace0SAndrew Turner #endif 123e5acd89cSAndrew Turner } 124e5acd89cSAndrew Turner 125e5acd89cSAndrew Turner void 126e5acd89cSAndrew Turner cpu_reset(void) 127e5acd89cSAndrew Turner { 128e5acd89cSAndrew Turner 129574a7c6cSAndrew Turner psci_reset(); 130574a7c6cSAndrew Turner 131574a7c6cSAndrew Turner printf("cpu_reset failed"); 132e5acd89cSAndrew Turner while(1) 133e5acd89cSAndrew Turner __asm volatile("wfi" ::: "memory"); 134e5acd89cSAndrew Turner } 135e5acd89cSAndrew Turner 136e5acd89cSAndrew Turner void 137e5acd89cSAndrew Turner cpu_thread_swapin(struct thread *td) 138e5acd89cSAndrew Turner { 139e5acd89cSAndrew Turner } 140e5acd89cSAndrew Turner 141e5acd89cSAndrew Turner void 142e5acd89cSAndrew Turner cpu_thread_swapout(struct thread *td) 143e5acd89cSAndrew Turner { 144e5acd89cSAndrew Turner } 145e5acd89cSAndrew Turner 146e5acd89cSAndrew Turner void 147e5acd89cSAndrew Turner cpu_set_syscall_retval(struct thread *td, int error) 148e5acd89cSAndrew Turner { 149e5acd89cSAndrew Turner struct trapframe *frame; 150e5acd89cSAndrew Turner 151e5acd89cSAndrew Turner frame = td->td_frame; 152e5acd89cSAndrew Turner 153fb8c2f74SEdward Tomasz Napierala if (__predict_true(error == 0)) { 154e5acd89cSAndrew Turner frame->tf_x[0] = td->td_retval[0]; 155e5acd89cSAndrew Turner frame->tf_x[1] = td->td_retval[1]; 156e5acd89cSAndrew Turner frame->tf_spsr &= ~PSR_C; /* carry bit */ 157fb8c2f74SEdward Tomasz Napierala return; 158fb8c2f74SEdward Tomasz Napierala } 159fb8c2f74SEdward Tomasz Napierala 160fb8c2f74SEdward Tomasz Napierala switch (error) { 161e5acd89cSAndrew Turner case ERESTART: 162e5acd89cSAndrew Turner frame->tf_elr -= 4; 163e5acd89cSAndrew Turner break; 164e5acd89cSAndrew Turner case EJUSTRETURN: 165e5acd89cSAndrew Turner break; 166e5acd89cSAndrew Turner default: 167e5acd89cSAndrew Turner frame->tf_spsr |= PSR_C; /* carry bit */ 168c26391f4SEdward Tomasz Napierala frame->tf_x[0] = error; 169e5acd89cSAndrew Turner break; 170e5acd89cSAndrew Turner } 171e5acd89cSAndrew Turner } 172e5acd89cSAndrew Turner 173e5acd89cSAndrew Turner /* 1745c2cf818SKonstantin Belousov * Initialize machine state, mostly pcb and trap frame for a new 1755c2cf818SKonstantin Belousov * thread, about to return to userspace. Put enough state in the new 1765c2cf818SKonstantin Belousov * thread's PCB to get it to go back to the fork_return(), which 1775c2cf818SKonstantin Belousov * finalizes the thread state and handles peculiarities of the first 1785c2cf818SKonstantin Belousov * return to userspace for the new thread. 179e5acd89cSAndrew Turner */ 180e5acd89cSAndrew Turner void 1815c2cf818SKonstantin Belousov cpu_copy_thread(struct thread *td, struct thread *td0) 182e5acd89cSAndrew Turner { 183e5acd89cSAndrew Turner bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); 184e5acd89cSAndrew Turner bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb)); 185e5acd89cSAndrew Turner 1861c33a94aSAndrew Turner td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return; 1871c33a94aSAndrew Turner td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td; 1881c1f31a5SAndrew Turner td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline; 189e5acd89cSAndrew Turner td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; 190baf8f20aSAndrew Turner 191baf8f20aSAndrew Turner /* Update VFP state for the new thread */ 192baf8f20aSAndrew Turner vfp_new_thread(td, td0, false); 193e5acd89cSAndrew Turner 194e5acd89cSAndrew Turner /* Setup to release spin count in fork_exit(). */ 195e5acd89cSAndrew Turner td->td_md.md_spinlock_count = 1; 19617b6ee96SAndrew Turner td->td_md.md_saved_daif = PSR_DAIF_DEFAULT; 197ae92ace0SAndrew Turner 198ae92ace0SAndrew Turner #if defined(PERTHREAD_SSP) 199ae92ace0SAndrew Turner /* Set the new canary */ 200ae92ace0SAndrew Turner arc4random_buf(&td->td_md.md_canary, sizeof(td->td_md.md_canary)); 201ae92ace0SAndrew Turner #endif 20285b7c566SAndrew Turner 20385b7c566SAndrew Turner /* Generate new pointer authentication keys. */ 20485b7c566SAndrew Turner ptrauth_copy_thread(td, td0); 205e5acd89cSAndrew Turner } 206e5acd89cSAndrew Turner 207e5acd89cSAndrew Turner /* 2085c2cf818SKonstantin Belousov * Set that machine state for performing an upcall that starts 2095c2cf818SKonstantin Belousov * the entry function with the given argument. 210e5acd89cSAndrew Turner */ 211e5acd89cSAndrew Turner void 2125c2cf818SKonstantin Belousov cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg, 213e5acd89cSAndrew Turner stack_t *stack) 214e5acd89cSAndrew Turner { 2155f858389SAndrew Turner struct trapframe *tf = td->td_frame; 216e5acd89cSAndrew Turner 2178c9c3144SOlivier Houchard /* 32bits processes use r13 for sp */ 21821914737SOlivier Houchard if (td->td_frame->tf_spsr & PSR_M_32) { 2198c9c3144SOlivier Houchard tf->tf_x[13] = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); 22021914737SOlivier Houchard if ((register_t)entry & 1) 22121914737SOlivier Houchard tf->tf_spsr |= PSR_T; 22221914737SOlivier Houchard } else 223aa949be5SJohn Baldwin tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); 2245f858389SAndrew Turner tf->tf_elr = (register_t)entry; 2255f858389SAndrew Turner tf->tf_x[0] = (register_t)arg; 226e5acd89cSAndrew Turner } 227e5acd89cSAndrew Turner 228e5acd89cSAndrew Turner int 229e5acd89cSAndrew Turner cpu_set_user_tls(struct thread *td, void *tls_base) 230e5acd89cSAndrew Turner { 2315f858389SAndrew Turner struct pcb *pcb; 232e5acd89cSAndrew Turner 2335f858389SAndrew Turner if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS) 2345f858389SAndrew Turner return (EINVAL); 2355f858389SAndrew Turner 2365f858389SAndrew Turner pcb = td->td_pcb; 2378c9c3144SOlivier Houchard if (td->td_frame->tf_spsr & PSR_M_32) { 2388c9c3144SOlivier Houchard /* 32bits arm stores the user TLS into tpidrro */ 2398c9c3144SOlivier Houchard pcb->pcb_tpidrro_el0 = (register_t)tls_base; 2408c9c3144SOlivier Houchard pcb->pcb_tpidr_el0 = (register_t)tls_base; 2418c9c3144SOlivier Houchard if (td == curthread) { 2428c9c3144SOlivier Houchard WRITE_SPECIALREG(tpidrro_el0, tls_base); 2438c9c3144SOlivier Houchard WRITE_SPECIALREG(tpidr_el0, tls_base); 2448c9c3144SOlivier Houchard } 2458c9c3144SOlivier Houchard } else { 2465f858389SAndrew Turner pcb->pcb_tpidr_el0 = (register_t)tls_base; 24766bd4c2eSEd Schouten if (td == curthread) 24866bd4c2eSEd Schouten WRITE_SPECIALREG(tpidr_el0, tls_base); 2498c9c3144SOlivier Houchard } 2505f858389SAndrew Turner 2515f858389SAndrew Turner return (0); 252e5acd89cSAndrew Turner } 253e5acd89cSAndrew Turner 254e5acd89cSAndrew Turner void 255e5acd89cSAndrew Turner cpu_thread_exit(struct thread *td) 256e5acd89cSAndrew Turner { 257e5acd89cSAndrew Turner } 258e5acd89cSAndrew Turner 259e5acd89cSAndrew Turner void 260e5acd89cSAndrew Turner cpu_thread_alloc(struct thread *td) 261e5acd89cSAndrew Turner { 262e5acd89cSAndrew Turner 263e5acd89cSAndrew Turner td->td_pcb = (struct pcb *)(td->td_kstack + 264e5acd89cSAndrew Turner td->td_kstack_pages * PAGE_SIZE) - 1; 265e5acd89cSAndrew Turner td->td_frame = (struct trapframe *)STACKALIGN( 26680e21aabSAndrew Turner (struct trapframe *)td->td_pcb - 1); 26785b7c566SAndrew Turner ptrauth_thread_alloc(td); 268e5acd89cSAndrew Turner } 269e5acd89cSAndrew Turner 270e5acd89cSAndrew Turner void 271e5acd89cSAndrew Turner cpu_thread_free(struct thread *td) 272e5acd89cSAndrew Turner { 273e5acd89cSAndrew Turner } 274e5acd89cSAndrew Turner 275e5acd89cSAndrew Turner void 276e5acd89cSAndrew Turner cpu_thread_clean(struct thread *td) 277e5acd89cSAndrew Turner { 278e5acd89cSAndrew Turner } 279e5acd89cSAndrew Turner 280e5acd89cSAndrew Turner /* 281e5acd89cSAndrew Turner * Intercept the return address from a freshly forked process that has NOT 282e5acd89cSAndrew Turner * been scheduled yet. 283e5acd89cSAndrew Turner * 284e5acd89cSAndrew Turner * This is needed to make kernel threads stay in kernel mode. 285e5acd89cSAndrew Turner */ 286e5acd89cSAndrew Turner void 2875c2cf818SKonstantin Belousov cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg) 288e5acd89cSAndrew Turner { 289e5acd89cSAndrew Turner 2901c33a94aSAndrew Turner td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)func; 2911c33a94aSAndrew Turner td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)arg; 292e5acd89cSAndrew Turner } 293e5acd89cSAndrew Turner 294e5acd89cSAndrew Turner void 295e5acd89cSAndrew Turner cpu_exit(struct thread *td) 296e5acd89cSAndrew Turner { 297e5acd89cSAndrew Turner } 298e5acd89cSAndrew Turner 2996f1fe330SKonstantin Belousov bool 3006f1fe330SKonstantin Belousov cpu_exec_vmspace_reuse(struct proc *p __unused, vm_map_t map __unused) 3016f1fe330SKonstantin Belousov { 3026f1fe330SKonstantin Belousov 3036f1fe330SKonstantin Belousov return (true); 3046f1fe330SKonstantin Belousov } 3056f1fe330SKonstantin Belousov 306fd8d844fSKonstantin Belousov int 307fd8d844fSKonstantin Belousov cpu_procctl(struct thread *td __unused, int idtype __unused, id_t id __unused, 308fd8d844fSKonstantin Belousov int com __unused, void *data __unused) 309fd8d844fSKonstantin Belousov { 310fd8d844fSKonstantin Belousov 311fd8d844fSKonstantin Belousov return (EINVAL); 312fd8d844fSKonstantin Belousov } 313