1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Andrew Turner under sponsorship from 7 * the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include "assym.inc" 33#include "opt_kstack_pages.h" 34#include "opt_sched.h" 35 36#include <machine/asm.h> 37#include <machine/armreg.h> 38 39__FBSDID("$FreeBSD$"); 40 41.macro clear_step_flag pcbflags, tmp 42 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 43 mrs \tmp, mdscr_el1 44 bic \tmp, \tmp, #MDSCR_SS 45 msr mdscr_el1, \tmp 46 isb 47999: 48.endm 49 50.macro set_step_flag pcbflags, tmp 51 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 52 mrs \tmp, mdscr_el1 53 orr \tmp, \tmp, #MDSCR_SS 54 msr mdscr_el1, \tmp 55 isb 56999: 57.endm 58 59/* 60 * void cpu_throw(struct thread *old, struct thread *new) 61 */ 62ENTRY(cpu_throw) 63 /* Of old == NULL skip disabling stepping */ 64 cbz x0, 1f 65 66 /* If we were single stepping, disable it */ 67 ldr x4, [x0, #TD_PCB] 68 ldr w5, [x4, #PCB_FLAGS] 69 clear_step_flag w5, x6 701: 71 72#ifdef VFP 73 /* Backup the new thread pointer around a call to C code */ 74 mov x19, x1 75 bl vfp_discard 76 mov x0, x19 77#else 78 mov x0, x1 79#endif 80 81 /* This returns the thread pointer so no need to save it */ 82 bl ptrauth_switch 83#ifdef PERTHREAD_SSP 84 mov x19, x0 85#endif 86 /* This returns the thread pcb */ 87 bl pmap_switch 88 mov x4, x0 89#ifdef PERTHREAD_SSP 90 /* Update the per-thread stack canary pointer. */ 91 add x19, x19, #(TD_MD_CANARY) 92 msr sp_el0, x19 93#endif 94 95 /* If we are single stepping, enable it */ 96 ldr w5, [x4, #PCB_FLAGS] 97 set_step_flag w5, x6 98 99 /* Restore the registers */ 100 ldp x5, x6, [x4, #PCB_SP] 101 mov sp, x5 102 msr tpidr_el0, x6 103 ldr x6, [x4, #PCB_TPIDRRO] 104 msr tpidrro_el0, x6 105 ldp x19, x20, [x4, #PCB_REGS + 19 * 8] 106 ldp x21, x22, [x4, #PCB_REGS + 21 * 8] 107 ldp x23, x24, [x4, #PCB_REGS + 23 * 8] 108 ldp x25, x26, [x4, #PCB_REGS + 25 * 8] 109 ldp x27, x28, [x4, #PCB_REGS + 27 * 8] 110 ldp x29, lr, [x4, #PCB_REGS + 29 * 8] 111 112 ret 113END(cpu_throw) 114 115/* 116 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 117 * 118 * x0 = old 119 * x1 = new 120 * x2 = mtx 121 * x3 to x7, x16 and x17 are caller saved 122 */ 123ENTRY(cpu_switch) 124 /* 125 * Save the old context. 126 */ 127 ldr x4, [x0, #TD_PCB] 128 129 /* Store the callee-saved registers */ 130 stp x19, x20, [x4, #PCB_REGS + 19 * 8] 131 stp x21, x22, [x4, #PCB_REGS + 21 * 8] 132 stp x23, x24, [x4, #PCB_REGS + 23 * 8] 133 stp x25, x26, [x4, #PCB_REGS + 25 * 8] 134 stp x27, x28, [x4, #PCB_REGS + 27 * 8] 135 stp x29, lr, [x4, #PCB_REGS + 29 * 8] 136 /* And the old stack pointer */ 137 mov x5, sp 138 mrs x6, tpidrro_el0 139 str x6, [x4, #PCB_TPIDRRO] 140 mrs x6, tpidr_el0 141 stp x5, x6, [x4, #PCB_SP] 142 143 /* If we were single stepping, disable it */ 144 ldr w5, [x4, #PCB_FLAGS] 145 clear_step_flag w5, x6 146 147 mov x19, x0 148 mov x20, x1 149 mov x21, x2 150 151#ifdef VFP 152 /* Load the pcb address */ 153 mov x1, x4 154 bl vfp_save_state 155 mov x0, x20 156#else 157 mov x0, x1 158#endif 159 160 /* This returns the thread pointer so no need to save it */ 161 bl ptrauth_switch 162 /* This returns the thread pcb */ 163 bl pmap_switch 164 /* Move the new pcb out of the way */ 165 mov x4, x0 166 167 mov x2, x21 168 mov x1, x20 169 mov x0, x19 170#ifdef PERTHREAD_SSP 171 /* Update the per-thread stack canary pointer. */ 172 add x20, x20, #(TD_MD_CANARY) 173 msr sp_el0, x20 174#endif 175 176 /* 177 * Release the old thread. 178 */ 179 stlr x2, [x0, #TD_LOCK] 180#if defined(SCHED_ULE) && defined(SMP) 181 /* Spin if TD_LOCK points to a blocked_lock */ 182 ldr x2, =_C_LABEL(blocked_lock) 1831: 184 ldar x3, [x1, #TD_LOCK] 185 cmp x3, x2 186 b.eq 1b 187#endif 188 189 /* If we are single stepping, enable it */ 190 ldr w5, [x4, #PCB_FLAGS] 191 set_step_flag w5, x6 192 193 /* Restore the registers */ 194 ldp x5, x6, [x4, #PCB_SP] 195 mov sp, x5 196 msr tpidr_el0, x6 197 ldr x6, [x4, #PCB_TPIDRRO] 198 msr tpidrro_el0, x6 199 ldp x19, x20, [x4, #PCB_REGS + 19 * 8] 200 ldp x21, x22, [x4, #PCB_REGS + 21 * 8] 201 ldp x23, x24, [x4, #PCB_REGS + 23 * 8] 202 ldp x25, x26, [x4, #PCB_REGS + 25 * 8] 203 ldp x27, x28, [x4, #PCB_REGS + 27 * 8] 204 ldp x29, lr, [x4, #PCB_REGS + 29 * 8] 205 206 str xzr, [x4, #PCB_REGS + 18 * 8] 207 ret 208END(cpu_switch) 209 210ENTRY(fork_trampoline) 211 mov x0, x19 212 mov x1, x20 213 mov x2, sp 214 mov fp, #0 /* Stack traceback stops here. */ 215 bl _C_LABEL(fork_exit) 216 217 /* 218 * Disable interrupts as we are setting userspace specific 219 * state that we won't handle correctly in an interrupt while 220 * in the kernel. 221 */ 222 msr daifset, #(DAIF_D | DAIF_INTR) 223 224 ldr x0, [x18, #PC_CURTHREAD] 225 bl ptrauth_enter_el0 226 227 /* Restore sp, lr, elr, and spsr */ 228 ldp x18, lr, [sp, #TF_SP] 229 ldp x10, x11, [sp, #TF_ELR] 230 msr sp_el0, x18 231 msr spsr_el1, x11 232 msr elr_el1, x10 233 234 /* Restore the CPU registers */ 235 ldp x0, x1, [sp, #TF_X + 0 * 8] 236 ldp x2, x3, [sp, #TF_X + 2 * 8] 237 ldp x4, x5, [sp, #TF_X + 4 * 8] 238 ldp x6, x7, [sp, #TF_X + 6 * 8] 239 ldp x8, x9, [sp, #TF_X + 8 * 8] 240 ldp x10, x11, [sp, #TF_X + 10 * 8] 241 ldp x12, x13, [sp, #TF_X + 12 * 8] 242 ldp x14, x15, [sp, #TF_X + 14 * 8] 243 ldp x16, x17, [sp, #TF_X + 16 * 8] 244 ldp x18, x19, [sp, #TF_X + 18 * 8] 245 ldp x20, x21, [sp, #TF_X + 20 * 8] 246 ldp x22, x23, [sp, #TF_X + 22 * 8] 247 ldp x24, x25, [sp, #TF_X + 24 * 8] 248 ldp x26, x27, [sp, #TF_X + 26 * 8] 249 ldp x28, x29, [sp, #TF_X + 28 * 8] 250 251 /* 252 * No need for interrupts reenabling since PSR 253 * will be set to the desired value anyway. 254 */ 255 ERET 256 257END(fork_trampoline) 258 259ENTRY(savectx) 260 /* Store the callee-saved registers */ 261 stp x19, x20, [x0, #PCB_REGS + 19 * 8] 262 stp x21, x22, [x0, #PCB_REGS + 21 * 8] 263 stp x23, x24, [x0, #PCB_REGS + 23 * 8] 264 stp x25, x26, [x0, #PCB_REGS + 25 * 8] 265 stp x27, x28, [x0, #PCB_REGS + 27 * 8] 266 stp x29, lr, [x0, #PCB_REGS + 29 * 8] 267 /* And the old stack pointer */ 268 mov x5, sp 269 mrs x6, tpidrro_el0 270 str x6, [x0, #PCB_TPIDRRO] 271 mrs x6, tpidr_el0 272 stp x5, x6, [x0, #PCB_SP] 273 274 /* Store the VFP registers */ 275#ifdef VFP 276 mov x28, lr 277 mov x1, x0 /* move pcb to the correct register */ 278 mov x0, xzr /* td = NULL */ 279 bl vfp_save_state 280 mov lr, x28 281#endif 282 283 ret 284END(savectx) 285 286