1/*- 2 * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#include "assym.s" 36#include "opt_sched.h" 37 38#include <machine/param.h> 39#include <machine/asm.h> 40#include <machine/riscvreg.h> 41#include <machine/pte.h> 42 43__FBSDID("$FreeBSD$"); 44 45#ifdef FPE 46.macro __fpe_state_save p 47 /* 48 * Enable FPE usage in supervisor mode, 49 * so we can access registers. 50 */ 51 li t0, SSTATUS_FS_INITIAL 52 csrs sstatus, t0 53 54 /* Store registers */ 55 frcsr t0 56 sd t0, (PCB_FCSR)(\p) 57 fsd f0, (PCB_X + 0 * 16)(\p) 58 fsd f1, (PCB_X + 1 * 16)(\p) 59 fsd f2, (PCB_X + 2 * 16)(\p) 60 fsd f3, (PCB_X + 3 * 16)(\p) 61 fsd f4, (PCB_X + 4 * 16)(\p) 62 fsd f5, (PCB_X + 5 * 16)(\p) 63 fsd f6, (PCB_X + 6 * 16)(\p) 64 fsd f7, (PCB_X + 7 * 16)(\p) 65 fsd f8, (PCB_X + 8 * 16)(\p) 66 fsd f9, (PCB_X + 9 * 16)(\p) 67 fsd f10, (PCB_X + 10 * 16)(\p) 68 fsd f11, (PCB_X + 11 * 16)(\p) 69 fsd f12, (PCB_X + 12 * 16)(\p) 70 fsd f13, (PCB_X + 13 * 16)(\p) 71 fsd f14, (PCB_X + 14 * 16)(\p) 72 fsd f15, (PCB_X + 15 * 16)(\p) 73 fsd f16, (PCB_X + 16 * 16)(\p) 74 fsd f17, (PCB_X + 17 * 16)(\p) 75 fsd f18, (PCB_X + 18 * 16)(\p) 76 fsd f19, (PCB_X + 19 * 16)(\p) 77 fsd f20, (PCB_X + 20 * 16)(\p) 78 fsd f21, (PCB_X + 21 * 16)(\p) 79 fsd f22, (PCB_X + 22 * 16)(\p) 80 fsd f23, (PCB_X + 23 * 16)(\p) 81 fsd f24, (PCB_X + 24 * 16)(\p) 82 fsd f25, (PCB_X + 25 * 16)(\p) 83 fsd f26, (PCB_X + 26 * 16)(\p) 84 fsd f27, (PCB_X + 27 * 16)(\p) 85 fsd f28, (PCB_X + 28 * 16)(\p) 86 fsd f29, (PCB_X + 29 * 16)(\p) 87 fsd f30, (PCB_X + 30 * 16)(\p) 88 fsd f31, (PCB_X + 31 * 16)(\p) 89 90 /* Disable FPE usage in supervisor mode. */ 91 li t0, SSTATUS_FS_MASK 92 csrc sstatus, t0 93.endm 94 95.macro __fpe_state_load p 96 /* 97 * Enable FPE usage in supervisor mode, 98 * so we can access registers. 99 */ 100 li t0, SSTATUS_FS_INITIAL 101 csrs sstatus, t0 102 103 /* Restore registers */ 104 ld t0, (PCB_FCSR)(\p) 105 fscsr t0 106 fld f0, (PCB_X + 0 * 16)(\p) 107 fld f1, (PCB_X + 1 * 16)(\p) 108 fld f2, (PCB_X + 2 * 16)(\p) 109 fld f3, (PCB_X + 3 * 16)(\p) 110 fld f4, (PCB_X + 4 * 16)(\p) 111 fld f5, (PCB_X + 5 * 16)(\p) 112 fld f6, (PCB_X + 6 * 16)(\p) 113 fld f7, (PCB_X + 7 * 16)(\p) 114 fld f8, (PCB_X + 8 * 16)(\p) 115 fld f9, (PCB_X + 9 * 16)(\p) 116 fld f10, (PCB_X + 10 * 16)(\p) 117 fld f11, (PCB_X + 11 * 16)(\p) 118 fld f12, (PCB_X + 12 * 16)(\p) 119 fld f13, (PCB_X + 13 * 16)(\p) 120 fld f14, (PCB_X + 14 * 16)(\p) 121 fld f15, (PCB_X + 15 * 16)(\p) 122 fld f16, (PCB_X + 16 * 16)(\p) 123 fld f17, (PCB_X + 17 * 16)(\p) 124 fld f18, (PCB_X + 18 * 16)(\p) 125 fld f19, (PCB_X + 19 * 16)(\p) 126 fld f20, (PCB_X + 20 * 16)(\p) 127 fld f21, (PCB_X + 21 * 16)(\p) 128 fld f22, (PCB_X + 22 * 16)(\p) 129 fld f23, (PCB_X + 23 * 16)(\p) 130 fld f24, (PCB_X + 24 * 16)(\p) 131 fld f25, (PCB_X + 25 * 16)(\p) 132 fld f26, (PCB_X + 26 * 16)(\p) 133 fld f27, (PCB_X + 27 * 16)(\p) 134 fld f28, (PCB_X + 28 * 16)(\p) 135 fld f29, (PCB_X + 29 * 16)(\p) 136 fld f30, (PCB_X + 30 * 16)(\p) 137 fld f31, (PCB_X + 31 * 16)(\p) 138 139 /* Disable FPE usage in supervisor mode. */ 140 li t0, SSTATUS_FS_MASK 141 csrc sstatus, t0 142.endm 143 144/* 145 * void 146 * fpe_state_save(struct thread *td) 147 */ 148ENTRY(fpe_state_save) 149 /* Get pointer to PCB */ 150 ld a0, TD_PCB(a0) 151 __fpe_state_save a0 152 ret 153END(fpe_state_save) 154#endif /* FPE */ 155 156/* 157 * void cpu_throw(struct thread *old, struct thread *new) 158 */ 159ENTRY(cpu_throw) 160 /* Store the new curthread */ 161 sd a1, PC_CURTHREAD(gp) 162 /* And the new pcb */ 163 ld x13, TD_PCB(a1) 164 sd x13, PC_CURPCB(gp) 165 166 sfence.vm 167 168 /* Switch to the new pmap */ 169 ld t0, PCB_L1ADDR(x13) 170 srli t0, t0, PAGE_SHIFT 171 csrw sptbr, t0 172 173 /* TODO: Invalidate the TLB */ 174 175 sfence.vm 176 177 /* Load registers */ 178 ld ra, (PCB_RA)(x13) 179 ld sp, (PCB_SP)(x13) 180 181 /* s[0-11] */ 182 ld s0, (PCB_S + 0 * 8)(x13) 183 ld s1, (PCB_S + 1 * 8)(x13) 184 ld s2, (PCB_S + 2 * 8)(x13) 185 ld s3, (PCB_S + 3 * 8)(x13) 186 ld s4, (PCB_S + 4 * 8)(x13) 187 ld s5, (PCB_S + 5 * 8)(x13) 188 ld s6, (PCB_S + 6 * 8)(x13) 189 ld s7, (PCB_S + 7 * 8)(x13) 190 ld s8, (PCB_S + 8 * 8)(x13) 191 ld s9, (PCB_S + 9 * 8)(x13) 192 ld s10, (PCB_S + 10 * 8)(x13) 193 ld s11, (PCB_S + 11 * 8)(x13) 194 195#ifdef FPE 196 /* Is FPE enabled for new thread? */ 197 ld t0, TD_FRAME(a1) 198 ld t1, (TF_SSTATUS)(t0) 199 li t2, SSTATUS_FS_MASK 200 and t3, t1, t2 201 beqz t3, 1f /* No, skip. */ 202 203 /* Restore registers. */ 204 __fpe_state_load x13 2051: 206#endif 207 208 ret 209.Lcpu_throw_panic_str: 210 .asciz "cpu_throw: %p\0" 211END(cpu_throw) 212 213/* 214 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 215 * 216 * a0 = old 217 * a1 = new 218 * a2 = mtx 219 * x3 to x7, x16 and x17 are caller saved 220 */ 221ENTRY(cpu_switch) 222 /* Store the new curthread */ 223 sd a1, PC_CURTHREAD(gp) 224 /* And the new pcb */ 225 ld x13, TD_PCB(a1) 226 sd x13, PC_CURPCB(gp) 227 228 /* Save the old context. */ 229 ld x13, TD_PCB(a0) 230 231 /* Store ra, sp and the callee-saved registers */ 232 sd ra, (PCB_RA)(x13) 233 sd sp, (PCB_SP)(x13) 234 235 /* s[0-11] */ 236 sd s0, (PCB_S + 0 * 8)(x13) 237 sd s1, (PCB_S + 1 * 8)(x13) 238 sd s2, (PCB_S + 2 * 8)(x13) 239 sd s3, (PCB_S + 3 * 8)(x13) 240 sd s4, (PCB_S + 4 * 8)(x13) 241 sd s5, (PCB_S + 5 * 8)(x13) 242 sd s6, (PCB_S + 6 * 8)(x13) 243 sd s7, (PCB_S + 7 * 8)(x13) 244 sd s8, (PCB_S + 8 * 8)(x13) 245 sd s9, (PCB_S + 9 * 8)(x13) 246 sd s10, (PCB_S + 10 * 8)(x13) 247 sd s11, (PCB_S + 11 * 8)(x13) 248 249#ifdef FPE 250 /* 251 * Is FPE enabled and is it in dirty state 252 * for the old thread? 253 */ 254 ld t0, TD_FRAME(a0) 255 ld t1, (TF_SSTATUS)(t0) 256 li t2, SSTATUS_FS_MASK 257 and t3, t1, t2 258 li t2, SSTATUS_FS_DIRTY 259 bne t3, t2, 1f /* No, skip. */ 260 261 /* Yes, mark FPE state clean and save registers. */ 262 li t2, ~SSTATUS_FS_MASK 263 and t3, t1, t2 264 li t2, SSTATUS_FS_CLEAN 265 or t3, t3, t2 266 sd t3, (TF_SSTATUS)(t0) 267 268 __fpe_state_save x13 2691: 270#endif 271 272 /* 273 * Restore the saved context. 274 */ 275 ld x13, TD_PCB(a1) 276 277 /* 278 * TODO: We may need to flush the cache here if switching 279 * to a user process. 280 */ 281 282 sfence.vm 283 284 /* Switch to the new pmap */ 285 ld t0, PCB_L1ADDR(x13) 286 srli t0, t0, PAGE_SHIFT 287 csrw sptbr, t0 288 289 /* TODO: Invalidate the TLB */ 290 291 sfence.vm 292 293 /* Release the old thread */ 294 sd a2, TD_LOCK(a0) 295#if defined(SCHED_ULE) && defined(SMP) 296 /* Spin if TD_LOCK points to a blocked_lock */ 297 la a2, _C_LABEL(blocked_lock) 2981: 299 ld t0, TD_LOCK(a1) 300 beq t0, a2, 1b 301#endif 302 303 /* Restore the registers */ 304 ld ra, (PCB_RA)(x13) 305 ld sp, (PCB_SP)(x13) 306 307 /* s[0-11] */ 308 ld s0, (PCB_S + 0 * 8)(x13) 309 ld s1, (PCB_S + 1 * 8)(x13) 310 ld s2, (PCB_S + 2 * 8)(x13) 311 ld s3, (PCB_S + 3 * 8)(x13) 312 ld s4, (PCB_S + 4 * 8)(x13) 313 ld s5, (PCB_S + 5 * 8)(x13) 314 ld s6, (PCB_S + 6 * 8)(x13) 315 ld s7, (PCB_S + 7 * 8)(x13) 316 ld s8, (PCB_S + 8 * 8)(x13) 317 ld s9, (PCB_S + 9 * 8)(x13) 318 ld s10, (PCB_S + 10 * 8)(x13) 319 ld s11, (PCB_S + 11 * 8)(x13) 320 321#ifdef FPE 322 /* Is FPE enabled for new thread? */ 323 ld t0, TD_FRAME(a1) 324 ld t1, (TF_SSTATUS)(t0) 325 li t2, SSTATUS_FS_MASK 326 and t3, t1, t2 327 beqz t3, 1f /* No, skip. */ 328 329 /* Restore registers. */ 330 __fpe_state_load x13 3311: 332#endif 333 334 ret 335.Lcpu_switch_panic_str: 336 .asciz "cpu_switch: %p\0" 337END(cpu_switch) 338 339/* 340 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, 341 * struct trapframe *frame) 342 */ 343 344ENTRY(fork_trampoline) 345 mv a0, s0 346 mv a1, s1 347 mv a2, sp 348 call _C_LABEL(fork_exit) 349 350 /* Restore sstatus */ 351 ld t0, (TF_SSTATUS)(sp) 352 /* Ensure interrupts disabled */ 353 li t1, ~SSTATUS_SIE 354 and t0, t0, t1 355 csrw sstatus, t0 356 357 /* Restore exception program counter */ 358 ld t0, (TF_SEPC)(sp) 359 csrw sepc, t0 360 361 /* Restore the registers */ 362 ld t0, (TF_T + 0 * 8)(sp) 363 ld t1, (TF_T + 1 * 8)(sp) 364 ld t2, (TF_T + 2 * 8)(sp) 365 ld t3, (TF_T + 3 * 8)(sp) 366 ld t4, (TF_T + 4 * 8)(sp) 367 ld t5, (TF_T + 5 * 8)(sp) 368 ld t6, (TF_T + 6 * 8)(sp) 369 370 ld s0, (TF_S + 0 * 8)(sp) 371 ld s1, (TF_S + 1 * 8)(sp) 372 ld s2, (TF_S + 2 * 8)(sp) 373 ld s3, (TF_S + 3 * 8)(sp) 374 ld s4, (TF_S + 4 * 8)(sp) 375 ld s5, (TF_S + 5 * 8)(sp) 376 ld s6, (TF_S + 6 * 8)(sp) 377 ld s7, (TF_S + 7 * 8)(sp) 378 ld s8, (TF_S + 8 * 8)(sp) 379 ld s9, (TF_S + 9 * 8)(sp) 380 ld s10, (TF_S + 10 * 8)(sp) 381 ld s11, (TF_S + 11 * 8)(sp) 382 383 ld a0, (TF_A + 0 * 8)(sp) 384 ld a1, (TF_A + 1 * 8)(sp) 385 ld a2, (TF_A + 2 * 8)(sp) 386 ld a3, (TF_A + 3 * 8)(sp) 387 ld a4, (TF_A + 4 * 8)(sp) 388 ld a5, (TF_A + 5 * 8)(sp) 389 ld a6, (TF_A + 6 * 8)(sp) 390 ld a7, (TF_A + 7 * 8)(sp) 391 392 /* Load user ra and sp */ 393 ld tp, (TF_TP)(sp) 394 ld ra, (TF_RA)(sp) 395 396 /* 397 * Store our pcpup on stack, we will load it back 398 * on kernel mode trap. 399 */ 400 sd gp, (TF_SIZE)(sp) 401 ld gp, (TF_GP)(sp) 402 403 /* Save kernel stack so we can use it doing a user trap */ 404 addi sp, sp, TF_SIZE 405 csrw sscratch, sp 406 407 /* Load user stack */ 408 ld sp, (TF_SP - TF_SIZE)(sp) 409 410 sret 411END(fork_trampoline) 412 413ENTRY(savectx) 414 /* Store ra, sp and the callee-saved registers */ 415 sd ra, (PCB_RA)(a0) 416 sd sp, (PCB_SP)(a0) 417 418 /* s[0-11] */ 419 sd s0, (PCB_S + 0 * 8)(a0) 420 sd s1, (PCB_S + 1 * 8)(a0) 421 sd s2, (PCB_S + 2 * 8)(a0) 422 sd s3, (PCB_S + 3 * 8)(a0) 423 sd s4, (PCB_S + 4 * 8)(a0) 424 sd s5, (PCB_S + 5 * 8)(a0) 425 sd s6, (PCB_S + 6 * 8)(a0) 426 sd s7, (PCB_S + 7 * 8)(a0) 427 sd s8, (PCB_S + 8 * 8)(a0) 428 sd s9, (PCB_S + 9 * 8)(a0) 429 sd s10, (PCB_S + 10 * 8)(a0) 430 sd s11, (PCB_S + 11 * 8)(a0) 431 432#ifdef FPE 433 __fpe_state_save a0 434#endif 435 ret 436END(savectx) 437