1/*- 2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#include "assym.inc" 36#include "opt_sched.h" 37 38#include <machine/param.h> 39#include <machine/asm.h> 40#include <machine/riscvreg.h> 41#include <machine/pte.h> 42 43__FBSDID("$FreeBSD$"); 44 45#ifdef FPE 46.macro __fpe_state_save p 47 /* 48 * Enable FPE usage in supervisor mode, 49 * so we can access registers. 50 */ 51 li t0, SSTATUS_FS_INITIAL 52 csrs sstatus, t0 53 54 /* Store registers */ 55 frcsr t0 56 sd t0, (PCB_FCSR)(\p) 57 fsd f0, (PCB_X + 0 * 16)(\p) 58 fsd f1, (PCB_X + 1 * 16)(\p) 59 fsd f2, (PCB_X + 2 * 16)(\p) 60 fsd f3, (PCB_X + 3 * 16)(\p) 61 fsd f4, (PCB_X + 4 * 16)(\p) 62 fsd f5, (PCB_X + 5 * 16)(\p) 63 fsd f6, (PCB_X + 6 * 16)(\p) 64 fsd f7, (PCB_X + 7 * 16)(\p) 65 fsd f8, (PCB_X + 8 * 16)(\p) 66 fsd f9, (PCB_X + 9 * 16)(\p) 67 fsd f10, (PCB_X + 10 * 16)(\p) 68 fsd f11, (PCB_X + 11 * 16)(\p) 69 fsd f12, (PCB_X + 12 * 16)(\p) 70 fsd f13, (PCB_X + 13 * 16)(\p) 71 fsd f14, (PCB_X + 14 * 16)(\p) 72 fsd f15, (PCB_X + 15 * 16)(\p) 73 fsd f16, (PCB_X + 16 * 16)(\p) 74 fsd f17, (PCB_X + 17 * 16)(\p) 75 fsd f18, (PCB_X + 18 * 16)(\p) 76 fsd f19, (PCB_X + 19 * 16)(\p) 77 fsd f20, (PCB_X + 20 * 16)(\p) 78 fsd f21, (PCB_X + 21 * 16)(\p) 79 fsd f22, (PCB_X + 22 * 16)(\p) 80 fsd f23, (PCB_X + 23 * 16)(\p) 81 fsd f24, (PCB_X + 24 * 16)(\p) 82 fsd f25, (PCB_X + 25 * 16)(\p) 83 fsd f26, (PCB_X + 26 * 16)(\p) 84 fsd f27, (PCB_X + 27 * 16)(\p) 85 fsd f28, (PCB_X + 28 * 16)(\p) 86 fsd f29, (PCB_X + 29 * 16)(\p) 87 fsd f30, (PCB_X + 30 * 16)(\p) 88 fsd f31, (PCB_X + 31 * 16)(\p) 89 90 /* Disable FPE usage in supervisor mode. */ 91 li t0, SSTATUS_FS_MASK 92 csrc sstatus, t0 93.endm 94 95.macro __fpe_state_load p 96 /* 97 * Enable FPE usage in supervisor mode, 98 * so we can access registers. 99 */ 100 li t0, SSTATUS_FS_INITIAL 101 csrs sstatus, t0 102 103 /* Restore registers */ 104 ld t0, (PCB_FCSR)(\p) 105 fscsr t0 106 fld f0, (PCB_X + 0 * 16)(\p) 107 fld f1, (PCB_X + 1 * 16)(\p) 108 fld f2, (PCB_X + 2 * 16)(\p) 109 fld f3, (PCB_X + 3 * 16)(\p) 110 fld f4, (PCB_X + 4 * 16)(\p) 111 fld f5, (PCB_X + 5 * 16)(\p) 112 fld f6, (PCB_X + 6 * 16)(\p) 113 fld f7, (PCB_X + 7 * 16)(\p) 114 fld f8, (PCB_X + 8 * 16)(\p) 115 fld f9, (PCB_X + 9 * 16)(\p) 116 fld f10, (PCB_X + 10 * 16)(\p) 117 fld f11, (PCB_X + 11 * 16)(\p) 118 fld f12, (PCB_X + 12 * 16)(\p) 119 fld f13, (PCB_X + 13 * 16)(\p) 120 fld f14, (PCB_X + 14 * 16)(\p) 121 fld f15, (PCB_X + 15 * 16)(\p) 122 fld f16, (PCB_X + 16 * 16)(\p) 123 fld f17, (PCB_X + 17 * 16)(\p) 124 fld f18, (PCB_X + 18 * 16)(\p) 125 fld f19, (PCB_X + 19 * 16)(\p) 126 fld f20, (PCB_X + 20 * 16)(\p) 127 fld f21, (PCB_X + 21 * 16)(\p) 128 fld f22, (PCB_X + 22 * 16)(\p) 129 fld f23, (PCB_X + 23 * 16)(\p) 130 fld f24, (PCB_X + 24 * 16)(\p) 131 fld f25, (PCB_X + 25 * 16)(\p) 132 fld f26, (PCB_X + 26 * 16)(\p) 133 fld f27, (PCB_X + 27 * 16)(\p) 134 fld f28, (PCB_X + 28 * 16)(\p) 135 fld f29, (PCB_X + 29 * 16)(\p) 136 fld f30, (PCB_X + 30 * 16)(\p) 137 fld f31, (PCB_X + 31 * 16)(\p) 138 139 /* Disable FPE usage in supervisor mode. */ 140 li t0, SSTATUS_FS_MASK 141 csrc sstatus, t0 142.endm 143 144/* 145 * void 146 * fpe_state_save(struct thread *td) 147 */ 148ENTRY(fpe_state_save) 149 /* Get pointer to PCB */ 150 ld a0, TD_PCB(a0) 151 __fpe_state_save a0 152 ret 153END(fpe_state_save) 154 155/* 156 * void 157 * fpe_state_clear(void) 158 */ 159ENTRY(fpe_state_clear) 160 /* 161 * Enable FPE usage in supervisor mode, 162 * so we can access registers. 163 */ 164 li t0, SSTATUS_FS_INITIAL 165 csrs sstatus, t0 166 167 fscsr zero 168 fcvt.d.l f0, zero 169 fcvt.d.l f1, zero 170 fcvt.d.l f2, zero 171 fcvt.d.l f3, zero 172 fcvt.d.l f4, zero 173 fcvt.d.l f5, zero 174 fcvt.d.l f6, zero 175 fcvt.d.l f7, zero 176 fcvt.d.l f8, zero 177 fcvt.d.l f9, zero 178 fcvt.d.l f10, zero 179 fcvt.d.l f11, zero 180 fcvt.d.l f12, zero 181 fcvt.d.l f13, zero 182 fcvt.d.l f14, zero 183 fcvt.d.l f15, zero 184 fcvt.d.l f16, zero 185 fcvt.d.l f17, zero 186 fcvt.d.l f18, zero 187 fcvt.d.l f19, zero 188 fcvt.d.l f20, zero 189 fcvt.d.l f21, zero 190 fcvt.d.l f22, zero 191 fcvt.d.l f23, zero 192 fcvt.d.l f24, zero 193 fcvt.d.l f25, zero 194 fcvt.d.l f26, zero 195 fcvt.d.l f27, zero 196 fcvt.d.l f28, zero 197 fcvt.d.l f29, zero 198 fcvt.d.l f30, zero 199 fcvt.d.l f31, zero 200 201 /* Disable FPE usage in supervisor mode. */ 202 li t0, SSTATUS_FS_MASK 203 csrc sstatus, t0 204 205 ret 206END(fpe_state_clear) 207#endif /* FPE */ 208 209/* 210 * void cpu_throw(struct thread *old __unused, struct thread *new) 211 */ 212ENTRY(cpu_throw) 213 /* Activate the new thread's pmap. */ 214 mv s0, a1 215 mv a0, a1 216 call _C_LABEL(pmap_activate_sw) 217 mv a0, s0 218 219 /* Store the new curthread */ 220 sd a0, PC_CURTHREAD(tp) 221 /* And the new pcb */ 222 ld x13, TD_PCB(a0) 223 sd x13, PC_CURPCB(tp) 224 225 /* Load registers */ 226 ld ra, (PCB_RA)(x13) 227 ld sp, (PCB_SP)(x13) 228 229 /* s[0-11] */ 230 ld s0, (PCB_S + 0 * 8)(x13) 231 ld s1, (PCB_S + 1 * 8)(x13) 232 ld s2, (PCB_S + 2 * 8)(x13) 233 ld s3, (PCB_S + 3 * 8)(x13) 234 ld s4, (PCB_S + 4 * 8)(x13) 235 ld s5, (PCB_S + 5 * 8)(x13) 236 ld s6, (PCB_S + 6 * 8)(x13) 237 ld s7, (PCB_S + 7 * 8)(x13) 238 ld s8, (PCB_S + 8 * 8)(x13) 239 ld s9, (PCB_S + 9 * 8)(x13) 240 ld s10, (PCB_S + 10 * 8)(x13) 241 ld s11, (PCB_S + 11 * 8)(x13) 242 243#ifdef FPE 244 /* Is FPE enabled for new thread? */ 245 ld t0, TD_FRAME(a0) 246 ld t1, (TF_SSTATUS)(t0) 247 li t2, SSTATUS_FS_MASK 248 and t3, t1, t2 249 beqz t3, 1f /* No, skip. */ 250 251 /* Restore registers. */ 252 __fpe_state_load x13 2531: 254#endif 255 256 ret 257END(cpu_throw) 258 259/* 260 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 261 * 262 * a0 = old 263 * a1 = new 264 * a2 = mtx 265 * x3 to x7, x16 and x17 are caller saved 266 */ 267ENTRY(cpu_switch) 268 /* Store the new curthread */ 269 sd a1, PC_CURTHREAD(tp) 270 /* And the new pcb */ 271 ld x13, TD_PCB(a1) 272 sd x13, PC_CURPCB(tp) 273 274 /* Save the old context. */ 275 ld x13, TD_PCB(a0) 276 277 /* Store ra, sp and the callee-saved registers */ 278 sd ra, (PCB_RA)(x13) 279 sd sp, (PCB_SP)(x13) 280 281 /* s[0-11] */ 282 sd s0, (PCB_S + 0 * 8)(x13) 283 sd s1, (PCB_S + 1 * 8)(x13) 284 sd s2, (PCB_S + 2 * 8)(x13) 285 sd s3, (PCB_S + 3 * 8)(x13) 286 sd s4, (PCB_S + 4 * 8)(x13) 287 sd s5, (PCB_S + 5 * 8)(x13) 288 sd s6, (PCB_S + 6 * 8)(x13) 289 sd s7, (PCB_S + 7 * 8)(x13) 290 sd s8, (PCB_S + 8 * 8)(x13) 291 sd s9, (PCB_S + 9 * 8)(x13) 292 sd s10, (PCB_S + 10 * 8)(x13) 293 sd s11, (PCB_S + 11 * 8)(x13) 294 295#ifdef FPE 296 /* 297 * Is FPE enabled and is it in dirty state 298 * for the old thread? 299 */ 300 ld t0, TD_FRAME(a0) 301 ld t1, (TF_SSTATUS)(t0) 302 li t2, SSTATUS_FS_MASK 303 and t3, t1, t2 304 li t2, SSTATUS_FS_DIRTY 305 bne t3, t2, 1f /* No, skip. */ 306 307 /* Yes, mark FPE state clean and save registers. */ 308 li t2, ~SSTATUS_FS_MASK 309 and t3, t1, t2 310 li t2, SSTATUS_FS_CLEAN 311 or t3, t3, t2 312 sd t3, (TF_SSTATUS)(t0) 313 314 __fpe_state_save x13 3151: 316#endif 317 318 /* Activate the new thread's pmap */ 319 mv s0, a0 320 mv s1, a1 321 mv s2, a2 322 mv a0, a1 323 call _C_LABEL(pmap_activate_sw) 324 mv a1, s1 325 326 /* Release the old thread */ 327 sd s2, TD_LOCK(s0) 328#if defined(SCHED_ULE) && defined(SMP) 329 /* Spin if TD_LOCK points to a blocked_lock */ 330 la s2, _C_LABEL(blocked_lock) 3311: 332 ld t0, TD_LOCK(a1) 333 beq t0, s2, 1b 334#endif 335 /* 336 * Restore the saved context. 337 */ 338 ld x13, TD_PCB(a1) 339 340 /* Restore the registers */ 341 ld ra, (PCB_RA)(x13) 342 ld sp, (PCB_SP)(x13) 343 344 /* s[0-11] */ 345 ld s0, (PCB_S + 0 * 8)(x13) 346 ld s1, (PCB_S + 1 * 8)(x13) 347 ld s2, (PCB_S + 2 * 8)(x13) 348 ld s3, (PCB_S + 3 * 8)(x13) 349 ld s4, (PCB_S + 4 * 8)(x13) 350 ld s5, (PCB_S + 5 * 8)(x13) 351 ld s6, (PCB_S + 6 * 8)(x13) 352 ld s7, (PCB_S + 7 * 8)(x13) 353 ld s8, (PCB_S + 8 * 8)(x13) 354 ld s9, (PCB_S + 9 * 8)(x13) 355 ld s10, (PCB_S + 10 * 8)(x13) 356 ld s11, (PCB_S + 11 * 8)(x13) 357 358#ifdef FPE 359 /* Is FPE enabled for new thread? */ 360 ld t0, TD_FRAME(a1) 361 ld t1, (TF_SSTATUS)(t0) 362 li t2, SSTATUS_FS_MASK 363 and t3, t1, t2 364 beqz t3, 1f /* No, skip. */ 365 366 /* Restore registers. */ 367 __fpe_state_load x13 3681: 369#endif 370 371 ret 372.Lcpu_switch_panic_str: 373 .asciz "cpu_switch: %p\0" 374END(cpu_switch) 375 376/* 377 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, 378 * struct trapframe *frame) 379 */ 380 381ENTRY(fork_trampoline) 382 mv a0, s0 383 mv a1, s1 384 mv a2, sp 385 call _C_LABEL(fork_exit) 386 387 /* Restore sstatus */ 388 ld t0, (TF_SSTATUS)(sp) 389 /* Ensure interrupts disabled */ 390 li t1, ~SSTATUS_SIE 391 and t0, t0, t1 392 csrw sstatus, t0 393 394 /* Restore exception program counter */ 395 ld t0, (TF_SEPC)(sp) 396 csrw sepc, t0 397 398 /* Restore the registers */ 399 ld t0, (TF_T + 0 * 8)(sp) 400 ld t1, (TF_T + 1 * 8)(sp) 401 ld t2, (TF_T + 2 * 8)(sp) 402 ld t3, (TF_T + 3 * 8)(sp) 403 ld t4, (TF_T + 4 * 8)(sp) 404 ld t5, (TF_T + 5 * 8)(sp) 405 ld t6, (TF_T + 6 * 8)(sp) 406 407 ld s0, (TF_S + 0 * 8)(sp) 408 ld s1, (TF_S + 1 * 8)(sp) 409 ld s2, (TF_S + 2 * 8)(sp) 410 ld s3, (TF_S + 3 * 8)(sp) 411 ld s4, (TF_S + 4 * 8)(sp) 412 ld s5, (TF_S + 5 * 8)(sp) 413 ld s6, (TF_S + 6 * 8)(sp) 414 ld s7, (TF_S + 7 * 8)(sp) 415 ld s8, (TF_S + 8 * 8)(sp) 416 ld s9, (TF_S + 9 * 8)(sp) 417 ld s10, (TF_S + 10 * 8)(sp) 418 ld s11, (TF_S + 11 * 8)(sp) 419 420 ld a0, (TF_A + 0 * 8)(sp) 421 ld a1, (TF_A + 1 * 8)(sp) 422 ld a2, (TF_A + 2 * 8)(sp) 423 ld a3, (TF_A + 3 * 8)(sp) 424 ld a4, (TF_A + 4 * 8)(sp) 425 ld a5, (TF_A + 5 * 8)(sp) 426 ld a6, (TF_A + 6 * 8)(sp) 427 ld a7, (TF_A + 7 * 8)(sp) 428 429 /* Load user ra and gp */ 430 ld ra, (TF_RA)(sp) 431 ld gp, (TF_GP)(sp) 432 433 /* 434 * Store our pcpup on stack, we will load it back 435 * on kernel mode trap. 436 */ 437 sd tp, (TF_SIZE)(sp) 438 ld tp, (TF_TP)(sp) 439 440 /* Save kernel stack so we can use it doing a user trap */ 441 addi sp, sp, TF_SIZE 442 csrw sscratch, sp 443 444 /* Load user stack */ 445 ld sp, (TF_SP - TF_SIZE)(sp) 446 447 sret 448END(fork_trampoline) 449 450ENTRY(savectx) 451 /* Store ra, sp and the callee-saved registers */ 452 sd ra, (PCB_RA)(a0) 453 sd sp, (PCB_SP)(a0) 454 sd tp, (PCB_TP)(a0) 455 sd gp, (PCB_GP)(a0) 456 457 /* s[0-11] */ 458 sd s0, (PCB_S + 0 * 8)(a0) 459 sd s1, (PCB_S + 1 * 8)(a0) 460 sd s2, (PCB_S + 2 * 8)(a0) 461 sd s3, (PCB_S + 3 * 8)(a0) 462 sd s4, (PCB_S + 4 * 8)(a0) 463 sd s5, (PCB_S + 5 * 8)(a0) 464 sd s6, (PCB_S + 6 * 8)(a0) 465 sd s7, (PCB_S + 7 * 8)(a0) 466 sd s8, (PCB_S + 8 * 8)(a0) 467 sd s9, (PCB_S + 9 * 8)(a0) 468 sd s10, (PCB_S + 10 * 8)(a0) 469 sd s11, (PCB_S + 11 * 8)(a0) 470 471#ifdef FPE 472 __fpe_state_save a0 473#endif 474 ret 475END(savectx) 476