1/*- 2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#include "assym.inc" 36#include "opt_sched.h" 37 38#include <machine/param.h> 39#include <machine/asm.h> 40#include <machine/riscvreg.h> 41#include <machine/pte.h> 42 43__FBSDID("$FreeBSD$"); 44 45#ifdef FPE 46.macro __fpe_state_save p 47 /* 48 * Enable FPE usage in supervisor mode, 49 * so we can access registers. 50 */ 51 li t0, SSTATUS_FS_INITIAL 52 csrs sstatus, t0 53 54 /* Store registers */ 55 frcsr t0 56 sd t0, (PCB_FCSR)(\p) 57 fsd f0, (PCB_X + 0 * 16)(\p) 58 fsd f1, (PCB_X + 1 * 16)(\p) 59 fsd f2, (PCB_X + 2 * 16)(\p) 60 fsd f3, (PCB_X + 3 * 16)(\p) 61 fsd f4, (PCB_X + 4 * 16)(\p) 62 fsd f5, (PCB_X + 5 * 16)(\p) 63 fsd f6, (PCB_X + 6 * 16)(\p) 64 fsd f7, (PCB_X + 7 * 16)(\p) 65 fsd f8, (PCB_X + 8 * 16)(\p) 66 fsd f9, (PCB_X + 9 * 16)(\p) 67 fsd f10, (PCB_X + 10 * 16)(\p) 68 fsd f11, (PCB_X + 11 * 16)(\p) 69 fsd f12, (PCB_X + 12 * 16)(\p) 70 fsd f13, (PCB_X + 13 * 16)(\p) 71 fsd f14, (PCB_X + 14 * 16)(\p) 72 fsd f15, (PCB_X + 15 * 16)(\p) 73 fsd f16, (PCB_X + 16 * 16)(\p) 74 fsd f17, (PCB_X + 17 * 16)(\p) 75 fsd f18, (PCB_X + 18 * 16)(\p) 76 fsd f19, (PCB_X + 19 * 16)(\p) 77 fsd f20, (PCB_X + 20 * 16)(\p) 78 fsd f21, (PCB_X + 21 * 16)(\p) 79 fsd f22, (PCB_X + 22 * 16)(\p) 80 fsd f23, (PCB_X + 23 * 16)(\p) 81 fsd f24, (PCB_X + 24 * 16)(\p) 82 fsd f25, (PCB_X + 25 * 16)(\p) 83 fsd f26, (PCB_X + 26 * 16)(\p) 84 fsd f27, (PCB_X + 27 * 16)(\p) 85 fsd f28, (PCB_X + 28 * 16)(\p) 86 fsd f29, (PCB_X + 29 * 16)(\p) 87 fsd f30, (PCB_X + 30 * 16)(\p) 88 fsd f31, (PCB_X + 31 * 16)(\p) 89 90 /* Disable FPE usage in supervisor mode. */ 91 li t0, SSTATUS_FS_MASK 92 csrc sstatus, t0 93.endm 94 95.macro __fpe_state_load p 96 /* 97 * Enable FPE usage in supervisor mode, 98 * so we can access registers. 99 */ 100 li t0, SSTATUS_FS_INITIAL 101 csrs sstatus, t0 102 103 /* Restore registers */ 104 ld t0, (PCB_FCSR)(\p) 105 fscsr t0 106 fld f0, (PCB_X + 0 * 16)(\p) 107 fld f1, (PCB_X + 1 * 16)(\p) 108 fld f2, (PCB_X + 2 * 16)(\p) 109 fld f3, (PCB_X + 3 * 16)(\p) 110 fld f4, (PCB_X + 4 * 16)(\p) 111 fld f5, (PCB_X + 5 * 16)(\p) 112 fld f6, (PCB_X + 6 * 16)(\p) 113 fld f7, (PCB_X + 7 * 16)(\p) 114 fld f8, (PCB_X + 8 * 16)(\p) 115 fld f9, (PCB_X + 9 * 16)(\p) 116 fld f10, (PCB_X + 10 * 16)(\p) 117 fld f11, (PCB_X + 11 * 16)(\p) 118 fld f12, (PCB_X + 12 * 16)(\p) 119 fld f13, (PCB_X + 13 * 16)(\p) 120 fld f14, (PCB_X + 14 * 16)(\p) 121 fld f15, (PCB_X + 15 * 16)(\p) 122 fld f16, (PCB_X + 16 * 16)(\p) 123 fld f17, (PCB_X + 17 * 16)(\p) 124 fld f18, (PCB_X + 18 * 16)(\p) 125 fld f19, (PCB_X + 19 * 16)(\p) 126 fld f20, (PCB_X + 20 * 16)(\p) 127 fld f21, (PCB_X + 21 * 16)(\p) 128 fld f22, (PCB_X + 22 * 16)(\p) 129 fld f23, (PCB_X + 23 * 16)(\p) 130 fld f24, (PCB_X + 24 * 16)(\p) 131 fld f25, (PCB_X + 25 * 16)(\p) 132 fld f26, (PCB_X + 26 * 16)(\p) 133 fld f27, (PCB_X + 27 * 16)(\p) 134 fld f28, (PCB_X + 28 * 16)(\p) 135 fld f29, (PCB_X + 29 * 16)(\p) 136 fld f30, (PCB_X + 30 * 16)(\p) 137 fld f31, (PCB_X + 31 * 16)(\p) 138 139 /* Disable FPE usage in supervisor mode. */ 140 li t0, SSTATUS_FS_MASK 141 csrc sstatus, t0 142.endm 143 144/* 145 * void 146 * fpe_state_save(struct thread *td) 147 */ 148ENTRY(fpe_state_save) 149 /* Get pointer to PCB */ 150 ld a0, TD_PCB(a0) 151 __fpe_state_save a0 152 ret 153END(fpe_state_save) 154#endif /* FPE */ 155 156/* 157 * void 158 * fpe_state_clear(void) 159 */ 160ENTRY(fpe_state_clear) 161 /* 162 * Enable FPE usage in supervisor mode, 163 * so we can access registers. 164 */ 165 li t0, SSTATUS_FS_INITIAL 166 csrs sstatus, t0 167 168 fscsr zero 169 fcvt.d.l f0, zero 170 fcvt.d.l f1, zero 171 fcvt.d.l f2, zero 172 fcvt.d.l f3, zero 173 fcvt.d.l f4, zero 174 fcvt.d.l f5, zero 175 fcvt.d.l f6, zero 176 fcvt.d.l f7, zero 177 fcvt.d.l f8, zero 178 fcvt.d.l f9, zero 179 fcvt.d.l f10, zero 180 fcvt.d.l f11, zero 181 fcvt.d.l f12, zero 182 fcvt.d.l f13, zero 183 fcvt.d.l f14, zero 184 fcvt.d.l f15, zero 185 fcvt.d.l f16, zero 186 fcvt.d.l f17, zero 187 fcvt.d.l f18, zero 188 fcvt.d.l f19, zero 189 fcvt.d.l f20, zero 190 fcvt.d.l f21, zero 191 fcvt.d.l f22, zero 192 fcvt.d.l f23, zero 193 fcvt.d.l f24, zero 194 fcvt.d.l f25, zero 195 fcvt.d.l f26, zero 196 fcvt.d.l f27, zero 197 fcvt.d.l f28, zero 198 fcvt.d.l f29, zero 199 fcvt.d.l f30, zero 200 fcvt.d.l f31, zero 201 202 /* Disable FPE usage in supervisor mode. */ 203 li t0, SSTATUS_FS_MASK 204 csrc sstatus, t0 205 206 ret 207END(fpe_state_clear) 208 209/* 210 * void cpu_throw(struct thread *old __unused, struct thread *new) 211 */ 212ENTRY(cpu_throw) 213 /* Activate the new thread's pmap. */ 214 mv s0, a1 215 mv a0, a1 216 call _C_LABEL(pmap_activate_sw) 217 mv a0, s0 218 219 /* Store the new curthread */ 220 sd a0, PC_CURTHREAD(gp) 221 /* And the new pcb */ 222 ld x13, TD_PCB(a0) 223 sd x13, PC_CURPCB(gp) 224 225 /* Load registers */ 226 ld ra, (PCB_RA)(x13) 227 ld sp, (PCB_SP)(x13) 228 ld tp, (PCB_TP)(x13) 229 230 /* s[0-11] */ 231 ld s0, (PCB_S + 0 * 8)(x13) 232 ld s1, (PCB_S + 1 * 8)(x13) 233 ld s2, (PCB_S + 2 * 8)(x13) 234 ld s3, (PCB_S + 3 * 8)(x13) 235 ld s4, (PCB_S + 4 * 8)(x13) 236 ld s5, (PCB_S + 5 * 8)(x13) 237 ld s6, (PCB_S + 6 * 8)(x13) 238 ld s7, (PCB_S + 7 * 8)(x13) 239 ld s8, (PCB_S + 8 * 8)(x13) 240 ld s9, (PCB_S + 9 * 8)(x13) 241 ld s10, (PCB_S + 10 * 8)(x13) 242 ld s11, (PCB_S + 11 * 8)(x13) 243 244#ifdef FPE 245 /* Is FPE enabled for new thread? */ 246 ld t0, TD_FRAME(a0) 247 ld t1, (TF_SSTATUS)(t0) 248 li t2, SSTATUS_FS_MASK 249 and t3, t1, t2 250 beqz t3, 1f /* No, skip. */ 251 252 /* Restore registers. */ 253 __fpe_state_load x13 2541: 255#endif 256 257 ret 258END(cpu_throw) 259 260/* 261 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 262 * 263 * a0 = old 264 * a1 = new 265 * a2 = mtx 266 * x3 to x7, x16 and x17 are caller saved 267 */ 268ENTRY(cpu_switch) 269 /* Store the new curthread */ 270 sd a1, PC_CURTHREAD(gp) 271 /* And the new pcb */ 272 ld x13, TD_PCB(a1) 273 sd x13, PC_CURPCB(gp) 274 275 /* Save the old context. */ 276 ld x13, TD_PCB(a0) 277 278 /* Store ra, sp and the callee-saved registers */ 279 sd ra, (PCB_RA)(x13) 280 sd sp, (PCB_SP)(x13) 281 sd tp, (PCB_TP)(x13) 282 283 /* s[0-11] */ 284 sd s0, (PCB_S + 0 * 8)(x13) 285 sd s1, (PCB_S + 1 * 8)(x13) 286 sd s2, (PCB_S + 2 * 8)(x13) 287 sd s3, (PCB_S + 3 * 8)(x13) 288 sd s4, (PCB_S + 4 * 8)(x13) 289 sd s5, (PCB_S + 5 * 8)(x13) 290 sd s6, (PCB_S + 6 * 8)(x13) 291 sd s7, (PCB_S + 7 * 8)(x13) 292 sd s8, (PCB_S + 8 * 8)(x13) 293 sd s9, (PCB_S + 9 * 8)(x13) 294 sd s10, (PCB_S + 10 * 8)(x13) 295 sd s11, (PCB_S + 11 * 8)(x13) 296 297#ifdef FPE 298 /* 299 * Is FPE enabled and is it in dirty state 300 * for the old thread? 301 */ 302 ld t0, TD_FRAME(a0) 303 ld t1, (TF_SSTATUS)(t0) 304 li t2, SSTATUS_FS_MASK 305 and t3, t1, t2 306 li t2, SSTATUS_FS_DIRTY 307 bne t3, t2, 1f /* No, skip. */ 308 309 /* Yes, mark FPE state clean and save registers. */ 310 li t2, ~SSTATUS_FS_MASK 311 and t3, t1, t2 312 li t2, SSTATUS_FS_CLEAN 313 or t3, t3, t2 314 sd t3, (TF_SSTATUS)(t0) 315 316 __fpe_state_save x13 3171: 318#endif 319 320 /* Activate the new thread's pmap */ 321 mv s0, a0 322 mv s1, a1 323 mv s2, a2 324 mv a0, a1 325 call _C_LABEL(pmap_activate_sw) 326 mv a1, s1 327 328 /* Release the old thread */ 329 sd s2, TD_LOCK(s0) 330#if defined(SCHED_ULE) && defined(SMP) 331 /* Spin if TD_LOCK points to a blocked_lock */ 332 la s2, _C_LABEL(blocked_lock) 3331: 334 ld t0, TD_LOCK(a1) 335 beq t0, s2, 1b 336#endif 337 /* 338 * Restore the saved context. 339 */ 340 ld x13, TD_PCB(a1) 341 342 /* Restore the registers */ 343 ld tp, (PCB_TP)(x13) 344 ld ra, (PCB_RA)(x13) 345 ld sp, (PCB_SP)(x13) 346 347 /* s[0-11] */ 348 ld s0, (PCB_S + 0 * 8)(x13) 349 ld s1, (PCB_S + 1 * 8)(x13) 350 ld s2, (PCB_S + 2 * 8)(x13) 351 ld s3, (PCB_S + 3 * 8)(x13) 352 ld s4, (PCB_S + 4 * 8)(x13) 353 ld s5, (PCB_S + 5 * 8)(x13) 354 ld s6, (PCB_S + 6 * 8)(x13) 355 ld s7, (PCB_S + 7 * 8)(x13) 356 ld s8, (PCB_S + 8 * 8)(x13) 357 ld s9, (PCB_S + 9 * 8)(x13) 358 ld s10, (PCB_S + 10 * 8)(x13) 359 ld s11, (PCB_S + 11 * 8)(x13) 360 361#ifdef FPE 362 /* Is FPE enabled for new thread? */ 363 ld t0, TD_FRAME(a1) 364 ld t1, (TF_SSTATUS)(t0) 365 li t2, SSTATUS_FS_MASK 366 and t3, t1, t2 367 beqz t3, 1f /* No, skip. */ 368 369 /* Restore registers. */ 370 __fpe_state_load x13 3711: 372#endif 373 374 ret 375.Lcpu_switch_panic_str: 376 .asciz "cpu_switch: %p\0" 377END(cpu_switch) 378 379/* 380 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, 381 * struct trapframe *frame) 382 */ 383 384ENTRY(fork_trampoline) 385 mv a0, s0 386 mv a1, s1 387 mv a2, sp 388 call _C_LABEL(fork_exit) 389 390 /* Restore sstatus */ 391 ld t0, (TF_SSTATUS)(sp) 392 /* Ensure interrupts disabled */ 393 li t1, ~SSTATUS_SIE 394 and t0, t0, t1 395 csrw sstatus, t0 396 397 /* Restore exception program counter */ 398 ld t0, (TF_SEPC)(sp) 399 csrw sepc, t0 400 401 /* Restore the registers */ 402 ld t0, (TF_T + 0 * 8)(sp) 403 ld t1, (TF_T + 1 * 8)(sp) 404 ld t2, (TF_T + 2 * 8)(sp) 405 ld t3, (TF_T + 3 * 8)(sp) 406 ld t4, (TF_T + 4 * 8)(sp) 407 ld t5, (TF_T + 5 * 8)(sp) 408 ld t6, (TF_T + 6 * 8)(sp) 409 410 ld s0, (TF_S + 0 * 8)(sp) 411 ld s1, (TF_S + 1 * 8)(sp) 412 ld s2, (TF_S + 2 * 8)(sp) 413 ld s3, (TF_S + 3 * 8)(sp) 414 ld s4, (TF_S + 4 * 8)(sp) 415 ld s5, (TF_S + 5 * 8)(sp) 416 ld s6, (TF_S + 6 * 8)(sp) 417 ld s7, (TF_S + 7 * 8)(sp) 418 ld s8, (TF_S + 8 * 8)(sp) 419 ld s9, (TF_S + 9 * 8)(sp) 420 ld s10, (TF_S + 10 * 8)(sp) 421 ld s11, (TF_S + 11 * 8)(sp) 422 423 ld a0, (TF_A + 0 * 8)(sp) 424 ld a1, (TF_A + 1 * 8)(sp) 425 ld a2, (TF_A + 2 * 8)(sp) 426 ld a3, (TF_A + 3 * 8)(sp) 427 ld a4, (TF_A + 4 * 8)(sp) 428 ld a5, (TF_A + 5 * 8)(sp) 429 ld a6, (TF_A + 6 * 8)(sp) 430 ld a7, (TF_A + 7 * 8)(sp) 431 432 /* Load user ra and sp */ 433 ld ra, (TF_RA)(sp) 434 435 /* 436 * Store our pcpup on stack, we will load it back 437 * on kernel mode trap. 438 */ 439 sd gp, (TF_SIZE)(sp) 440 ld gp, (TF_GP)(sp) 441 442 /* Save kernel stack so we can use it doing a user trap */ 443 addi sp, sp, TF_SIZE 444 csrw sscratch, sp 445 446 /* Load user stack */ 447 ld sp, (TF_SP - TF_SIZE)(sp) 448 449 sret 450END(fork_trampoline) 451 452ENTRY(savectx) 453 /* Store ra, sp and the callee-saved registers */ 454 sd ra, (PCB_RA)(a0) 455 sd sp, (PCB_SP)(a0) 456 sd tp, (PCB_TP)(a0) 457 458 /* s[0-11] */ 459 sd s0, (PCB_S + 0 * 8)(a0) 460 sd s1, (PCB_S + 1 * 8)(a0) 461 sd s2, (PCB_S + 2 * 8)(a0) 462 sd s3, (PCB_S + 3 * 8)(a0) 463 sd s4, (PCB_S + 4 * 8)(a0) 464 sd s5, (PCB_S + 5 * 8)(a0) 465 sd s6, (PCB_S + 6 * 8)(a0) 466 sd s7, (PCB_S + 7 * 8)(a0) 467 sd s8, (PCB_S + 8 * 8)(a0) 468 sd s9, (PCB_S + 9 * 8)(a0) 469 sd s10, (PCB_S + 10 * 8)(a0) 470 sd s11, (PCB_S + 11 * 8)(a0) 471 472#ifdef FPE 473 __fpe_state_save a0 474#endif 475 ret 476END(savectx) 477