1/*- 2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#include "assym.inc" 36#include "opt_sched.h" 37 38#include <machine/param.h> 39#include <machine/asm.h> 40#include <machine/riscvreg.h> 41#include <machine/pte.h> 42.macro __fpe_state_save p 43 /* 44 * Enable FPE usage in supervisor mode, 45 * so we can access registers. 46 */ 47 li t0, SSTATUS_FS_INITIAL 48 csrs sstatus, t0 49 50 /* Store registers */ 51 frcsr t0 52 sd t0, (PCB_FCSR)(\p) 53 fsd f0, (PCB_X + 0 * 16)(\p) 54 fsd f1, (PCB_X + 1 * 16)(\p) 55 fsd f2, (PCB_X + 2 * 16)(\p) 56 fsd f3, (PCB_X + 3 * 16)(\p) 57 fsd f4, (PCB_X + 4 * 16)(\p) 58 fsd f5, (PCB_X + 5 * 16)(\p) 59 fsd f6, (PCB_X + 6 * 16)(\p) 60 fsd f7, (PCB_X + 7 * 16)(\p) 61 fsd f8, (PCB_X + 8 * 16)(\p) 62 fsd f9, (PCB_X + 9 * 16)(\p) 63 fsd f10, (PCB_X + 10 * 16)(\p) 64 fsd f11, (PCB_X + 11 * 16)(\p) 65 fsd f12, (PCB_X + 12 * 16)(\p) 66 fsd f13, (PCB_X + 13 * 16)(\p) 67 fsd f14, (PCB_X + 14 * 16)(\p) 68 fsd f15, (PCB_X + 15 * 16)(\p) 69 fsd f16, (PCB_X + 16 * 16)(\p) 70 fsd f17, (PCB_X + 17 * 16)(\p) 71 fsd f18, (PCB_X + 18 * 16)(\p) 72 fsd f19, (PCB_X + 19 * 16)(\p) 73 fsd f20, (PCB_X + 20 * 16)(\p) 74 fsd f21, (PCB_X + 21 * 16)(\p) 75 fsd f22, (PCB_X + 22 * 16)(\p) 76 fsd f23, (PCB_X + 23 * 16)(\p) 77 fsd f24, (PCB_X + 24 * 16)(\p) 78 fsd f25, (PCB_X + 25 * 16)(\p) 79 fsd f26, (PCB_X + 26 * 16)(\p) 80 fsd f27, (PCB_X + 27 * 16)(\p) 81 fsd f28, (PCB_X + 28 * 16)(\p) 82 fsd f29, (PCB_X + 29 * 16)(\p) 83 fsd f30, (PCB_X + 30 * 16)(\p) 84 fsd f31, (PCB_X + 31 * 16)(\p) 85 86 /* Disable FPE usage in supervisor mode. */ 87 li t0, SSTATUS_FS_MASK 88 csrc sstatus, t0 89.endm 90 91.macro __fpe_state_load p 92 /* 93 * Enable FPE usage in supervisor mode, 94 * so we can access registers. 95 */ 96 li t0, SSTATUS_FS_INITIAL 97 csrs sstatus, t0 98 99 /* Restore registers */ 100 ld t0, (PCB_FCSR)(\p) 101 fscsr t0 102 fld f0, (PCB_X + 0 * 16)(\p) 103 fld f1, (PCB_X + 1 * 16)(\p) 104 fld f2, (PCB_X + 2 * 16)(\p) 105 fld f3, (PCB_X + 3 * 16)(\p) 106 fld f4, (PCB_X + 4 * 16)(\p) 107 fld f5, (PCB_X + 5 * 16)(\p) 108 fld f6, (PCB_X + 6 * 16)(\p) 109 fld f7, (PCB_X + 7 * 16)(\p) 110 fld f8, (PCB_X + 8 * 16)(\p) 111 fld f9, (PCB_X + 9 * 16)(\p) 112 fld f10, (PCB_X + 10 * 16)(\p) 113 fld f11, (PCB_X + 11 * 16)(\p) 114 fld f12, (PCB_X + 12 * 16)(\p) 115 fld f13, (PCB_X + 13 * 16)(\p) 116 fld f14, (PCB_X + 14 * 16)(\p) 117 fld f15, (PCB_X + 15 * 16)(\p) 118 fld f16, (PCB_X + 16 * 16)(\p) 119 fld f17, (PCB_X + 17 * 16)(\p) 120 fld f18, (PCB_X + 18 * 16)(\p) 121 fld f19, (PCB_X + 19 * 16)(\p) 122 fld f20, (PCB_X + 20 * 16)(\p) 123 fld f21, (PCB_X + 21 * 16)(\p) 124 fld f22, (PCB_X + 22 * 16)(\p) 125 fld f23, (PCB_X + 23 * 16)(\p) 126 fld f24, (PCB_X + 24 * 16)(\p) 127 fld f25, (PCB_X + 25 * 16)(\p) 128 fld f26, (PCB_X + 26 * 16)(\p) 129 fld f27, (PCB_X + 27 * 16)(\p) 130 fld f28, (PCB_X + 28 * 16)(\p) 131 fld f29, (PCB_X + 29 * 16)(\p) 132 fld f30, (PCB_X + 30 * 16)(\p) 133 fld f31, (PCB_X + 31 * 16)(\p) 134 135 /* Disable FPE usage in supervisor mode. */ 136 li t0, SSTATUS_FS_MASK 137 csrc sstatus, t0 138.endm 139 140/* 141 * void 142 * fpe_state_save(struct thread *td) 143 */ 144ENTRY(fpe_state_save) 145 /* Get pointer to PCB */ 146 ld a0, TD_PCB(a0) 147 __fpe_state_save a0 148 ret 149END(fpe_state_save) 150 151/* 152 * void 153 * fpe_state_clear(void) 154 */ 155ENTRY(fpe_state_clear) 156 /* 157 * Enable FPE usage in supervisor mode, 158 * so we can access registers. 159 */ 160 li t0, SSTATUS_FS_INITIAL 161 csrs sstatus, t0 162 163 fscsr zero 164 fcvt.d.l f0, zero 165 fcvt.d.l f1, zero 166 fcvt.d.l f2, zero 167 fcvt.d.l f3, zero 168 fcvt.d.l f4, zero 169 fcvt.d.l f5, zero 170 fcvt.d.l f6, zero 171 fcvt.d.l f7, zero 172 fcvt.d.l f8, zero 173 fcvt.d.l f9, zero 174 fcvt.d.l f10, zero 175 fcvt.d.l f11, zero 176 fcvt.d.l f12, zero 177 fcvt.d.l f13, zero 178 fcvt.d.l f14, zero 179 fcvt.d.l f15, zero 180 fcvt.d.l f16, zero 181 fcvt.d.l f17, zero 182 fcvt.d.l f18, zero 183 fcvt.d.l f19, zero 184 fcvt.d.l f20, zero 185 fcvt.d.l f21, zero 186 fcvt.d.l f22, zero 187 fcvt.d.l f23, zero 188 fcvt.d.l f24, zero 189 fcvt.d.l f25, zero 190 fcvt.d.l f26, zero 191 fcvt.d.l f27, zero 192 fcvt.d.l f28, zero 193 fcvt.d.l f29, zero 194 fcvt.d.l f30, zero 195 fcvt.d.l f31, zero 196 197 /* Disable FPE usage in supervisor mode. */ 198 li t0, SSTATUS_FS_MASK 199 csrc sstatus, t0 200 201 ret 202END(fpe_state_clear) 203 204/* 205 * void cpu_throw(struct thread *old __unused, struct thread *new) 206 */ 207ENTRY(cpu_throw) 208 /* Activate the new thread's pmap. */ 209 mv s0, a1 210 mv a0, a1 211 call _C_LABEL(pmap_activate_sw) 212 mv a0, s0 213 214 /* Store the new curthread */ 215 sd a0, PC_CURTHREAD(tp) 216 /* And the new pcb */ 217 ld x13, TD_PCB(a0) 218 sd x13, PC_CURPCB(tp) 219 220 /* Load registers */ 221 ld ra, (PCB_RA)(x13) 222 ld sp, (PCB_SP)(x13) 223 224 /* s[0-11] */ 225 ld s0, (PCB_S + 0 * 8)(x13) 226 ld s1, (PCB_S + 1 * 8)(x13) 227 ld s2, (PCB_S + 2 * 8)(x13) 228 ld s3, (PCB_S + 3 * 8)(x13) 229 ld s4, (PCB_S + 4 * 8)(x13) 230 ld s5, (PCB_S + 5 * 8)(x13) 231 ld s6, (PCB_S + 6 * 8)(x13) 232 ld s7, (PCB_S + 7 * 8)(x13) 233 ld s8, (PCB_S + 8 * 8)(x13) 234 ld s9, (PCB_S + 9 * 8)(x13) 235 ld s10, (PCB_S + 10 * 8)(x13) 236 ld s11, (PCB_S + 11 * 8)(x13) 237 238 /* Is FPE enabled for new thread? */ 239 ld t0, TD_FRAME(a0) 240 ld t1, (TF_SSTATUS)(t0) 241 li t2, SSTATUS_FS_MASK 242 and t3, t1, t2 243 beqz t3, 1f /* No, skip. */ 244 245 /* Restore registers. */ 246 __fpe_state_load x13 2471: 248 ret 249END(cpu_throw) 250 251/* 252 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 253 * 254 * a0 = old 255 * a1 = new 256 * a2 = mtx 257 * x3 to x7, x16 and x17 are caller saved 258 */ 259ENTRY(cpu_switch) 260 /* Store the new curthread */ 261 sd a1, PC_CURTHREAD(tp) 262 /* And the new pcb */ 263 ld x13, TD_PCB(a1) 264 sd x13, PC_CURPCB(tp) 265 266 /* Save the old context. */ 267 ld x13, TD_PCB(a0) 268 269 /* Store ra, sp and the callee-saved registers */ 270 sd ra, (PCB_RA)(x13) 271 sd sp, (PCB_SP)(x13) 272 273 /* s[0-11] */ 274 sd s0, (PCB_S + 0 * 8)(x13) 275 sd s1, (PCB_S + 1 * 8)(x13) 276 sd s2, (PCB_S + 2 * 8)(x13) 277 sd s3, (PCB_S + 3 * 8)(x13) 278 sd s4, (PCB_S + 4 * 8)(x13) 279 sd s5, (PCB_S + 5 * 8)(x13) 280 sd s6, (PCB_S + 6 * 8)(x13) 281 sd s7, (PCB_S + 7 * 8)(x13) 282 sd s8, (PCB_S + 8 * 8)(x13) 283 sd s9, (PCB_S + 9 * 8)(x13) 284 sd s10, (PCB_S + 10 * 8)(x13) 285 sd s11, (PCB_S + 11 * 8)(x13) 286 287 /* 288 * Is FPE enabled and is it in dirty state 289 * for the old thread? 290 */ 291 ld t0, TD_FRAME(a0) 292 ld t1, (TF_SSTATUS)(t0) 293 li t2, SSTATUS_FS_MASK 294 and t3, t1, t2 295 li t2, SSTATUS_FS_DIRTY 296 bne t3, t2, 1f /* No, skip. */ 297 298 /* Yes, mark FPE state clean and save registers. */ 299 li t2, ~SSTATUS_FS_MASK 300 and t3, t1, t2 301 li t2, SSTATUS_FS_CLEAN 302 or t3, t3, t2 303 sd t3, (TF_SSTATUS)(t0) 304 305 __fpe_state_save x13 3061: 307 308 /* Activate the new thread's pmap */ 309 mv s0, a0 310 mv s1, a1 311 mv s2, a2 312 mv a0, a1 313 call _C_LABEL(pmap_activate_sw) 314 mv a1, s1 315 316 /* Release the old thread */ 317 sd s2, TD_LOCK(s0) 318#if defined(SCHED_ULE) && defined(SMP) 319 /* Spin if TD_LOCK points to a blocked_lock */ 320 la s2, _C_LABEL(blocked_lock) 3211: 322 ld t0, TD_LOCK(a1) 323 beq t0, s2, 1b 324#endif 325 /* 326 * Restore the saved context. 327 */ 328 ld x13, TD_PCB(a1) 329 330 /* Restore the registers */ 331 ld ra, (PCB_RA)(x13) 332 ld sp, (PCB_SP)(x13) 333 334 /* s[0-11] */ 335 ld s0, (PCB_S + 0 * 8)(x13) 336 ld s1, (PCB_S + 1 * 8)(x13) 337 ld s2, (PCB_S + 2 * 8)(x13) 338 ld s3, (PCB_S + 3 * 8)(x13) 339 ld s4, (PCB_S + 4 * 8)(x13) 340 ld s5, (PCB_S + 5 * 8)(x13) 341 ld s6, (PCB_S + 6 * 8)(x13) 342 ld s7, (PCB_S + 7 * 8)(x13) 343 ld s8, (PCB_S + 8 * 8)(x13) 344 ld s9, (PCB_S + 9 * 8)(x13) 345 ld s10, (PCB_S + 10 * 8)(x13) 346 ld s11, (PCB_S + 11 * 8)(x13) 347 348 /* Is FPE enabled for new thread? */ 349 ld t0, TD_FRAME(a1) 350 ld t1, (TF_SSTATUS)(t0) 351 li t2, SSTATUS_FS_MASK 352 and t3, t1, t2 353 beqz t3, 1f /* No, skip. */ 354 355 /* Restore registers. */ 356 __fpe_state_load x13 3571: 358 ret 359END(cpu_switch) 360 361/* 362 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, 363 * struct trapframe *frame) 364 */ 365 366ENTRY(fork_trampoline) 367 mv a0, s0 368 mv a1, s1 369 mv a2, sp 370 call _C_LABEL(fork_exit) 371 372 /* Restore sstatus */ 373 ld t0, (TF_SSTATUS)(sp) 374 /* Ensure interrupts disabled */ 375 li t1, ~SSTATUS_SIE 376 and t0, t0, t1 377 csrw sstatus, t0 378 379 /* Restore exception program counter */ 380 ld t0, (TF_SEPC)(sp) 381 csrw sepc, t0 382 383 /* Restore the registers */ 384 ld t0, (TF_T + 0 * 8)(sp) 385 ld t1, (TF_T + 1 * 8)(sp) 386 ld t2, (TF_T + 2 * 8)(sp) 387 ld t3, (TF_T + 3 * 8)(sp) 388 ld t4, (TF_T + 4 * 8)(sp) 389 ld t5, (TF_T + 5 * 8)(sp) 390 ld t6, (TF_T + 6 * 8)(sp) 391 392 ld s0, (TF_S + 0 * 8)(sp) 393 ld s1, (TF_S + 1 * 8)(sp) 394 ld s2, (TF_S + 2 * 8)(sp) 395 ld s3, (TF_S + 3 * 8)(sp) 396 ld s4, (TF_S + 4 * 8)(sp) 397 ld s5, (TF_S + 5 * 8)(sp) 398 ld s6, (TF_S + 6 * 8)(sp) 399 ld s7, (TF_S + 7 * 8)(sp) 400 ld s8, (TF_S + 8 * 8)(sp) 401 ld s9, (TF_S + 9 * 8)(sp) 402 ld s10, (TF_S + 10 * 8)(sp) 403 ld s11, (TF_S + 11 * 8)(sp) 404 405 ld a0, (TF_A + 0 * 8)(sp) 406 ld a1, (TF_A + 1 * 8)(sp) 407 ld a2, (TF_A + 2 * 8)(sp) 408 ld a3, (TF_A + 3 * 8)(sp) 409 ld a4, (TF_A + 4 * 8)(sp) 410 ld a5, (TF_A + 5 * 8)(sp) 411 ld a6, (TF_A + 6 * 8)(sp) 412 ld a7, (TF_A + 7 * 8)(sp) 413 414 /* Load user ra and gp */ 415 ld ra, (TF_RA)(sp) 416 ld gp, (TF_GP)(sp) 417 418 /* 419 * Store our pcpup on stack, we will load it back 420 * on kernel mode trap. 421 */ 422 sd tp, (TF_SIZE)(sp) 423 ld tp, (TF_TP)(sp) 424 425 /* Save kernel stack so we can use it doing a user trap */ 426 addi sp, sp, TF_SIZE 427 csrw sscratch, sp 428 429 /* Load user stack */ 430 ld sp, (TF_SP - TF_SIZE)(sp) 431 432 sret 433END(fork_trampoline) 434 435ENTRY(savectx) 436 /* Store ra, sp and the callee-saved registers */ 437 sd ra, (PCB_RA)(a0) 438 sd sp, (PCB_SP)(a0) 439 sd tp, (PCB_TP)(a0) 440 sd gp, (PCB_GP)(a0) 441 442 /* s[0-11] */ 443 sd s0, (PCB_S + 0 * 8)(a0) 444 sd s1, (PCB_S + 1 * 8)(a0) 445 sd s2, (PCB_S + 2 * 8)(a0) 446 sd s3, (PCB_S + 3 * 8)(a0) 447 sd s4, (PCB_S + 4 * 8)(a0) 448 sd s5, (PCB_S + 5 * 8)(a0) 449 sd s6, (PCB_S + 6 * 8)(a0) 450 sd s7, (PCB_S + 7 * 8)(a0) 451 sd s8, (PCB_S + 8 * 8)(a0) 452 sd s9, (PCB_S + 9 * 8)(a0) 453 sd s10, (PCB_S + 10 * 8)(a0) 454 sd s11, (PCB_S + 11 * 8)(a0) 455 456 __fpe_state_save a0 457 ret 458END(savectx) 459