1/*- 2 * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#include <machine/asm.h> 36__FBSDID("$FreeBSD$"); 37 38#include "assym.s" 39 40#include <machine/trap.h> 41#include <machine/riscvreg.h> 42 43.macro save_registers el 44 addi sp, sp, -(TF_SIZE) 45 46 sd ra, (TF_RA)(sp) 47 sd tp, (TF_TP)(sp) 48 49.if \el == 0 /* We came from userspace. Load our pcpu */ 50 sd gp, (TF_GP)(sp) 51 ld gp, (TF_SIZE)(sp) 52.endif 53 54 sd t0, (TF_T + 0 * 8)(sp) 55 sd t1, (TF_T + 1 * 8)(sp) 56 sd t2, (TF_T + 2 * 8)(sp) 57 sd t3, (TF_T + 3 * 8)(sp) 58 sd t4, (TF_T + 4 * 8)(sp) 59 sd t5, (TF_T + 5 * 8)(sp) 60 sd t6, (TF_T + 6 * 8)(sp) 61 62 sd s0, (TF_S + 0 * 8)(sp) 63 sd s1, (TF_S + 1 * 8)(sp) 64 sd s2, (TF_S + 2 * 8)(sp) 65 sd s3, (TF_S + 3 * 8)(sp) 66 sd s4, (TF_S + 4 * 8)(sp) 67 sd s5, (TF_S + 5 * 8)(sp) 68 sd s6, (TF_S + 6 * 8)(sp) 69 sd s7, (TF_S + 7 * 8)(sp) 70 sd s8, (TF_S + 8 * 8)(sp) 71 sd s9, (TF_S + 9 * 8)(sp) 72 sd s10, (TF_S + 10 * 8)(sp) 73 sd s11, (TF_S + 11 * 8)(sp) 74 75 sd a0, (TF_A + 0 * 8)(sp) 76 sd a1, (TF_A + 1 * 8)(sp) 77 sd a2, (TF_A + 2 * 8)(sp) 78 sd a3, (TF_A + 3 * 8)(sp) 79 sd a4, (TF_A + 4 * 8)(sp) 80 sd a5, (TF_A + 5 * 8)(sp) 81 sd a6, (TF_A + 6 * 8)(sp) 82 sd a7, (TF_A + 7 * 8)(sp) 83 84#if 0 85 /* XXX: temporary test: spin if stack is not kernel one */ 86.if \el == 1 /* kernel */ 87 mv t0, sp 88 srli t0, t0, 63 891: 90 beqz t0, 1b 91.endif 92#endif 93 94.if \el == 1 95 /* Store kernel sp */ 96 sd sp, (TF_SP)(sp) 97.else 98 /* Store user sp */ 99 csrr t0, sscratch 100 sd t0, (TF_SP)(sp) 101.endif 102 li t0, 0 103 csrw sscratch, t0 104 csrr t0, sepc 105 sd t0, (TF_SEPC)(sp) 106 csrr t0, sstatus 107 sd t0, (TF_SSTATUS)(sp) 108 csrr t0, sbadaddr 109 sd t0, (TF_SBADADDR)(sp) 110 csrr t0, scause 111 sd t0, (TF_SCAUSE)(sp) 112.endm 113 114.macro load_registers el 115 ld t0, (TF_SSTATUS)(sp) 116.if \el == 0 117 /* Ensure user interrupts will be enabled on eret. */ 118 ori t0, t0, SSTATUS_PIE 119.else 120 /* 121 * Disable interrupts for supervisor mode exceptions. 122 * For user mode exceptions we have already done this 123 * in do_ast. 124 */ 125 li t1, ~SSTATUS_IE 126 and t0, t0, t1 127.endif 128 csrw sstatus, t0 129 130 ld t0, (TF_SEPC)(sp) 131 csrw sepc, t0 132 133.if \el == 0 134 /* We go to userspace. Load user sp */ 135 ld t0, (TF_SP)(sp) 136 csrw sscratch, t0 137 138 /* And store our pcpu */ 139 sd gp, (TF_SIZE)(sp) 140 ld gp, (TF_GP)(sp) 141.endif 142 143 ld ra, (TF_RA)(sp) 144 ld tp, (TF_TP)(sp) 145 146 ld t0, (TF_T + 0 * 8)(sp) 147 ld t1, (TF_T + 1 * 8)(sp) 148 ld t2, (TF_T + 2 * 8)(sp) 149 ld t3, (TF_T + 3 * 8)(sp) 150 ld t4, (TF_T + 4 * 8)(sp) 151 ld t5, (TF_T + 5 * 8)(sp) 152 ld t6, (TF_T + 6 * 8)(sp) 153 154 ld s0, (TF_S + 0 * 8)(sp) 155 ld s1, (TF_S + 1 * 8)(sp) 156 ld s2, (TF_S + 2 * 8)(sp) 157 ld s3, (TF_S + 3 * 8)(sp) 158 ld s4, (TF_S + 4 * 8)(sp) 159 ld s5, (TF_S + 5 * 8)(sp) 160 ld s6, (TF_S + 6 * 8)(sp) 161 ld s7, (TF_S + 7 * 8)(sp) 162 ld s8, (TF_S + 8 * 8)(sp) 163 ld s9, (TF_S + 9 * 8)(sp) 164 ld s10, (TF_S + 10 * 8)(sp) 165 ld s11, (TF_S + 11 * 8)(sp) 166 167 ld a0, (TF_A + 0 * 8)(sp) 168 ld a1, (TF_A + 1 * 8)(sp) 169 ld a2, (TF_A + 2 * 8)(sp) 170 ld a3, (TF_A + 3 * 8)(sp) 171 ld a4, (TF_A + 4 * 8)(sp) 172 ld a5, (TF_A + 5 * 8)(sp) 173 ld a6, (TF_A + 6 * 8)(sp) 174 ld a7, (TF_A + 7 * 8)(sp) 175 176 addi sp, sp, (TF_SIZE) 177.endm 178 179.macro do_ast 180 /* Disable interrupts */ 181 csrr a4, sstatus 1821: 183 csrci sstatus, SSTATUS_IE 184 185 ld a1, PC_CURTHREAD(gp) 186 lw a2, TD_FLAGS(a1) 187 188 li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED) 189 and a2, a2, a3 190 beqz a2, 2f 191 192 /* Restore interrupts */ 193 andi a4, a4, SSTATUS_IE 194 csrs sstatus, a4 195 196 /* Handle the ast */ 197 mv a0, sp 198 call _C_LABEL(ast) 199 200 /* Re-check for new ast scheduled */ 201 j 1b 2022: 203.endm 204 205ENTRY(cpu_exception_handler_supervisor) 206 save_registers 1 207 mv a0, sp 208 call _C_LABEL(do_trap_supervisor) 209 load_registers 1 210 eret 211END(cpu_exception_handler_supervisor) 212 213ENTRY(cpu_exception_handler_user) 214 csrrw sp, sscratch, sp 215 save_registers 0 216 mv a0, sp 217 call _C_LABEL(do_trap_user) 218 do_ast 219 load_registers 0 220 csrrw sp, sscratch, sp 221 eret 222END(cpu_exception_handler_user) 223 224/* 225 * Trap handlers 226 */ 227 .text 228bad_trap: 229 j bad_trap 230 231user_trap: 232 /* Save state */ 233 csrrw sp, mscratch, sp 234 addi sp, sp, -64 235 sd t0, (8 * 0)(sp) 236 sd t1, (8 * 1)(sp) 237 sd t2, (8 * 2)(sp) 238 sd t3, (8 * 3)(sp) 239 sd t4, (8 * 4)(sp) 240 sd t5, (8 * 5)(sp) 241 sd a0, (8 * 7)(sp) 242 243 la t2, _C_LABEL(cpu_exception_handler_user) 244 245 csrr t0, mcause 246 bltz t0, machine_interrupt 247 j exit_mrts 248 249supervisor_trap: 250 /* Save state */ 251 csrrw sp, mscratch, sp 252 addi sp, sp, -64 253 sd t0, (8 * 0)(sp) 254 sd t1, (8 * 1)(sp) 255 sd t2, (8 * 2)(sp) 256 sd t3, (8 * 3)(sp) 257 sd t4, (8 * 4)(sp) 258 sd t5, (8 * 5)(sp) 259 sd a0, (8 * 7)(sp) 260 261 la t2, _C_LABEL(cpu_exception_handler_supervisor) 262 263 csrr t0, mcause 264 bltz t0, machine_interrupt 265 266 li t1, EXCP_SMODE_ENV_CALL 267 beq t0, t1, supervisor_call 268 j exit_mrts 269 270machine_interrupt: 271 /* Type of interrupt ? */ 272 csrr t0, mcause 273 andi t0, t0, 3 274 li t1, 0 275 beq t1, t0, software_interrupt 276 li t1, 1 277 beq t1, t0, timer_interrupt 278 li t1, 2 279 beq t1, t0, htif_interrupt 280 281 /* not reached */ 2821: 283 j 1b 284 285software_interrupt: 286 li t0, MIP_MSIP 287 csrc mip, t0 288 li t0, MIP_SSIP 289 csrs mip, t0 290 291 /* If PRV1 is PRV_U (user) then serve the trap */ 292 csrr t0, mstatus 293 li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT) 294 and t0, t0, t1 295 beqz t0, 1f 296 297 /* 298 * If PRV1 is supervisor and interrupts were enabled, 299 * then serve the trap. 300 */ 301 csrr t0, mstatus 302 li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)) 303 and t0, t0, t1 304 li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT)) 305 beq t0, t1, 1f 306 307 j exit 308 3091: 310 /* Serve a trap in supervisor mode */ 311 j exit_mrts 312 313timer_interrupt: 314 /* Disable machine timer interrupts */ 315 li t0, MIE_MTIE 316 csrc mie, t0 317 318 /* Clear machine pending */ 319 li t0, MIP_MTIP 320 csrc mip, t0 321 322 /* Post supervisor timer interrupt */ 323 li t0, MIP_STIP 324 csrs mip, t0 325 326 /* If PRV1 is PRV_U (user) then serve the trap */ 327 csrr t0, mstatus 328 li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT) 329 and t0, t0, t1 330 beqz t0, 1f 331 332 /* 333 * If PRV1 is supervisor and interrupts were enabled, 334 * then serve the trap. 335 */ 336 csrr t0, mstatus 337 li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)) 338 and t0, t0, t1 339 li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT)) 340 beq t0, t1, 1f 341 342 j exit 343 3441: 345 /* Serve a trap in supervisor mode */ 346 j exit_mrts 347 348htif_interrupt: 3491: 350 li t5, 0 351 csrrw t5, mfromhost, t5 352 beqz t5, 3f 353 354 /* Console PUT intr ? */ 355 mv t1, t5 356 li t0, 0x101 357 srli t1, t1, 48 358 bne t1, t0, 2f 359 /* Yes */ 360 la t0, console_intr 361 li t1, 1 362 sd t1, 0(t0) 363 364 /* Check if there is any other pending event */ 365 j 1b 366 3672: 368 /* Save entry */ 369 la t0, htif_ring 370 csrr t1, mhartid 371 li t4, (HTIF_RING_SIZE + 16) 372 mulw t4, t4, t1 373 add t0, t0, t4 374 li t4, (HTIF_RING_SIZE) 375 add t0, t0, t4 /* t0 == htif_ring_cursor */ 376 377 ld t1, 0(t0) /* load ptr to cursor */ 378 sd t5, 0(t1) /* put entry */ 379 li t4, 1 380 sd t4, 8(t1) /* mark used */ 381 ld t4, 16(t1) /* take next */ 382 /* Update cursor */ 383 sd t4, 0(t0) 384 385 /* Post supervisor software interrupt */ 386 li t0, MIP_SSIP 387 csrs mip, t0 388 389 /* Check if there is any other pending event */ 390 j 1b 391 3923: 393 j exit 394 395supervisor_call: 396 csrr t1, mepc 397 addi t1, t1, 4 /* Next instruction in t1 */ 398 li t4, ECALL_HTIF_CMD 399 beq t5, t4, htif_cmd 400 li t4, ECALL_HTIF_GET_ENTRY 401 beq t5, t4, htif_get_entry 402 li t4, ECALL_MTIMECMP 403 beq t5, t4, set_mtimecmp 404 li t4, ECALL_CLEAR_PENDING 405 beq t5, t4, clear_pending 406 li t4, ECALL_MCPUID_GET 407 beq t5, t4, mcpuid_get 408 li t4, ECALL_MIMPID_GET 409 beq t5, t4, mimpid_get 410 li t4, ECALL_SEND_IPI 411 beq t5, t4, send_ipi 412 li t4, ECALL_CLEAR_IPI 413 beq t5, t4, clear_ipi 414 li t4, ECALL_HTIF_LOWPUTC 415 beq t5, t4, htif_lowputc 416 li t4, ECALL_MIE_SET 417 beq t5, t4, mie_set 418 j exit_next_instr 419 420mie_set: 421 csrs mie, t6 422 j exit_next_instr 423 424mcpuid_get: 425 csrr t6, mcpuid 426 j exit_next_instr 427 428mimpid_get: 429 csrr t6, mimpid 430 j exit_next_instr 431 432send_ipi: 433 /* CPU mmio base in t6 */ 434 mv t0, t6 435 li t2, (CSR_IPI * XLEN) 436 add t0, t0, t2 /* t0 = CSR_IPI */ 437 li t2, 1 438 sd t2, 0(t0) 439 j exit_next_instr 440 441clear_ipi: 442 /* Do only clear if there are no new entries in HTIF ring */ 443 la t0, htif_ring 444 csrr t2, mhartid 445 li t4, (HTIF_RING_SIZE + 16) 446 mulw t4, t4, t2 447 add t0, t0, t4 448 li t4, (HTIF_RING_SIZE) 449 add t0, t0, t4 /* t0 == ptr to htif_ring_cursor */ 450 ld t2, 8(t0) /* load htif_ring_last */ 451 ld t2, 8(t2) /* load used */ 452 bnez t2, 1f 453 454 /* Clear supervisor software interrupt pending bit */ 455 li t0, MIP_SSIP 456 csrc mip, t0 457 4581: 459 j exit_next_instr 460 461htif_get_entry: 462 /* Get a htif_ring for current core */ 463 la t0, htif_ring 464 csrr t2, mhartid 465 li t4, (HTIF_RING_SIZE + 16) 466 mulw t4, t4, t2 467 add t0, t0, t4 468 li t4, (HTIF_RING_SIZE + 8) 469 add t0, t0, t4 /* t0 == htif_ring_last */ 470 471 /* Check for new entries */ 472 li t6, 0 /* preset return value */ 473 ld t2, 0(t0) /* load ptr to last */ 474 ld t4, 8(t2) /* get used */ 475 beqz t4, 1f /* No new entries. Exit */ 476 477 /* Get one */ 478 ld t6, 0(t2) /* get entry */ 479 li t4, 0 480 sd t4, 8(t2) /* mark free */ 481 sd t4, 0(t2) /* free entry, just in case */ 482 ld t4, 16(t2) /* take next */ 483 sd t4, 0(t0) /* update ptr to last */ 4841: 485 /* Exit. Result is stored in t6 */ 486 j exit_next_instr 487 488htif_cmd: 4891: 490 mv t0, t6 491 csrrw t0, mtohost, t0 492 bnez t0, 1b 493 j exit_next_instr 494 495htif_lowputc: 4961: 497 mv t0, t6 498 csrrw t0, mtohost, t0 499 bnez t0, 1b 500 5012: 502 li t4, 0 503 csrrw t5, mfromhost, t4 504 beqz t5, 2b 505 506 /* Console PUT intr ? */ 507 mv t2, t5 508 srli t2, t2, 48 509 li t3, 0x0101 510 beq t2, t3, 3f 511 512 /* Not a console PUT, so save entry */ 513 la t0, htif_ring 514 csrr t2, mhartid 515 li t4, (HTIF_RING_SIZE + 16) 516 mulw t4, t4, t2 517 add t0, t0, t4 518 li t4, (HTIF_RING_SIZE) 519 add t0, t0, t4 /* t0 == htif_ring_cursor */ 520 521 ld t2, 0(t0) /* load ptr to cursor */ 522 sd t5, 0(t2) /* put entry */ 523 li t4, 1 524 sd t4, 8(t2) /* mark used */ 525 ld t4, 16(t2) /* take next */ 526 /* Update cursor */ 527 sd t4, 0(t0) 528 529 /* Post supervisor software interrupt */ 530 li t0, MIP_SSIP 531 csrs mip, t0 532 533 /* Wait for console intr again */ 534 j 2b 535 5363: 537 j exit_next_instr 538 539set_mtimecmp: 540 csrr t2, stime 541 add t6, t6, t2 542 csrw mtimecmp, t6 543 544 /* Enable interrupts */ 545 li t0, (MIE_MTIE | MIE_STIE) 546 csrs mie, t0 547 j exit_next_instr 548 549clear_pending: 550 li t0, MIP_STIP 551 csrc mip, t0 552 j exit_next_instr 553 554/* 555 * Trap exit functions 556 */ 557exit_next_instr: 558 /* Next instruction is in t1 */ 559 csrw mepc, t1 560exit: 561 /* Restore state */ 562 ld t0, (8 * 0)(sp) 563 ld t1, (8 * 1)(sp) 564 ld t2, (8 * 2)(sp) 565 ld t3, (8 * 3)(sp) 566 ld t4, (8 * 4)(sp) 567 ld t5, (8 * 5)(sp) 568 ld a0, (8 * 7)(sp) 569 addi sp, sp, 64 570 csrrw sp, mscratch, sp 571 eret 572 573/* 574 * Redirect to supervisor 575 */ 576exit_mrts: 577 /* Setup exception handler */ 578 li t1, KERNBASE 579 add t2, t2, t1 580 csrw stvec, t2 581 582 /* Restore state */ 583 ld t0, (8 * 0)(sp) 584 ld t1, (8 * 1)(sp) 585 ld t2, (8 * 2)(sp) 586 ld t3, (8 * 3)(sp) 587 ld t4, (8 * 4)(sp) 588 ld t5, (8 * 5)(sp) 589 ld a0, (8 * 7)(sp) 590 addi sp, sp, 64 591 csrrw sp, mscratch, sp 592 593 /* Redirect to supervisor */ 594 mrts 595