1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2017 SiFive 5 */ 6 7#include <linux/init.h> 8#include <linux/linkage.h> 9 10#include <asm/asm.h> 11#include <asm/csr.h> 12#include <asm/unistd.h> 13#include <asm/thread_info.h> 14#include <asm/asm-offsets.h> 15#include <asm/errata_list.h> 16 17#if !IS_ENABLED(CONFIG_PREEMPTION) 18.set resume_kernel, restore_all 19#endif 20 21ENTRY(handle_exception) 22 /* 23 * If coming from userspace, preserve the user thread pointer and load 24 * the kernel thread pointer. If we came from the kernel, the scratch 25 * register will contain 0, and we should continue on the current TP. 26 */ 27 csrrw tp, CSR_SCRATCH, tp 28 bnez tp, _save_context 29 30_restore_kernel_tpsp: 31 csrr tp, CSR_SCRATCH 32 REG_S sp, TASK_TI_KERNEL_SP(tp) 33 34#ifdef CONFIG_VMAP_STACK 35 addi sp, sp, -(PT_SIZE_ON_STACK) 36 srli sp, sp, THREAD_SHIFT 37 andi sp, sp, 0x1 38 bnez sp, handle_kernel_stack_overflow 39 REG_L sp, TASK_TI_KERNEL_SP(tp) 40#endif 41 42_save_context: 43 REG_S sp, TASK_TI_USER_SP(tp) 44 REG_L sp, TASK_TI_KERNEL_SP(tp) 45 addi sp, sp, -(PT_SIZE_ON_STACK) 46 REG_S x1, PT_RA(sp) 47 REG_S x3, PT_GP(sp) 48 REG_S x5, PT_T0(sp) 49 REG_S x6, PT_T1(sp) 50 REG_S x7, PT_T2(sp) 51 REG_S x8, PT_S0(sp) 52 REG_S x9, PT_S1(sp) 53 REG_S x10, PT_A0(sp) 54 REG_S x11, PT_A1(sp) 55 REG_S x12, PT_A2(sp) 56 REG_S x13, PT_A3(sp) 57 REG_S x14, PT_A4(sp) 58 REG_S x15, PT_A5(sp) 59 REG_S x16, PT_A6(sp) 60 REG_S x17, PT_A7(sp) 61 REG_S x18, PT_S2(sp) 62 REG_S x19, PT_S3(sp) 63 REG_S x20, PT_S4(sp) 64 REG_S x21, PT_S5(sp) 65 REG_S x22, PT_S6(sp) 66 REG_S x23, PT_S7(sp) 67 REG_S x24, PT_S8(sp) 68 REG_S x25, PT_S9(sp) 69 REG_S x26, PT_S10(sp) 70 REG_S x27, PT_S11(sp) 71 REG_S x28, PT_T3(sp) 72 REG_S x29, PT_T4(sp) 73 REG_S x30, PT_T5(sp) 74 REG_S x31, PT_T6(sp) 75 76 /* 77 * Disable user-mode memory access as it should only be set in the 78 * actual user copy routines. 79 * 80 * Disable the FPU to detect illegal usage of floating point in kernel 81 * space. 82 */ 83 li t0, SR_SUM | SR_FS 84 85 REG_L s0, TASK_TI_USER_SP(tp) 86 csrrc s1, CSR_STATUS, t0 87 csrr s2, CSR_EPC 88 csrr s3, CSR_TVAL 89 csrr s4, CSR_CAUSE 90 csrr s5, CSR_SCRATCH 91 REG_S s0, PT_SP(sp) 92 REG_S s1, PT_STATUS(sp) 93 REG_S s2, PT_EPC(sp) 94 REG_S s3, PT_BADADDR(sp) 95 REG_S s4, PT_CAUSE(sp) 96 REG_S s5, PT_TP(sp) 97 98 /* 99 * Set the scratch register to 0, so that if a recursive exception 100 * occurs, the exception vector knows it came from the kernel 101 */ 102 csrw CSR_SCRATCH, x0 103 104 /* Load the global pointer */ 105.option push 106.option norelax 107 la gp, __global_pointer$ 108.option pop 109 110#ifdef CONFIG_TRACE_IRQFLAGS 111 call __trace_hardirqs_off 112#endif 113 114#ifdef CONFIG_CONTEXT_TRACKING 115 /* If previous state is in user mode, call context_tracking_user_exit. */ 116 li a0, SR_PP 117 and a0, s1, a0 118 bnez a0, skip_context_tracking 119 call context_tracking_user_exit 120skip_context_tracking: 121#endif 122 123 /* 124 * MSB of cause differentiates between 125 * interrupts and exceptions 126 */ 127 bge s4, zero, 1f 128 129 la ra, ret_from_exception 130 131 /* Handle interrupts */ 132 move a0, sp /* pt_regs */ 133 la a1, generic_handle_arch_irq 134 jr a1 1351: 136 /* 137 * Exceptions run with interrupts enabled or disabled depending on the 138 * state of SR_PIE in m/sstatus. 139 */ 140 andi t0, s1, SR_PIE 141 beqz t0, 1f 142 /* kprobes, entered via ebreak, must have interrupts disabled. */ 143 li t0, EXC_BREAKPOINT 144 beq s4, t0, 1f 145#ifdef CONFIG_TRACE_IRQFLAGS 146 call __trace_hardirqs_on 147#endif 148 csrs CSR_STATUS, SR_IE 149 1501: 151 la ra, ret_from_exception 152 /* Handle syscalls */ 153 li t0, EXC_SYSCALL 154 beq s4, t0, handle_syscall 155 156 /* Handle other exceptions */ 157 slli t0, s4, RISCV_LGPTR 158 la t1, excp_vect_table 159 la t2, excp_vect_table_end 160 move a0, sp /* pt_regs */ 161 add t0, t1, t0 162 /* Check if exception code lies within bounds */ 163 bgeu t0, t2, 1f 164 REG_L t0, 0(t0) 165 jr t0 1661: 167 tail do_trap_unknown 168 169handle_syscall: 170#ifdef CONFIG_RISCV_M_MODE 171 /* 172 * When running is M-Mode (no MMU config), MPIE does not get set. 173 * As a result, we need to force enable interrupts here because 174 * handle_exception did not do set SR_IE as it always sees SR_PIE 175 * being cleared. 176 */ 177 csrs CSR_STATUS, SR_IE 178#endif 179#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) 180 /* Recover a0 - a7 for system calls */ 181 REG_L a0, PT_A0(sp) 182 REG_L a1, PT_A1(sp) 183 REG_L a2, PT_A2(sp) 184 REG_L a3, PT_A3(sp) 185 REG_L a4, PT_A4(sp) 186 REG_L a5, PT_A5(sp) 187 REG_L a6, PT_A6(sp) 188 REG_L a7, PT_A7(sp) 189#endif 190 /* save the initial A0 value (needed in signal handlers) */ 191 REG_S a0, PT_ORIG_A0(sp) 192 /* 193 * Advance SEPC to avoid executing the original 194 * scall instruction on sret 195 */ 196 addi s2, s2, 0x4 197 REG_S s2, PT_EPC(sp) 198 /* Trace syscalls, but only if requested by the user. */ 199 REG_L t0, TASK_TI_FLAGS(tp) 200 andi t0, t0, _TIF_SYSCALL_WORK 201 bnez t0, handle_syscall_trace_enter 202check_syscall_nr: 203 /* Check to make sure we don't jump to a bogus syscall number. */ 204 li t0, __NR_syscalls 205 la s0, sys_ni_syscall 206 /* 207 * Syscall number held in a7. 208 * If syscall number is above allowed value, redirect to ni_syscall. 209 */ 210 bgeu a7, t0, 1f 211 /* Call syscall */ 212 la s0, sys_call_table 213 slli t0, a7, RISCV_LGPTR 214 add s0, s0, t0 215 REG_L s0, 0(s0) 2161: 217 jalr s0 218 219ret_from_syscall: 220 /* Set user a0 to kernel a0 */ 221 REG_S a0, PT_A0(sp) 222 /* 223 * We didn't execute the actual syscall. 224 * Seccomp already set return value for the current task pt_regs. 225 * (If it was configured with SECCOMP_RET_ERRNO/TRACE) 226 */ 227ret_from_syscall_rejected: 228#ifdef CONFIG_DEBUG_RSEQ 229 move a0, sp 230 call rseq_syscall 231#endif 232 /* Trace syscalls, but only if requested by the user. */ 233 REG_L t0, TASK_TI_FLAGS(tp) 234 andi t0, t0, _TIF_SYSCALL_WORK 235 bnez t0, handle_syscall_trace_exit 236 237ret_from_exception: 238 REG_L s0, PT_STATUS(sp) 239 csrc CSR_STATUS, SR_IE 240#ifdef CONFIG_TRACE_IRQFLAGS 241 call __trace_hardirqs_off 242#endif 243#ifdef CONFIG_RISCV_M_MODE 244 /* the MPP value is too large to be used as an immediate arg for addi */ 245 li t0, SR_MPP 246 and s0, s0, t0 247#else 248 andi s0, s0, SR_SPP 249#endif 250 bnez s0, resume_kernel 251 252resume_userspace: 253 /* Interrupts must be disabled here so flags are checked atomically */ 254 REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */ 255 andi s1, s0, _TIF_WORK_MASK 256 bnez s1, work_pending 257 258#ifdef CONFIG_CONTEXT_TRACKING 259 call context_tracking_user_enter 260#endif 261 262 /* Save unwound kernel stack pointer in thread_info */ 263 addi s0, sp, PT_SIZE_ON_STACK 264 REG_S s0, TASK_TI_KERNEL_SP(tp) 265 266 /* 267 * Save TP into the scratch register , so we can find the kernel data 268 * structures again. 269 */ 270 csrw CSR_SCRATCH, tp 271 272restore_all: 273#ifdef CONFIG_TRACE_IRQFLAGS 274 REG_L s1, PT_STATUS(sp) 275 andi t0, s1, SR_PIE 276 beqz t0, 1f 277 call __trace_hardirqs_on 278 j 2f 2791: 280 call __trace_hardirqs_off 2812: 282#endif 283 REG_L a0, PT_STATUS(sp) 284 /* 285 * The current load reservation is effectively part of the processor's 286 * state, in the sense that load reservations cannot be shared between 287 * different hart contexts. We can't actually save and restore a load 288 * reservation, so instead here we clear any existing reservation -- 289 * it's always legal for implementations to clear load reservations at 290 * any point (as long as the forward progress guarantee is kept, but 291 * we'll ignore that here). 292 * 293 * Dangling load reservations can be the result of taking a trap in the 294 * middle of an LR/SC sequence, but can also be the result of a taken 295 * forward branch around an SC -- which is how we implement CAS. As a 296 * result we need to clear reservations between the last CAS and the 297 * jump back to the new context. While it is unlikely the store 298 * completes, implementations are allowed to expand reservations to be 299 * arbitrarily large. 300 */ 301 REG_L a2, PT_EPC(sp) 302 REG_SC x0, a2, PT_EPC(sp) 303 304 csrw CSR_STATUS, a0 305 csrw CSR_EPC, a2 306 307 REG_L x1, PT_RA(sp) 308 REG_L x3, PT_GP(sp) 309 REG_L x4, PT_TP(sp) 310 REG_L x5, PT_T0(sp) 311 REG_L x6, PT_T1(sp) 312 REG_L x7, PT_T2(sp) 313 REG_L x8, PT_S0(sp) 314 REG_L x9, PT_S1(sp) 315 REG_L x10, PT_A0(sp) 316 REG_L x11, PT_A1(sp) 317 REG_L x12, PT_A2(sp) 318 REG_L x13, PT_A3(sp) 319 REG_L x14, PT_A4(sp) 320 REG_L x15, PT_A5(sp) 321 REG_L x16, PT_A6(sp) 322 REG_L x17, PT_A7(sp) 323 REG_L x18, PT_S2(sp) 324 REG_L x19, PT_S3(sp) 325 REG_L x20, PT_S4(sp) 326 REG_L x21, PT_S5(sp) 327 REG_L x22, PT_S6(sp) 328 REG_L x23, PT_S7(sp) 329 REG_L x24, PT_S8(sp) 330 REG_L x25, PT_S9(sp) 331 REG_L x26, PT_S10(sp) 332 REG_L x27, PT_S11(sp) 333 REG_L x28, PT_T3(sp) 334 REG_L x29, PT_T4(sp) 335 REG_L x30, PT_T5(sp) 336 REG_L x31, PT_T6(sp) 337 338 REG_L x2, PT_SP(sp) 339 340#ifdef CONFIG_RISCV_M_MODE 341 mret 342#else 343 sret 344#endif 345 346#if IS_ENABLED(CONFIG_PREEMPTION) 347resume_kernel: 348 REG_L s0, TASK_TI_PREEMPT_COUNT(tp) 349 bnez s0, restore_all 350 REG_L s0, TASK_TI_FLAGS(tp) 351 andi s0, s0, _TIF_NEED_RESCHED 352 beqz s0, restore_all 353 call preempt_schedule_irq 354 j restore_all 355#endif 356 357work_pending: 358 /* Enter slow path for supplementary processing */ 359 la ra, ret_from_exception 360 andi s1, s0, _TIF_NEED_RESCHED 361 bnez s1, work_resched 362work_notifysig: 363 /* Handle pending signals and notify-resume requests */ 364 csrs CSR_STATUS, SR_IE /* Enable interrupts for do_notify_resume() */ 365 move a0, sp /* pt_regs */ 366 move a1, s0 /* current_thread_info->flags */ 367 tail do_notify_resume 368work_resched: 369 tail schedule 370 371/* Slow paths for ptrace. */ 372handle_syscall_trace_enter: 373 move a0, sp 374 call do_syscall_trace_enter 375 move t0, a0 376 REG_L a0, PT_A0(sp) 377 REG_L a1, PT_A1(sp) 378 REG_L a2, PT_A2(sp) 379 REG_L a3, PT_A3(sp) 380 REG_L a4, PT_A4(sp) 381 REG_L a5, PT_A5(sp) 382 REG_L a6, PT_A6(sp) 383 REG_L a7, PT_A7(sp) 384 bnez t0, ret_from_syscall_rejected 385 j check_syscall_nr 386handle_syscall_trace_exit: 387 move a0, sp 388 call do_syscall_trace_exit 389 j ret_from_exception 390 391#ifdef CONFIG_VMAP_STACK 392handle_kernel_stack_overflow: 393 la sp, shadow_stack 394 addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE 395 396 //save caller register to shadow stack 397 addi sp, sp, -(PT_SIZE_ON_STACK) 398 REG_S x1, PT_RA(sp) 399 REG_S x5, PT_T0(sp) 400 REG_S x6, PT_T1(sp) 401 REG_S x7, PT_T2(sp) 402 REG_S x10, PT_A0(sp) 403 REG_S x11, PT_A1(sp) 404 REG_S x12, PT_A2(sp) 405 REG_S x13, PT_A3(sp) 406 REG_S x14, PT_A4(sp) 407 REG_S x15, PT_A5(sp) 408 REG_S x16, PT_A6(sp) 409 REG_S x17, PT_A7(sp) 410 REG_S x28, PT_T3(sp) 411 REG_S x29, PT_T4(sp) 412 REG_S x30, PT_T5(sp) 413 REG_S x31, PT_T6(sp) 414 415 la ra, restore_caller_reg 416 tail get_overflow_stack 417 418restore_caller_reg: 419 //save per-cpu overflow stack 420 REG_S a0, -8(sp) 421 //restore caller register from shadow_stack 422 REG_L x1, PT_RA(sp) 423 REG_L x5, PT_T0(sp) 424 REG_L x6, PT_T1(sp) 425 REG_L x7, PT_T2(sp) 426 REG_L x10, PT_A0(sp) 427 REG_L x11, PT_A1(sp) 428 REG_L x12, PT_A2(sp) 429 REG_L x13, PT_A3(sp) 430 REG_L x14, PT_A4(sp) 431 REG_L x15, PT_A5(sp) 432 REG_L x16, PT_A6(sp) 433 REG_L x17, PT_A7(sp) 434 REG_L x28, PT_T3(sp) 435 REG_L x29, PT_T4(sp) 436 REG_L x30, PT_T5(sp) 437 REG_L x31, PT_T6(sp) 438 439 //load per-cpu overflow stack 440 REG_L sp, -8(sp) 441 addi sp, sp, -(PT_SIZE_ON_STACK) 442 443 //save context to overflow stack 444 REG_S x1, PT_RA(sp) 445 REG_S x3, PT_GP(sp) 446 REG_S x5, PT_T0(sp) 447 REG_S x6, PT_T1(sp) 448 REG_S x7, PT_T2(sp) 449 REG_S x8, PT_S0(sp) 450 REG_S x9, PT_S1(sp) 451 REG_S x10, PT_A0(sp) 452 REG_S x11, PT_A1(sp) 453 REG_S x12, PT_A2(sp) 454 REG_S x13, PT_A3(sp) 455 REG_S x14, PT_A4(sp) 456 REG_S x15, PT_A5(sp) 457 REG_S x16, PT_A6(sp) 458 REG_S x17, PT_A7(sp) 459 REG_S x18, PT_S2(sp) 460 REG_S x19, PT_S3(sp) 461 REG_S x20, PT_S4(sp) 462 REG_S x21, PT_S5(sp) 463 REG_S x22, PT_S6(sp) 464 REG_S x23, PT_S7(sp) 465 REG_S x24, PT_S8(sp) 466 REG_S x25, PT_S9(sp) 467 REG_S x26, PT_S10(sp) 468 REG_S x27, PT_S11(sp) 469 REG_S x28, PT_T3(sp) 470 REG_S x29, PT_T4(sp) 471 REG_S x30, PT_T5(sp) 472 REG_S x31, PT_T6(sp) 473 474 REG_L s0, TASK_TI_KERNEL_SP(tp) 475 csrr s1, CSR_STATUS 476 csrr s2, CSR_EPC 477 csrr s3, CSR_TVAL 478 csrr s4, CSR_CAUSE 479 csrr s5, CSR_SCRATCH 480 REG_S s0, PT_SP(sp) 481 REG_S s1, PT_STATUS(sp) 482 REG_S s2, PT_EPC(sp) 483 REG_S s3, PT_BADADDR(sp) 484 REG_S s4, PT_CAUSE(sp) 485 REG_S s5, PT_TP(sp) 486 move a0, sp 487 tail handle_bad_stack 488#endif 489 490END(handle_exception) 491 492ENTRY(ret_from_fork) 493 la ra, ret_from_exception 494 tail schedule_tail 495ENDPROC(ret_from_fork) 496 497ENTRY(ret_from_kernel_thread) 498 call schedule_tail 499 /* Call fn(arg) */ 500 la ra, ret_from_exception 501 move a0, s1 502 jr s0 503ENDPROC(ret_from_kernel_thread) 504 505 506/* 507 * Integer register context switch 508 * The callee-saved registers must be saved and restored. 509 * 510 * a0: previous task_struct (must be preserved across the switch) 511 * a1: next task_struct 512 * 513 * The value of a0 and a1 must be preserved by this function, as that's how 514 * arguments are passed to schedule_tail. 515 */ 516ENTRY(__switch_to) 517 /* Save context into prev->thread */ 518 li a4, TASK_THREAD_RA 519 add a3, a0, a4 520 add a4, a1, a4 521 REG_S ra, TASK_THREAD_RA_RA(a3) 522 REG_S sp, TASK_THREAD_SP_RA(a3) 523 REG_S s0, TASK_THREAD_S0_RA(a3) 524 REG_S s1, TASK_THREAD_S1_RA(a3) 525 REG_S s2, TASK_THREAD_S2_RA(a3) 526 REG_S s3, TASK_THREAD_S3_RA(a3) 527 REG_S s4, TASK_THREAD_S4_RA(a3) 528 REG_S s5, TASK_THREAD_S5_RA(a3) 529 REG_S s6, TASK_THREAD_S6_RA(a3) 530 REG_S s7, TASK_THREAD_S7_RA(a3) 531 REG_S s8, TASK_THREAD_S8_RA(a3) 532 REG_S s9, TASK_THREAD_S9_RA(a3) 533 REG_S s10, TASK_THREAD_S10_RA(a3) 534 REG_S s11, TASK_THREAD_S11_RA(a3) 535 /* Restore context from next->thread */ 536 REG_L ra, TASK_THREAD_RA_RA(a4) 537 REG_L sp, TASK_THREAD_SP_RA(a4) 538 REG_L s0, TASK_THREAD_S0_RA(a4) 539 REG_L s1, TASK_THREAD_S1_RA(a4) 540 REG_L s2, TASK_THREAD_S2_RA(a4) 541 REG_L s3, TASK_THREAD_S3_RA(a4) 542 REG_L s4, TASK_THREAD_S4_RA(a4) 543 REG_L s5, TASK_THREAD_S5_RA(a4) 544 REG_L s6, TASK_THREAD_S6_RA(a4) 545 REG_L s7, TASK_THREAD_S7_RA(a4) 546 REG_L s8, TASK_THREAD_S8_RA(a4) 547 REG_L s9, TASK_THREAD_S9_RA(a4) 548 REG_L s10, TASK_THREAD_S10_RA(a4) 549 REG_L s11, TASK_THREAD_S11_RA(a4) 550 /* The offset of thread_info in task_struct is zero. */ 551 move tp, a1 552 ret 553ENDPROC(__switch_to) 554 555#ifndef CONFIG_MMU 556#define do_page_fault do_trap_unknown 557#endif 558 559 .section ".rodata" 560 .align LGREG 561 /* Exception vector table */ 562ENTRY(excp_vect_table) 563 RISCV_PTR do_trap_insn_misaligned 564 ALT_INSN_FAULT(RISCV_PTR do_trap_insn_fault) 565 RISCV_PTR do_trap_insn_illegal 566 RISCV_PTR do_trap_break 567 RISCV_PTR do_trap_load_misaligned 568 RISCV_PTR do_trap_load_fault 569 RISCV_PTR do_trap_store_misaligned 570 RISCV_PTR do_trap_store_fault 571 RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */ 572 RISCV_PTR do_trap_ecall_s 573 RISCV_PTR do_trap_unknown 574 RISCV_PTR do_trap_ecall_m 575 /* instruciton page fault */ 576 ALT_PAGE_FAULT(RISCV_PTR do_page_fault) 577 RISCV_PTR do_page_fault /* load page fault */ 578 RISCV_PTR do_trap_unknown 579 RISCV_PTR do_page_fault /* store page fault */ 580excp_vect_table_end: 581END(excp_vect_table) 582 583#ifndef CONFIG_MMU 584ENTRY(__user_rt_sigreturn) 585 li a7, __NR_rt_sigreturn 586 scall 587END(__user_rt_sigreturn) 588#endif 589