1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * FP/SIMD context switching and fault handling 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Author: Catalin Marinas <catalin.marinas@arm.com> 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/bitops.h> 11 #include <linux/bottom_half.h> 12 #include <linux/bug.h> 13 #include <linux/cache.h> 14 #include <linux/compat.h> 15 #include <linux/cpu.h> 16 #include <linux/cpu_pm.h> 17 #include <linux/kernel.h> 18 #include <linux/linkage.h> 19 #include <linux/irqflags.h> 20 #include <linux/init.h> 21 #include <linux/percpu.h> 22 #include <linux/prctl.h> 23 #include <linux/preempt.h> 24 #include <linux/ptrace.h> 25 #include <linux/sched/signal.h> 26 #include <linux/sched/task_stack.h> 27 #include <linux/signal.h> 28 #include <linux/slab.h> 29 #include <linux/stddef.h> 30 #include <linux/sysctl.h> 31 #include <linux/swab.h> 32 33 #include <asm/esr.h> 34 #include <asm/fpsimd.h> 35 #include <asm/cpufeature.h> 36 #include <asm/cputype.h> 37 #include <asm/processor.h> 38 #include <asm/simd.h> 39 #include <asm/sigcontext.h> 40 #include <asm/sysreg.h> 41 #include <asm/traps.h> 42 #include <asm/virt.h> 43 44 #define FPEXC_IOF (1 << 0) 45 #define FPEXC_DZF (1 << 1) 46 #define FPEXC_OFF (1 << 2) 47 #define FPEXC_UFF (1 << 3) 48 #define FPEXC_IXF (1 << 4) 49 #define FPEXC_IDF (1 << 7) 50 51 /* 52 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.) 53 * 54 * In order to reduce the number of times the FPSIMD state is needlessly saved 55 * and restored, we need to keep track of two things: 56 * (a) for each task, we need to remember which CPU was the last one to have 57 * the task's FPSIMD state loaded into its FPSIMD registers; 58 * (b) for each CPU, we need to remember which task's userland FPSIMD state has 59 * been loaded into its FPSIMD registers most recently, or whether it has 60 * been used to perform kernel mode NEON in the meantime. 61 * 62 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to 63 * the id of the current CPU every time the state is loaded onto a CPU. For (b), 64 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the 65 * address of the userland FPSIMD state of the task that was loaded onto the CPU 66 * the most recently, or NULL if kernel mode NEON has been performed after that. 67 * 68 * With this in place, we no longer have to restore the next FPSIMD state right 69 * when switching between tasks. Instead, we can defer this check to userland 70 * resume, at which time we verify whether the CPU's fpsimd_last_state and the 71 * task's fpsimd_cpu are still mutually in sync. If this is the case, we 72 * can omit the FPSIMD restore. 73 * 74 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to 75 * indicate whether or not the userland FPSIMD state of the current task is 76 * present in the registers. The flag is set unless the FPSIMD registers of this 77 * CPU currently contain the most recent userland FPSIMD state of the current 78 * task. 79 * 80 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may 81 * save the task's FPSIMD context back to task_struct from softirq context. 82 * To prevent this from racing with the manipulation of the task's FPSIMD state 83 * from task context and thereby corrupting the state, it is necessary to 84 * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE 85 * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to 86 * run but prevent them to use FPSIMD. 87 * 88 * For a certain task, the sequence may look something like this: 89 * - the task gets scheduled in; if both the task's fpsimd_cpu field 90 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu 91 * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is 92 * cleared, otherwise it is set; 93 * 94 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's 95 * userland FPSIMD state is copied from memory to the registers, the task's 96 * fpsimd_cpu field is set to the id of the current CPU, the current 97 * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the 98 * TIF_FOREIGN_FPSTATE flag is cleared; 99 * 100 * - the task executes an ordinary syscall; upon return to userland, the 101 * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is 102 * restored; 103 * 104 * - the task executes a syscall which executes some NEON instructions; this is 105 * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD 106 * register contents to memory, clears the fpsimd_last_state per-cpu variable 107 * and sets the TIF_FOREIGN_FPSTATE flag; 108 * 109 * - the task gets preempted after kernel_neon_end() is called; as we have not 110 * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so 111 * whatever is in the FPSIMD registers is not saved to memory, but discarded. 112 */ 113 struct fpsimd_last_state_struct { 114 struct user_fpsimd_state *st; 115 void *sve_state; 116 unsigned int sve_vl; 117 }; 118 119 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state); 120 121 /* Default VL for tasks that don't set it explicitly: */ 122 static int sve_default_vl = -1; 123 124 #ifdef CONFIG_ARM64_SVE 125 126 /* Maximum supported vector length across all CPUs (initially poisoned) */ 127 int __ro_after_init sve_max_vl = SVE_VL_MIN; 128 int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN; 129 130 /* 131 * Set of available vector lengths, 132 * where length vq encoded as bit __vq_to_bit(vq): 133 */ 134 __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); 135 /* Set of vector lengths present on at least one cpu: */ 136 static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); 137 138 static void __percpu *efi_sve_state; 139 140 #else /* ! CONFIG_ARM64_SVE */ 141 142 /* Dummy declaration for code that will be optimised out: */ 143 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); 144 extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); 145 extern void __percpu *efi_sve_state; 146 147 #endif /* ! CONFIG_ARM64_SVE */ 148 149 DEFINE_PER_CPU(bool, fpsimd_context_busy); 150 EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy); 151 152 static void __get_cpu_fpsimd_context(void) 153 { 154 bool busy = __this_cpu_xchg(fpsimd_context_busy, true); 155 156 WARN_ON(busy); 157 } 158 159 /* 160 * Claim ownership of the CPU FPSIMD context for use by the calling context. 161 * 162 * The caller may freely manipulate the FPSIMD context metadata until 163 * put_cpu_fpsimd_context() is called. 164 * 165 * The double-underscore version must only be called if you know the task 166 * can't be preempted. 167 */ 168 static void get_cpu_fpsimd_context(void) 169 { 170 preempt_disable(); 171 __get_cpu_fpsimd_context(); 172 } 173 174 static void __put_cpu_fpsimd_context(void) 175 { 176 bool busy = __this_cpu_xchg(fpsimd_context_busy, false); 177 178 WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */ 179 } 180 181 /* 182 * Release the CPU FPSIMD context. 183 * 184 * Must be called from a context in which get_cpu_fpsimd_context() was 185 * previously called, with no call to put_cpu_fpsimd_context() in the 186 * meantime. 187 */ 188 static void put_cpu_fpsimd_context(void) 189 { 190 __put_cpu_fpsimd_context(); 191 preempt_enable(); 192 } 193 194 static bool have_cpu_fpsimd_context(void) 195 { 196 return !preemptible() && __this_cpu_read(fpsimd_context_busy); 197 } 198 199 /* 200 * Call __sve_free() directly only if you know task can't be scheduled 201 * or preempted. 202 */ 203 static void __sve_free(struct task_struct *task) 204 { 205 kfree(task->thread.sve_state); 206 task->thread.sve_state = NULL; 207 } 208 209 static void sve_free(struct task_struct *task) 210 { 211 WARN_ON(test_tsk_thread_flag(task, TIF_SVE)); 212 213 __sve_free(task); 214 } 215 216 /* 217 * TIF_SVE controls whether a task can use SVE without trapping while 218 * in userspace, and also the way a task's FPSIMD/SVE state is stored 219 * in thread_struct. 220 * 221 * The kernel uses this flag to track whether a user task is actively 222 * using SVE, and therefore whether full SVE register state needs to 223 * be tracked. If not, the cheaper FPSIMD context handling code can 224 * be used instead of the more costly SVE equivalents. 225 * 226 * * TIF_SVE set: 227 * 228 * The task can execute SVE instructions while in userspace without 229 * trapping to the kernel. 230 * 231 * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the 232 * corresponding Zn), P0-P15 and FFR are encoded in in 233 * task->thread.sve_state, formatted appropriately for vector 234 * length task->thread.sve_vl. 235 * 236 * task->thread.sve_state must point to a valid buffer at least 237 * sve_state_size(task) bytes in size. 238 * 239 * During any syscall, the kernel may optionally clear TIF_SVE and 240 * discard the vector state except for the FPSIMD subset. 241 * 242 * * TIF_SVE clear: 243 * 244 * An attempt by the user task to execute an SVE instruction causes 245 * do_sve_acc() to be called, which does some preparation and then 246 * sets TIF_SVE. 247 * 248 * When stored, FPSIMD registers V0-V31 are encoded in 249 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are 250 * logically zero but not stored anywhere; P0-P15 and FFR are not 251 * stored and have unspecified values from userspace's point of 252 * view. For hygiene purposes, the kernel zeroes them on next use, 253 * but userspace is discouraged from relying on this. 254 * 255 * task->thread.sve_state does not need to be non-NULL, valid or any 256 * particular size: it must not be dereferenced. 257 * 258 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state 259 * irrespective of whether TIF_SVE is clear or set, since these are 260 * not vector length dependent. 261 */ 262 263 /* 264 * Update current's FPSIMD/SVE registers from thread_struct. 265 * 266 * This function should be called only when the FPSIMD/SVE state in 267 * thread_struct is known to be up to date, when preparing to enter 268 * userspace. 269 */ 270 static void task_fpsimd_load(void) 271 { 272 WARN_ON(!system_supports_fpsimd()); 273 WARN_ON(!have_cpu_fpsimd_context()); 274 275 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 276 sve_load_state(sve_pffr(¤t->thread), 277 ¤t->thread.uw.fpsimd_state.fpsr, 278 sve_vq_from_vl(current->thread.sve_vl) - 1); 279 else 280 fpsimd_load_state(¤t->thread.uw.fpsimd_state); 281 } 282 283 /* 284 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to 285 * date with respect to the CPU registers. 286 */ 287 static void fpsimd_save(void) 288 { 289 struct fpsimd_last_state_struct const *last = 290 this_cpu_ptr(&fpsimd_last_state); 291 /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */ 292 293 WARN_ON(!system_supports_fpsimd()); 294 WARN_ON(!have_cpu_fpsimd_context()); 295 296 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 297 if (system_supports_sve() && test_thread_flag(TIF_SVE)) { 298 if (WARN_ON(sve_get_vl() != last->sve_vl)) { 299 /* 300 * Can't save the user regs, so current would 301 * re-enter user with corrupt state. 302 * There's no way to recover, so kill it: 303 */ 304 force_signal_inject(SIGKILL, SI_KERNEL, 0); 305 return; 306 } 307 308 sve_save_state((char *)last->sve_state + 309 sve_ffr_offset(last->sve_vl), 310 &last->st->fpsr); 311 } else 312 fpsimd_save_state(last->st); 313 } 314 } 315 316 /* 317 * All vector length selection from userspace comes through here. 318 * We're on a slow path, so some sanity-checks are included. 319 * If things go wrong there's a bug somewhere, but try to fall back to a 320 * safe choice. 321 */ 322 static unsigned int find_supported_vector_length(unsigned int vl) 323 { 324 int bit; 325 int max_vl = sve_max_vl; 326 327 if (WARN_ON(!sve_vl_valid(vl))) 328 vl = SVE_VL_MIN; 329 330 if (WARN_ON(!sve_vl_valid(max_vl))) 331 max_vl = SVE_VL_MIN; 332 333 if (vl > max_vl) 334 vl = max_vl; 335 336 bit = find_next_bit(sve_vq_map, SVE_VQ_MAX, 337 __vq_to_bit(sve_vq_from_vl(vl))); 338 return sve_vl_from_vq(__bit_to_vq(bit)); 339 } 340 341 #ifdef CONFIG_SYSCTL 342 343 static int sve_proc_do_default_vl(struct ctl_table *table, int write, 344 void __user *buffer, size_t *lenp, 345 loff_t *ppos) 346 { 347 int ret; 348 int vl = sve_default_vl; 349 struct ctl_table tmp_table = { 350 .data = &vl, 351 .maxlen = sizeof(vl), 352 }; 353 354 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); 355 if (ret || !write) 356 return ret; 357 358 /* Writing -1 has the special meaning "set to max": */ 359 if (vl == -1) 360 vl = sve_max_vl; 361 362 if (!sve_vl_valid(vl)) 363 return -EINVAL; 364 365 sve_default_vl = find_supported_vector_length(vl); 366 return 0; 367 } 368 369 static struct ctl_table sve_default_vl_table[] = { 370 { 371 .procname = "sve_default_vector_length", 372 .mode = 0644, 373 .proc_handler = sve_proc_do_default_vl, 374 }, 375 { } 376 }; 377 378 static int __init sve_sysctl_init(void) 379 { 380 if (system_supports_sve()) 381 if (!register_sysctl("abi", sve_default_vl_table)) 382 return -EINVAL; 383 384 return 0; 385 } 386 387 #else /* ! CONFIG_SYSCTL */ 388 static int __init sve_sysctl_init(void) { return 0; } 389 #endif /* ! CONFIG_SYSCTL */ 390 391 #define ZREG(sve_state, vq, n) ((char *)(sve_state) + \ 392 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET)) 393 394 #ifdef CONFIG_CPU_BIG_ENDIAN 395 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 396 { 397 u64 a = swab64(x); 398 u64 b = swab64(x >> 64); 399 400 return ((__uint128_t)a << 64) | b; 401 } 402 #else 403 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 404 { 405 return x; 406 } 407 #endif 408 409 #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) 410 411 static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, 412 unsigned int vq) 413 { 414 unsigned int i; 415 __uint128_t *p; 416 417 for (i = 0; i < SVE_NUM_ZREGS; ++i) { 418 p = (__uint128_t *)ZREG(sst, vq, i); 419 *p = arm64_cpu_to_le128(fst->vregs[i]); 420 } 421 } 422 423 /* 424 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to 425 * task->thread.sve_state. 426 * 427 * Task can be a non-runnable task, or current. In the latter case, 428 * the caller must have ownership of the cpu FPSIMD context before calling 429 * this function. 430 * task->thread.sve_state must point to at least sve_state_size(task) 431 * bytes of allocated kernel memory. 432 * task->thread.uw.fpsimd_state must be up to date before calling this 433 * function. 434 */ 435 static void fpsimd_to_sve(struct task_struct *task) 436 { 437 unsigned int vq; 438 void *sst = task->thread.sve_state; 439 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 440 441 if (!system_supports_sve()) 442 return; 443 444 vq = sve_vq_from_vl(task->thread.sve_vl); 445 __fpsimd_to_sve(sst, fst, vq); 446 } 447 448 /* 449 * Transfer the SVE state in task->thread.sve_state to 450 * task->thread.uw.fpsimd_state. 451 * 452 * Task can be a non-runnable task, or current. In the latter case, 453 * the caller must have ownership of the cpu FPSIMD context before calling 454 * this function. 455 * task->thread.sve_state must point to at least sve_state_size(task) 456 * bytes of allocated kernel memory. 457 * task->thread.sve_state must be up to date before calling this function. 458 */ 459 static void sve_to_fpsimd(struct task_struct *task) 460 { 461 unsigned int vq; 462 void const *sst = task->thread.sve_state; 463 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; 464 unsigned int i; 465 __uint128_t const *p; 466 467 if (!system_supports_sve()) 468 return; 469 470 vq = sve_vq_from_vl(task->thread.sve_vl); 471 for (i = 0; i < SVE_NUM_ZREGS; ++i) { 472 p = (__uint128_t const *)ZREG(sst, vq, i); 473 fst->vregs[i] = arm64_le128_to_cpu(*p); 474 } 475 } 476 477 #ifdef CONFIG_ARM64_SVE 478 479 /* 480 * Return how many bytes of memory are required to store the full SVE 481 * state for task, given task's currently configured vector length. 482 */ 483 size_t sve_state_size(struct task_struct const *task) 484 { 485 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl)); 486 } 487 488 /* 489 * Ensure that task->thread.sve_state is allocated and sufficiently large. 490 * 491 * This function should be used only in preparation for replacing 492 * task->thread.sve_state with new data. The memory is always zeroed 493 * here to prevent stale data from showing through: this is done in 494 * the interest of testability and predictability: except in the 495 * do_sve_acc() case, there is no ABI requirement to hide stale data 496 * written previously be task. 497 */ 498 void sve_alloc(struct task_struct *task) 499 { 500 if (task->thread.sve_state) { 501 memset(task->thread.sve_state, 0, sve_state_size(current)); 502 return; 503 } 504 505 /* This is a small allocation (maximum ~8KB) and Should Not Fail. */ 506 task->thread.sve_state = 507 kzalloc(sve_state_size(task), GFP_KERNEL); 508 509 /* 510 * If future SVE revisions can have larger vectors though, 511 * this may cease to be true: 512 */ 513 BUG_ON(!task->thread.sve_state); 514 } 515 516 517 /* 518 * Ensure that task->thread.sve_state is up to date with respect to 519 * the user task, irrespective of when SVE is in use or not. 520 * 521 * This should only be called by ptrace. task must be non-runnable. 522 * task->thread.sve_state must point to at least sve_state_size(task) 523 * bytes of allocated kernel memory. 524 */ 525 void fpsimd_sync_to_sve(struct task_struct *task) 526 { 527 if (!test_tsk_thread_flag(task, TIF_SVE)) 528 fpsimd_to_sve(task); 529 } 530 531 /* 532 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to 533 * the user task, irrespective of whether SVE is in use or not. 534 * 535 * This should only be called by ptrace. task must be non-runnable. 536 * task->thread.sve_state must point to at least sve_state_size(task) 537 * bytes of allocated kernel memory. 538 */ 539 void sve_sync_to_fpsimd(struct task_struct *task) 540 { 541 if (test_tsk_thread_flag(task, TIF_SVE)) 542 sve_to_fpsimd(task); 543 } 544 545 /* 546 * Ensure that task->thread.sve_state is up to date with respect to 547 * the task->thread.uw.fpsimd_state. 548 * 549 * This should only be called by ptrace to merge new FPSIMD register 550 * values into a task for which SVE is currently active. 551 * task must be non-runnable. 552 * task->thread.sve_state must point to at least sve_state_size(task) 553 * bytes of allocated kernel memory. 554 * task->thread.uw.fpsimd_state must already have been initialised with 555 * the new FPSIMD register values to be merged in. 556 */ 557 void sve_sync_from_fpsimd_zeropad(struct task_struct *task) 558 { 559 unsigned int vq; 560 void *sst = task->thread.sve_state; 561 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 562 563 if (!test_tsk_thread_flag(task, TIF_SVE)) 564 return; 565 566 vq = sve_vq_from_vl(task->thread.sve_vl); 567 568 memset(sst, 0, SVE_SIG_REGS_SIZE(vq)); 569 __fpsimd_to_sve(sst, fst, vq); 570 } 571 572 int sve_set_vector_length(struct task_struct *task, 573 unsigned long vl, unsigned long flags) 574 { 575 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT | 576 PR_SVE_SET_VL_ONEXEC)) 577 return -EINVAL; 578 579 if (!sve_vl_valid(vl)) 580 return -EINVAL; 581 582 /* 583 * Clamp to the maximum vector length that VL-agnostic SVE code can 584 * work with. A flag may be assigned in the future to allow setting 585 * of larger vector lengths without confusing older software. 586 */ 587 if (vl > SVE_VL_ARCH_MAX) 588 vl = SVE_VL_ARCH_MAX; 589 590 vl = find_supported_vector_length(vl); 591 592 if (flags & (PR_SVE_VL_INHERIT | 593 PR_SVE_SET_VL_ONEXEC)) 594 task->thread.sve_vl_onexec = vl; 595 else 596 /* Reset VL to system default on next exec: */ 597 task->thread.sve_vl_onexec = 0; 598 599 /* Only actually set the VL if not deferred: */ 600 if (flags & PR_SVE_SET_VL_ONEXEC) 601 goto out; 602 603 if (vl == task->thread.sve_vl) 604 goto out; 605 606 /* 607 * To ensure the FPSIMD bits of the SVE vector registers are preserved, 608 * write any live register state back to task_struct, and convert to a 609 * non-SVE thread. 610 */ 611 if (task == current) { 612 get_cpu_fpsimd_context(); 613 614 fpsimd_save(); 615 } 616 617 fpsimd_flush_task_state(task); 618 if (test_and_clear_tsk_thread_flag(task, TIF_SVE)) 619 sve_to_fpsimd(task); 620 621 if (task == current) 622 put_cpu_fpsimd_context(); 623 624 /* 625 * Force reallocation of task SVE state to the correct size 626 * on next use: 627 */ 628 sve_free(task); 629 630 task->thread.sve_vl = vl; 631 632 out: 633 update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT, 634 flags & PR_SVE_VL_INHERIT); 635 636 return 0; 637 } 638 639 /* 640 * Encode the current vector length and flags for return. 641 * This is only required for prctl(): ptrace has separate fields 642 * 643 * flags are as for sve_set_vector_length(). 644 */ 645 static int sve_prctl_status(unsigned long flags) 646 { 647 int ret; 648 649 if (flags & PR_SVE_SET_VL_ONEXEC) 650 ret = current->thread.sve_vl_onexec; 651 else 652 ret = current->thread.sve_vl; 653 654 if (test_thread_flag(TIF_SVE_VL_INHERIT)) 655 ret |= PR_SVE_VL_INHERIT; 656 657 return ret; 658 } 659 660 /* PR_SVE_SET_VL */ 661 int sve_set_current_vl(unsigned long arg) 662 { 663 unsigned long vl, flags; 664 int ret; 665 666 vl = arg & PR_SVE_VL_LEN_MASK; 667 flags = arg & ~vl; 668 669 if (!system_supports_sve()) 670 return -EINVAL; 671 672 ret = sve_set_vector_length(current, vl, flags); 673 if (ret) 674 return ret; 675 676 return sve_prctl_status(flags); 677 } 678 679 /* PR_SVE_GET_VL */ 680 int sve_get_current_vl(void) 681 { 682 if (!system_supports_sve()) 683 return -EINVAL; 684 685 return sve_prctl_status(0); 686 } 687 688 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX)) 689 { 690 unsigned int vq, vl; 691 unsigned long zcr; 692 693 bitmap_zero(map, SVE_VQ_MAX); 694 695 zcr = ZCR_ELx_LEN_MASK; 696 zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr; 697 698 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { 699 write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */ 700 vl = sve_get_vl(); 701 vq = sve_vq_from_vl(vl); /* skip intervening lengths */ 702 set_bit(__vq_to_bit(vq), map); 703 } 704 } 705 706 /* 707 * Initialise the set of known supported VQs for the boot CPU. 708 * This is called during kernel boot, before secondary CPUs are brought up. 709 */ 710 void __init sve_init_vq_map(void) 711 { 712 sve_probe_vqs(sve_vq_map); 713 bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX); 714 } 715 716 /* 717 * If we haven't committed to the set of supported VQs yet, filter out 718 * those not supported by the current CPU. 719 * This function is called during the bring-up of early secondary CPUs only. 720 */ 721 void sve_update_vq_map(void) 722 { 723 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 724 725 sve_probe_vqs(tmp_map); 726 bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX); 727 bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX); 728 } 729 730 /* 731 * Check whether the current CPU supports all VQs in the committed set. 732 * This function is called during the bring-up of late secondary CPUs only. 733 */ 734 int sve_verify_vq_map(void) 735 { 736 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 737 unsigned long b; 738 739 sve_probe_vqs(tmp_map); 740 741 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 742 if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) { 743 pr_warn("SVE: cpu%d: Required vector length(s) missing\n", 744 smp_processor_id()); 745 return -EINVAL; 746 } 747 748 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) 749 return 0; 750 751 /* 752 * For KVM, it is necessary to ensure that this CPU doesn't 753 * support any vector length that guests may have probed as 754 * unsupported. 755 */ 756 757 /* Recover the set of supported VQs: */ 758 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 759 /* Find VQs supported that are not globally supported: */ 760 bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX); 761 762 /* Find the lowest such VQ, if any: */ 763 b = find_last_bit(tmp_map, SVE_VQ_MAX); 764 if (b >= SVE_VQ_MAX) 765 return 0; /* no mismatches */ 766 767 /* 768 * Mismatches above sve_max_virtualisable_vl are fine, since 769 * no guest is allowed to configure ZCR_EL2.LEN to exceed this: 770 */ 771 if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) { 772 pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n", 773 smp_processor_id()); 774 return -EINVAL; 775 } 776 777 return 0; 778 } 779 780 static void __init sve_efi_setup(void) 781 { 782 if (!IS_ENABLED(CONFIG_EFI)) 783 return; 784 785 /* 786 * alloc_percpu() warns and prints a backtrace if this goes wrong. 787 * This is evidence of a crippled system and we are returning void, 788 * so no attempt is made to handle this situation here. 789 */ 790 if (!sve_vl_valid(sve_max_vl)) 791 goto fail; 792 793 efi_sve_state = __alloc_percpu( 794 SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES); 795 if (!efi_sve_state) 796 goto fail; 797 798 return; 799 800 fail: 801 panic("Cannot allocate percpu memory for EFI SVE save/restore"); 802 } 803 804 /* 805 * Enable SVE for EL1. 806 * Intended for use by the cpufeatures code during CPU boot. 807 */ 808 void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) 809 { 810 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1); 811 isb(); 812 } 813 814 /* 815 * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE 816 * vector length. 817 * 818 * Use only if SVE is present. 819 * This function clobbers the SVE vector length. 820 */ 821 u64 read_zcr_features(void) 822 { 823 u64 zcr; 824 unsigned int vq_max; 825 826 /* 827 * Set the maximum possible VL, and write zeroes to all other 828 * bits to see if they stick. 829 */ 830 sve_kernel_enable(NULL); 831 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1); 832 833 zcr = read_sysreg_s(SYS_ZCR_EL1); 834 zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */ 835 vq_max = sve_vq_from_vl(sve_get_vl()); 836 zcr |= vq_max - 1; /* set LEN field to maximum effective value */ 837 838 return zcr; 839 } 840 841 void __init sve_setup(void) 842 { 843 u64 zcr; 844 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 845 unsigned long b; 846 847 if (!system_supports_sve()) 848 return; 849 850 /* 851 * The SVE architecture mandates support for 128-bit vectors, 852 * so sve_vq_map must have at least SVE_VQ_MIN set. 853 * If something went wrong, at least try to patch it up: 854 */ 855 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map))) 856 set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map); 857 858 zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); 859 sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1); 860 861 /* 862 * Sanity-check that the max VL we determined through CPU features 863 * corresponds properly to sve_vq_map. If not, do our best: 864 */ 865 if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl))) 866 sve_max_vl = find_supported_vector_length(sve_max_vl); 867 868 /* 869 * For the default VL, pick the maximum supported value <= 64. 870 * VL == 64 is guaranteed not to grow the signal frame. 871 */ 872 sve_default_vl = find_supported_vector_length(64); 873 874 bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map, 875 SVE_VQ_MAX); 876 877 b = find_last_bit(tmp_map, SVE_VQ_MAX); 878 if (b >= SVE_VQ_MAX) 879 /* No non-virtualisable VLs found */ 880 sve_max_virtualisable_vl = SVE_VQ_MAX; 881 else if (WARN_ON(b == SVE_VQ_MAX - 1)) 882 /* No virtualisable VLs? This is architecturally forbidden. */ 883 sve_max_virtualisable_vl = SVE_VQ_MIN; 884 else /* b + 1 < SVE_VQ_MAX */ 885 sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); 886 887 if (sve_max_virtualisable_vl > sve_max_vl) 888 sve_max_virtualisable_vl = sve_max_vl; 889 890 pr_info("SVE: maximum available vector length %u bytes per vector\n", 891 sve_max_vl); 892 pr_info("SVE: default vector length %u bytes per vector\n", 893 sve_default_vl); 894 895 /* KVM decides whether to support mismatched systems. Just warn here: */ 896 if (sve_max_virtualisable_vl < sve_max_vl) 897 pr_warn("SVE: unvirtualisable vector lengths present\n"); 898 899 sve_efi_setup(); 900 } 901 902 /* 903 * Called from the put_task_struct() path, which cannot get here 904 * unless dead_task is really dead and not schedulable. 905 */ 906 void fpsimd_release_task(struct task_struct *dead_task) 907 { 908 __sve_free(dead_task); 909 } 910 911 #endif /* CONFIG_ARM64_SVE */ 912 913 /* 914 * Trapped SVE access 915 * 916 * Storage is allocated for the full SVE state, the current FPSIMD 917 * register contents are migrated across, and TIF_SVE is set so that 918 * the SVE access trap will be disabled the next time this task 919 * reaches ret_to_user. 920 * 921 * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load() 922 * would have disabled the SVE access trap for userspace during 923 * ret_to_user, making an SVE access trap impossible in that case. 924 */ 925 void do_sve_acc(unsigned int esr, struct pt_regs *regs) 926 { 927 /* Even if we chose not to use SVE, the hardware could still trap: */ 928 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) { 929 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 930 return; 931 } 932 933 sve_alloc(current); 934 935 get_cpu_fpsimd_context(); 936 937 fpsimd_save(); 938 939 /* Force ret_to_user to reload the registers: */ 940 fpsimd_flush_task_state(current); 941 942 fpsimd_to_sve(current); 943 if (test_and_set_thread_flag(TIF_SVE)) 944 WARN_ON(1); /* SVE access shouldn't have trapped */ 945 946 put_cpu_fpsimd_context(); 947 } 948 949 /* 950 * Trapped FP/ASIMD access. 951 */ 952 void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) 953 { 954 /* TODO: implement lazy context saving/restoring */ 955 WARN_ON(1); 956 } 957 958 /* 959 * Raise a SIGFPE for the current process. 960 */ 961 void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) 962 { 963 unsigned int si_code = FPE_FLTUNK; 964 965 if (esr & ESR_ELx_FP_EXC_TFV) { 966 if (esr & FPEXC_IOF) 967 si_code = FPE_FLTINV; 968 else if (esr & FPEXC_DZF) 969 si_code = FPE_FLTDIV; 970 else if (esr & FPEXC_OFF) 971 si_code = FPE_FLTOVF; 972 else if (esr & FPEXC_UFF) 973 si_code = FPE_FLTUND; 974 else if (esr & FPEXC_IXF) 975 si_code = FPE_FLTRES; 976 } 977 978 send_sig_fault(SIGFPE, si_code, 979 (void __user *)instruction_pointer(regs), 980 current); 981 } 982 983 void fpsimd_thread_switch(struct task_struct *next) 984 { 985 bool wrong_task, wrong_cpu; 986 987 if (!system_supports_fpsimd()) 988 return; 989 990 __get_cpu_fpsimd_context(); 991 992 /* Save unsaved fpsimd state, if any: */ 993 fpsimd_save(); 994 995 /* 996 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's 997 * state. For kernel threads, FPSIMD registers are never loaded 998 * and wrong_task and wrong_cpu will always be true. 999 */ 1000 wrong_task = __this_cpu_read(fpsimd_last_state.st) != 1001 &next->thread.uw.fpsimd_state; 1002 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); 1003 1004 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 1005 wrong_task || wrong_cpu); 1006 1007 __put_cpu_fpsimd_context(); 1008 } 1009 1010 void fpsimd_flush_thread(void) 1011 { 1012 int vl, supported_vl; 1013 1014 if (!system_supports_fpsimd()) 1015 return; 1016 1017 get_cpu_fpsimd_context(); 1018 1019 fpsimd_flush_task_state(current); 1020 memset(¤t->thread.uw.fpsimd_state, 0, 1021 sizeof(current->thread.uw.fpsimd_state)); 1022 1023 if (system_supports_sve()) { 1024 clear_thread_flag(TIF_SVE); 1025 sve_free(current); 1026 1027 /* 1028 * Reset the task vector length as required. 1029 * This is where we ensure that all user tasks have a valid 1030 * vector length configured: no kernel task can become a user 1031 * task without an exec and hence a call to this function. 1032 * By the time the first call to this function is made, all 1033 * early hardware probing is complete, so sve_default_vl 1034 * should be valid. 1035 * If a bug causes this to go wrong, we make some noise and 1036 * try to fudge thread.sve_vl to a safe value here. 1037 */ 1038 vl = current->thread.sve_vl_onexec ? 1039 current->thread.sve_vl_onexec : sve_default_vl; 1040 1041 if (WARN_ON(!sve_vl_valid(vl))) 1042 vl = SVE_VL_MIN; 1043 1044 supported_vl = find_supported_vector_length(vl); 1045 if (WARN_ON(supported_vl != vl)) 1046 vl = supported_vl; 1047 1048 current->thread.sve_vl = vl; 1049 1050 /* 1051 * If the task is not set to inherit, ensure that the vector 1052 * length will be reset by a subsequent exec: 1053 */ 1054 if (!test_thread_flag(TIF_SVE_VL_INHERIT)) 1055 current->thread.sve_vl_onexec = 0; 1056 } 1057 1058 put_cpu_fpsimd_context(); 1059 } 1060 1061 /* 1062 * Save the userland FPSIMD state of 'current' to memory, but only if the state 1063 * currently held in the registers does in fact belong to 'current' 1064 */ 1065 void fpsimd_preserve_current_state(void) 1066 { 1067 if (!system_supports_fpsimd()) 1068 return; 1069 1070 get_cpu_fpsimd_context(); 1071 fpsimd_save(); 1072 put_cpu_fpsimd_context(); 1073 } 1074 1075 /* 1076 * Like fpsimd_preserve_current_state(), but ensure that 1077 * current->thread.uw.fpsimd_state is updated so that it can be copied to 1078 * the signal frame. 1079 */ 1080 void fpsimd_signal_preserve_current_state(void) 1081 { 1082 fpsimd_preserve_current_state(); 1083 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 1084 sve_to_fpsimd(current); 1085 } 1086 1087 /* 1088 * Associate current's FPSIMD context with this cpu 1089 * The caller must have ownership of the cpu FPSIMD context before calling 1090 * this function. 1091 */ 1092 void fpsimd_bind_task_to_cpu(void) 1093 { 1094 struct fpsimd_last_state_struct *last = 1095 this_cpu_ptr(&fpsimd_last_state); 1096 1097 WARN_ON(!system_supports_fpsimd()); 1098 last->st = ¤t->thread.uw.fpsimd_state; 1099 last->sve_state = current->thread.sve_state; 1100 last->sve_vl = current->thread.sve_vl; 1101 current->thread.fpsimd_cpu = smp_processor_id(); 1102 1103 if (system_supports_sve()) { 1104 /* Toggle SVE trapping for userspace if needed */ 1105 if (test_thread_flag(TIF_SVE)) 1106 sve_user_enable(); 1107 else 1108 sve_user_disable(); 1109 1110 /* Serialised by exception return to user */ 1111 } 1112 } 1113 1114 void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state, 1115 unsigned int sve_vl) 1116 { 1117 struct fpsimd_last_state_struct *last = 1118 this_cpu_ptr(&fpsimd_last_state); 1119 1120 WARN_ON(!system_supports_fpsimd()); 1121 WARN_ON(!in_softirq() && !irqs_disabled()); 1122 1123 last->st = st; 1124 last->sve_state = sve_state; 1125 last->sve_vl = sve_vl; 1126 } 1127 1128 /* 1129 * Load the userland FPSIMD state of 'current' from memory, but only if the 1130 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD 1131 * state of 'current' 1132 */ 1133 void fpsimd_restore_current_state(void) 1134 { 1135 /* 1136 * For the tasks that were created before we detected the absence of 1137 * FP/SIMD, the TIF_FOREIGN_FPSTATE could be set via fpsimd_thread_switch(), 1138 * e.g, init. This could be then inherited by the children processes. 1139 * If we later detect that the system doesn't support FP/SIMD, 1140 * we must clear the flag for all the tasks to indicate that the 1141 * FPSTATE is clean (as we can't have one) to avoid looping for ever in 1142 * do_notify_resume(). 1143 */ 1144 if (!system_supports_fpsimd()) { 1145 clear_thread_flag(TIF_FOREIGN_FPSTATE); 1146 return; 1147 } 1148 1149 get_cpu_fpsimd_context(); 1150 1151 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { 1152 task_fpsimd_load(); 1153 fpsimd_bind_task_to_cpu(); 1154 } 1155 1156 put_cpu_fpsimd_context(); 1157 } 1158 1159 /* 1160 * Load an updated userland FPSIMD state for 'current' from memory and set the 1161 * flag that indicates that the FPSIMD register contents are the most recent 1162 * FPSIMD state of 'current' 1163 */ 1164 void fpsimd_update_current_state(struct user_fpsimd_state const *state) 1165 { 1166 if (WARN_ON(!system_supports_fpsimd())) 1167 return; 1168 1169 get_cpu_fpsimd_context(); 1170 1171 current->thread.uw.fpsimd_state = *state; 1172 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 1173 fpsimd_to_sve(current); 1174 1175 task_fpsimd_load(); 1176 fpsimd_bind_task_to_cpu(); 1177 1178 clear_thread_flag(TIF_FOREIGN_FPSTATE); 1179 1180 put_cpu_fpsimd_context(); 1181 } 1182 1183 /* 1184 * Invalidate live CPU copies of task t's FPSIMD state 1185 * 1186 * This function may be called with preemption enabled. The barrier() 1187 * ensures that the assignment to fpsimd_cpu is visible to any 1188 * preemption/softirq that could race with set_tsk_thread_flag(), so 1189 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared. 1190 * 1191 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any 1192 * subsequent code. 1193 */ 1194 void fpsimd_flush_task_state(struct task_struct *t) 1195 { 1196 t->thread.fpsimd_cpu = NR_CPUS; 1197 /* 1198 * If we don't support fpsimd, bail out after we have 1199 * reset the fpsimd_cpu for this task and clear the 1200 * FPSTATE. 1201 */ 1202 if (!system_supports_fpsimd()) 1203 return; 1204 barrier(); 1205 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE); 1206 1207 barrier(); 1208 } 1209 1210 /* 1211 * Invalidate any task's FPSIMD state that is present on this cpu. 1212 * The FPSIMD context should be acquired with get_cpu_fpsimd_context() 1213 * before calling this function. 1214 */ 1215 static void fpsimd_flush_cpu_state(void) 1216 { 1217 WARN_ON(!system_supports_fpsimd()); 1218 __this_cpu_write(fpsimd_last_state.st, NULL); 1219 set_thread_flag(TIF_FOREIGN_FPSTATE); 1220 } 1221 1222 /* 1223 * Save the FPSIMD state to memory and invalidate cpu view. 1224 * This function must be called with preemption disabled. 1225 */ 1226 void fpsimd_save_and_flush_cpu_state(void) 1227 { 1228 if (!system_supports_fpsimd()) 1229 return; 1230 WARN_ON(preemptible()); 1231 __get_cpu_fpsimd_context(); 1232 fpsimd_save(); 1233 fpsimd_flush_cpu_state(); 1234 __put_cpu_fpsimd_context(); 1235 } 1236 1237 #ifdef CONFIG_KERNEL_MODE_NEON 1238 1239 /* 1240 * Kernel-side NEON support functions 1241 */ 1242 1243 /* 1244 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling 1245 * context 1246 * 1247 * Must not be called unless may_use_simd() returns true. 1248 * Task context in the FPSIMD registers is saved back to memory as necessary. 1249 * 1250 * A matching call to kernel_neon_end() must be made before returning from the 1251 * calling context. 1252 * 1253 * The caller may freely use the FPSIMD registers until kernel_neon_end() is 1254 * called. 1255 */ 1256 void kernel_neon_begin(void) 1257 { 1258 if (WARN_ON(!system_supports_fpsimd())) 1259 return; 1260 1261 BUG_ON(!may_use_simd()); 1262 1263 get_cpu_fpsimd_context(); 1264 1265 /* Save unsaved fpsimd state, if any: */ 1266 fpsimd_save(); 1267 1268 /* Invalidate any task state remaining in the fpsimd regs: */ 1269 fpsimd_flush_cpu_state(); 1270 } 1271 EXPORT_SYMBOL(kernel_neon_begin); 1272 1273 /* 1274 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task 1275 * 1276 * Must be called from a context in which kernel_neon_begin() was previously 1277 * called, with no call to kernel_neon_end() in the meantime. 1278 * 1279 * The caller must not use the FPSIMD registers after this function is called, 1280 * unless kernel_neon_begin() is called again in the meantime. 1281 */ 1282 void kernel_neon_end(void) 1283 { 1284 if (!system_supports_fpsimd()) 1285 return; 1286 1287 put_cpu_fpsimd_context(); 1288 } 1289 EXPORT_SYMBOL(kernel_neon_end); 1290 1291 #ifdef CONFIG_EFI 1292 1293 static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state); 1294 static DEFINE_PER_CPU(bool, efi_fpsimd_state_used); 1295 static DEFINE_PER_CPU(bool, efi_sve_state_used); 1296 1297 /* 1298 * EFI runtime services support functions 1299 * 1300 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. 1301 * This means that for EFI (and only for EFI), we have to assume that FPSIMD 1302 * is always used rather than being an optional accelerator. 1303 * 1304 * These functions provide the necessary support for ensuring FPSIMD 1305 * save/restore in the contexts from which EFI is used. 1306 * 1307 * Do not use them for any other purpose -- if tempted to do so, you are 1308 * either doing something wrong or you need to propose some refactoring. 1309 */ 1310 1311 /* 1312 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call 1313 */ 1314 void __efi_fpsimd_begin(void) 1315 { 1316 if (!system_supports_fpsimd()) 1317 return; 1318 1319 WARN_ON(preemptible()); 1320 1321 if (may_use_simd()) { 1322 kernel_neon_begin(); 1323 } else { 1324 /* 1325 * If !efi_sve_state, SVE can't be in use yet and doesn't need 1326 * preserving: 1327 */ 1328 if (system_supports_sve() && likely(efi_sve_state)) { 1329 char *sve_state = this_cpu_ptr(efi_sve_state); 1330 1331 __this_cpu_write(efi_sve_state_used, true); 1332 1333 sve_save_state(sve_state + sve_ffr_offset(sve_max_vl), 1334 &this_cpu_ptr(&efi_fpsimd_state)->fpsr); 1335 } else { 1336 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); 1337 } 1338 1339 __this_cpu_write(efi_fpsimd_state_used, true); 1340 } 1341 } 1342 1343 /* 1344 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call 1345 */ 1346 void __efi_fpsimd_end(void) 1347 { 1348 if (!system_supports_fpsimd()) 1349 return; 1350 1351 if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) { 1352 kernel_neon_end(); 1353 } else { 1354 if (system_supports_sve() && 1355 likely(__this_cpu_read(efi_sve_state_used))) { 1356 char const *sve_state = this_cpu_ptr(efi_sve_state); 1357 1358 sve_load_state(sve_state + sve_ffr_offset(sve_max_vl), 1359 &this_cpu_ptr(&efi_fpsimd_state)->fpsr, 1360 sve_vq_from_vl(sve_get_vl()) - 1); 1361 1362 __this_cpu_write(efi_sve_state_used, false); 1363 } else { 1364 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); 1365 } 1366 } 1367 } 1368 1369 #endif /* CONFIG_EFI */ 1370 1371 #endif /* CONFIG_KERNEL_MODE_NEON */ 1372 1373 #ifdef CONFIG_CPU_PM 1374 static int fpsimd_cpu_pm_notifier(struct notifier_block *self, 1375 unsigned long cmd, void *v) 1376 { 1377 switch (cmd) { 1378 case CPU_PM_ENTER: 1379 fpsimd_save_and_flush_cpu_state(); 1380 break; 1381 case CPU_PM_EXIT: 1382 break; 1383 case CPU_PM_ENTER_FAILED: 1384 default: 1385 return NOTIFY_DONE; 1386 } 1387 return NOTIFY_OK; 1388 } 1389 1390 static struct notifier_block fpsimd_cpu_pm_notifier_block = { 1391 .notifier_call = fpsimd_cpu_pm_notifier, 1392 }; 1393 1394 static void __init fpsimd_pm_init(void) 1395 { 1396 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); 1397 } 1398 1399 #else 1400 static inline void fpsimd_pm_init(void) { } 1401 #endif /* CONFIG_CPU_PM */ 1402 1403 #ifdef CONFIG_HOTPLUG_CPU 1404 static int fpsimd_cpu_dead(unsigned int cpu) 1405 { 1406 per_cpu(fpsimd_last_state.st, cpu) = NULL; 1407 return 0; 1408 } 1409 1410 static inline void fpsimd_hotplug_init(void) 1411 { 1412 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead", 1413 NULL, fpsimd_cpu_dead); 1414 } 1415 1416 #else 1417 static inline void fpsimd_hotplug_init(void) { } 1418 #endif 1419 1420 /* 1421 * FP/SIMD support code initialisation. 1422 */ 1423 static int __init fpsimd_init(void) 1424 { 1425 if (cpu_have_named_feature(FP)) { 1426 fpsimd_pm_init(); 1427 fpsimd_hotplug_init(); 1428 } else { 1429 pr_notice("Floating-point is not implemented\n"); 1430 } 1431 1432 if (!cpu_have_named_feature(ASIMD)) 1433 pr_notice("Advanced SIMD is not implemented\n"); 1434 1435 return sve_sysctl_init(); 1436 } 1437 core_initcall(fpsimd_init); 1438