1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 #include <linux/nospec.h> 16 #include <linux/prctl.h> 17 #include <linux/sched/smt.h> 18 19 #include <asm/spec-ctrl.h> 20 #include <asm/cmdline.h> 21 #include <asm/bugs.h> 22 #include <asm/processor.h> 23 #include <asm/processor-flags.h> 24 #include <asm/fpu/internal.h> 25 #include <asm/msr.h> 26 #include <asm/vmx.h> 27 #include <asm/paravirt.h> 28 #include <asm/alternative.h> 29 #include <asm/pgtable.h> 30 #include <asm/set_memory.h> 31 #include <asm/intel-family.h> 32 #include <asm/e820/api.h> 33 #include <asm/hypervisor.h> 34 35 #include "cpu.h" 36 37 static void __init spectre_v1_select_mitigation(void); 38 static void __init spectre_v2_select_mitigation(void); 39 static void __init ssb_select_mitigation(void); 40 static void __init l1tf_select_mitigation(void); 41 static void __init mds_select_mitigation(void); 42 static void __init mds_print_mitigation(void); 43 static void __init taa_select_mitigation(void); 44 45 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */ 46 u64 x86_spec_ctrl_base; 47 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 48 static DEFINE_MUTEX(spec_ctrl_mutex); 49 50 /* 51 * The vendor and possibly platform specific bits which can be modified in 52 * x86_spec_ctrl_base. 53 */ 54 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS; 55 56 /* 57 * AMD specific MSR info for Speculative Store Bypass control. 58 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 59 */ 60 u64 __ro_after_init x86_amd_ls_cfg_base; 61 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 62 63 /* Control conditional STIBP in switch_to() */ 64 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 65 /* Control conditional IBPB in switch_mm() */ 66 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 67 /* Control unconditional IBPB in switch_mm() */ 68 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 69 70 /* Control MDS CPU buffer clear before returning to user space */ 71 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 72 EXPORT_SYMBOL_GPL(mds_user_clear); 73 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 74 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 75 EXPORT_SYMBOL_GPL(mds_idle_clear); 76 77 void __init check_bugs(void) 78 { 79 identify_boot_cpu(); 80 81 /* 82 * identify_boot_cpu() initialized SMT support information, let the 83 * core code know. 84 */ 85 cpu_smt_check_topology(); 86 87 if (!IS_ENABLED(CONFIG_SMP)) { 88 pr_info("CPU: "); 89 print_cpu_info(&boot_cpu_data); 90 } 91 92 /* 93 * Read the SPEC_CTRL MSR to account for reserved bits which may 94 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 95 * init code as it is not enumerated and depends on the family. 96 */ 97 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 98 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 99 100 /* Allow STIBP in MSR_SPEC_CTRL if supported */ 101 if (boot_cpu_has(X86_FEATURE_STIBP)) 102 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP; 103 104 /* Select the proper CPU mitigations before patching alternatives: */ 105 spectre_v1_select_mitigation(); 106 spectre_v2_select_mitigation(); 107 ssb_select_mitigation(); 108 l1tf_select_mitigation(); 109 mds_select_mitigation(); 110 taa_select_mitigation(); 111 112 /* 113 * As MDS and TAA mitigations are inter-related, print MDS 114 * mitigation until after TAA mitigation selection is done. 115 */ 116 mds_print_mitigation(); 117 118 arch_smt_update(); 119 120 #ifdef CONFIG_X86_32 121 /* 122 * Check whether we are able to run this kernel safely on SMP. 123 * 124 * - i386 is no longer supported. 125 * - In order to run on anything without a TSC, we need to be 126 * compiled for a i486. 127 */ 128 if (boot_cpu_data.x86 < 4) 129 panic("Kernel requires i486+ for 'invlpg' and other features"); 130 131 init_utsname()->machine[1] = 132 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 133 alternative_instructions(); 134 135 fpu__init_check_bugs(); 136 #else /* CONFIG_X86_64 */ 137 alternative_instructions(); 138 139 /* 140 * Make sure the first 2MB area is not mapped by huge pages 141 * There are typically fixed size MTRRs in there and overlapping 142 * MTRRs into large pages causes slow downs. 143 * 144 * Right now we don't do that with gbpages because there seems 145 * very little benefit for that case. 146 */ 147 if (!direct_gbpages) 148 set_memory_4k((unsigned long)__va(0), 1); 149 #endif 150 } 151 152 void 153 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) 154 { 155 u64 msrval, guestval, hostval = x86_spec_ctrl_base; 156 struct thread_info *ti = current_thread_info(); 157 158 /* Is MSR_SPEC_CTRL implemented ? */ 159 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) { 160 /* 161 * Restrict guest_spec_ctrl to supported values. Clear the 162 * modifiable bits in the host base value and or the 163 * modifiable bits from the guest value. 164 */ 165 guestval = hostval & ~x86_spec_ctrl_mask; 166 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask; 167 168 /* SSBD controlled in MSR_SPEC_CTRL */ 169 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 170 static_cpu_has(X86_FEATURE_AMD_SSBD)) 171 hostval |= ssbd_tif_to_spec_ctrl(ti->flags); 172 173 /* Conditional STIBP enabled? */ 174 if (static_branch_unlikely(&switch_to_cond_stibp)) 175 hostval |= stibp_tif_to_spec_ctrl(ti->flags); 176 177 if (hostval != guestval) { 178 msrval = setguest ? guestval : hostval; 179 wrmsrl(MSR_IA32_SPEC_CTRL, msrval); 180 } 181 } 182 183 /* 184 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 185 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 186 */ 187 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 188 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 189 return; 190 191 /* 192 * If the host has SSBD mitigation enabled, force it in the host's 193 * virtual MSR value. If its not permanently enabled, evaluate 194 * current's TIF_SSBD thread flag. 195 */ 196 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 197 hostval = SPEC_CTRL_SSBD; 198 else 199 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 200 201 /* Sanitize the guest value */ 202 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 203 204 if (hostval != guestval) { 205 unsigned long tif; 206 207 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 208 ssbd_spec_ctrl_to_tif(hostval); 209 210 speculation_ctrl_update(tif); 211 } 212 } 213 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 214 215 static void x86_amd_ssb_disable(void) 216 { 217 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 218 219 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 220 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 221 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 222 wrmsrl(MSR_AMD64_LS_CFG, msrval); 223 } 224 225 #undef pr_fmt 226 #define pr_fmt(fmt) "MDS: " fmt 227 228 /* Default mitigation for MDS-affected CPUs */ 229 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 230 static bool mds_nosmt __ro_after_init = false; 231 232 static const char * const mds_strings[] = { 233 [MDS_MITIGATION_OFF] = "Vulnerable", 234 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 235 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 236 }; 237 238 static void __init mds_select_mitigation(void) 239 { 240 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 241 mds_mitigation = MDS_MITIGATION_OFF; 242 return; 243 } 244 245 if (mds_mitigation == MDS_MITIGATION_FULL) { 246 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 247 mds_mitigation = MDS_MITIGATION_VMWERV; 248 249 static_branch_enable(&mds_user_clear); 250 251 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 252 (mds_nosmt || cpu_mitigations_auto_nosmt())) 253 cpu_smt_disable(false); 254 } 255 } 256 257 static void __init mds_print_mitigation(void) 258 { 259 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) 260 return; 261 262 pr_info("%s\n", mds_strings[mds_mitigation]); 263 } 264 265 static int __init mds_cmdline(char *str) 266 { 267 if (!boot_cpu_has_bug(X86_BUG_MDS)) 268 return 0; 269 270 if (!str) 271 return -EINVAL; 272 273 if (!strcmp(str, "off")) 274 mds_mitigation = MDS_MITIGATION_OFF; 275 else if (!strcmp(str, "full")) 276 mds_mitigation = MDS_MITIGATION_FULL; 277 else if (!strcmp(str, "full,nosmt")) { 278 mds_mitigation = MDS_MITIGATION_FULL; 279 mds_nosmt = true; 280 } 281 282 return 0; 283 } 284 early_param("mds", mds_cmdline); 285 286 #undef pr_fmt 287 #define pr_fmt(fmt) "TAA: " fmt 288 289 /* Default mitigation for TAA-affected CPUs */ 290 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 291 static bool taa_nosmt __ro_after_init; 292 293 static const char * const taa_strings[] = { 294 [TAA_MITIGATION_OFF] = "Vulnerable", 295 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 296 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 297 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 298 }; 299 300 static void __init taa_select_mitigation(void) 301 { 302 u64 ia32_cap; 303 304 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 305 taa_mitigation = TAA_MITIGATION_OFF; 306 return; 307 } 308 309 /* TSX previously disabled by tsx=off */ 310 if (!boot_cpu_has(X86_FEATURE_RTM)) { 311 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 312 goto out; 313 } 314 315 if (cpu_mitigations_off()) { 316 taa_mitigation = TAA_MITIGATION_OFF; 317 return; 318 } 319 320 /* 321 * TAA mitigation via VERW is turned off if both 322 * tsx_async_abort=off and mds=off are specified. 323 */ 324 if (taa_mitigation == TAA_MITIGATION_OFF && 325 mds_mitigation == MDS_MITIGATION_OFF) 326 goto out; 327 328 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 329 taa_mitigation = TAA_MITIGATION_VERW; 330 else 331 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 332 333 /* 334 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 335 * A microcode update fixes this behavior to clear CPU buffers. It also 336 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 337 * ARCH_CAP_TSX_CTRL_MSR bit. 338 * 339 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 340 * update is required. 341 */ 342 ia32_cap = x86_read_arch_cap_msr(); 343 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 344 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 345 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 346 347 /* 348 * TSX is enabled, select alternate mitigation for TAA which is 349 * the same as MDS. Enable MDS static branch to clear CPU buffers. 350 * 351 * For guests that can't determine whether the correct microcode is 352 * present on host, enable the mitigation for UCODE_NEEDED as well. 353 */ 354 static_branch_enable(&mds_user_clear); 355 356 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 357 cpu_smt_disable(false); 358 359 /* 360 * Update MDS mitigation, if necessary, as the mds_user_clear is 361 * now enabled for TAA mitigation. 362 */ 363 if (mds_mitigation == MDS_MITIGATION_OFF && 364 boot_cpu_has_bug(X86_BUG_MDS)) { 365 mds_mitigation = MDS_MITIGATION_FULL; 366 mds_select_mitigation(); 367 } 368 out: 369 pr_info("%s\n", taa_strings[taa_mitigation]); 370 } 371 372 static int __init tsx_async_abort_parse_cmdline(char *str) 373 { 374 if (!boot_cpu_has_bug(X86_BUG_TAA)) 375 return 0; 376 377 if (!str) 378 return -EINVAL; 379 380 if (!strcmp(str, "off")) { 381 taa_mitigation = TAA_MITIGATION_OFF; 382 } else if (!strcmp(str, "full")) { 383 taa_mitigation = TAA_MITIGATION_VERW; 384 } else if (!strcmp(str, "full,nosmt")) { 385 taa_mitigation = TAA_MITIGATION_VERW; 386 taa_nosmt = true; 387 } 388 389 return 0; 390 } 391 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 392 393 #undef pr_fmt 394 #define pr_fmt(fmt) "Spectre V1 : " fmt 395 396 enum spectre_v1_mitigation { 397 SPECTRE_V1_MITIGATION_NONE, 398 SPECTRE_V1_MITIGATION_AUTO, 399 }; 400 401 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 402 SPECTRE_V1_MITIGATION_AUTO; 403 404 static const char * const spectre_v1_strings[] = { 405 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 406 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 407 }; 408 409 /* 410 * Does SMAP provide full mitigation against speculative kernel access to 411 * userspace? 412 */ 413 static bool smap_works_speculatively(void) 414 { 415 if (!boot_cpu_has(X86_FEATURE_SMAP)) 416 return false; 417 418 /* 419 * On CPUs which are vulnerable to Meltdown, SMAP does not 420 * prevent speculative access to user data in the L1 cache. 421 * Consider SMAP to be non-functional as a mitigation on these 422 * CPUs. 423 */ 424 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 425 return false; 426 427 return true; 428 } 429 430 static void __init spectre_v1_select_mitigation(void) 431 { 432 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 433 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 434 return; 435 } 436 437 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 438 /* 439 * With Spectre v1, a user can speculatively control either 440 * path of a conditional swapgs with a user-controlled GS 441 * value. The mitigation is to add lfences to both code paths. 442 * 443 * If FSGSBASE is enabled, the user can put a kernel address in 444 * GS, in which case SMAP provides no protection. 445 * 446 * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the 447 * FSGSBASE enablement patches have been merged. ] 448 * 449 * If FSGSBASE is disabled, the user can only put a user space 450 * address in GS. That makes an attack harder, but still 451 * possible if there's no SMAP protection. 452 */ 453 if (!smap_works_speculatively()) { 454 /* 455 * Mitigation can be provided from SWAPGS itself or 456 * PTI as the CR3 write in the Meltdown mitigation 457 * is serializing. 458 * 459 * If neither is there, mitigate with an LFENCE to 460 * stop speculation through swapgs. 461 */ 462 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 463 !boot_cpu_has(X86_FEATURE_PTI)) 464 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 465 466 /* 467 * Enable lfences in the kernel entry (non-swapgs) 468 * paths, to prevent user entry from speculatively 469 * skipping swapgs. 470 */ 471 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 472 } 473 } 474 475 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 476 } 477 478 static int __init nospectre_v1_cmdline(char *str) 479 { 480 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 481 return 0; 482 } 483 early_param("nospectre_v1", nospectre_v1_cmdline); 484 485 #undef pr_fmt 486 #define pr_fmt(fmt) "Spectre V2 : " fmt 487 488 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = 489 SPECTRE_V2_NONE; 490 491 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init = 492 SPECTRE_V2_USER_NONE; 493 494 #ifdef CONFIG_RETPOLINE 495 static bool spectre_v2_bad_module; 496 497 bool retpoline_module_ok(bool has_retpoline) 498 { 499 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 500 return true; 501 502 pr_err("System may be vulnerable to spectre v2\n"); 503 spectre_v2_bad_module = true; 504 return false; 505 } 506 507 static inline const char *spectre_v2_module_string(void) 508 { 509 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 510 } 511 #else 512 static inline const char *spectre_v2_module_string(void) { return ""; } 513 #endif 514 515 static inline bool match_option(const char *arg, int arglen, const char *opt) 516 { 517 int len = strlen(opt); 518 519 return len == arglen && !strncmp(arg, opt, len); 520 } 521 522 /* The kernel command line selection for spectre v2 */ 523 enum spectre_v2_mitigation_cmd { 524 SPECTRE_V2_CMD_NONE, 525 SPECTRE_V2_CMD_AUTO, 526 SPECTRE_V2_CMD_FORCE, 527 SPECTRE_V2_CMD_RETPOLINE, 528 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 529 SPECTRE_V2_CMD_RETPOLINE_AMD, 530 }; 531 532 enum spectre_v2_user_cmd { 533 SPECTRE_V2_USER_CMD_NONE, 534 SPECTRE_V2_USER_CMD_AUTO, 535 SPECTRE_V2_USER_CMD_FORCE, 536 SPECTRE_V2_USER_CMD_PRCTL, 537 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 538 SPECTRE_V2_USER_CMD_SECCOMP, 539 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 540 }; 541 542 static const char * const spectre_v2_user_strings[] = { 543 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 544 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 545 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 546 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 547 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 548 }; 549 550 static const struct { 551 const char *option; 552 enum spectre_v2_user_cmd cmd; 553 bool secure; 554 } v2_user_options[] __initconst = { 555 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 556 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 557 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 558 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 559 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 560 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 561 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 562 }; 563 564 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 565 { 566 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 567 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 568 } 569 570 static enum spectre_v2_user_cmd __init 571 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) 572 { 573 char arg[20]; 574 int ret, i; 575 576 switch (v2_cmd) { 577 case SPECTRE_V2_CMD_NONE: 578 return SPECTRE_V2_USER_CMD_NONE; 579 case SPECTRE_V2_CMD_FORCE: 580 return SPECTRE_V2_USER_CMD_FORCE; 581 default: 582 break; 583 } 584 585 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 586 arg, sizeof(arg)); 587 if (ret < 0) 588 return SPECTRE_V2_USER_CMD_AUTO; 589 590 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 591 if (match_option(arg, ret, v2_user_options[i].option)) { 592 spec_v2_user_print_cond(v2_user_options[i].option, 593 v2_user_options[i].secure); 594 return v2_user_options[i].cmd; 595 } 596 } 597 598 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 599 return SPECTRE_V2_USER_CMD_AUTO; 600 } 601 602 static void __init 603 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) 604 { 605 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 606 bool smt_possible = IS_ENABLED(CONFIG_SMP); 607 enum spectre_v2_user_cmd cmd; 608 609 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 610 return; 611 612 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 613 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 614 smt_possible = false; 615 616 cmd = spectre_v2_parse_user_cmdline(v2_cmd); 617 switch (cmd) { 618 case SPECTRE_V2_USER_CMD_NONE: 619 goto set_mode; 620 case SPECTRE_V2_USER_CMD_FORCE: 621 mode = SPECTRE_V2_USER_STRICT; 622 break; 623 case SPECTRE_V2_USER_CMD_PRCTL: 624 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 625 mode = SPECTRE_V2_USER_PRCTL; 626 break; 627 case SPECTRE_V2_USER_CMD_AUTO: 628 case SPECTRE_V2_USER_CMD_SECCOMP: 629 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 630 if (IS_ENABLED(CONFIG_SECCOMP)) 631 mode = SPECTRE_V2_USER_SECCOMP; 632 else 633 mode = SPECTRE_V2_USER_PRCTL; 634 break; 635 } 636 637 /* 638 * At this point, an STIBP mode other than "off" has been set. 639 * If STIBP support is not being forced, check if STIBP always-on 640 * is preferred. 641 */ 642 if (mode != SPECTRE_V2_USER_STRICT && 643 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 644 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 645 646 /* Initialize Indirect Branch Prediction Barrier */ 647 if (boot_cpu_has(X86_FEATURE_IBPB)) { 648 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 649 650 switch (cmd) { 651 case SPECTRE_V2_USER_CMD_FORCE: 652 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 653 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 654 static_branch_enable(&switch_mm_always_ibpb); 655 break; 656 case SPECTRE_V2_USER_CMD_PRCTL: 657 case SPECTRE_V2_USER_CMD_AUTO: 658 case SPECTRE_V2_USER_CMD_SECCOMP: 659 static_branch_enable(&switch_mm_cond_ibpb); 660 break; 661 default: 662 break; 663 } 664 665 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 666 static_key_enabled(&switch_mm_always_ibpb) ? 667 "always-on" : "conditional"); 668 } 669 670 /* If enhanced IBRS is enabled no STIBP required */ 671 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) 672 return; 673 674 /* 675 * If SMT is not possible or STIBP is not available clear the STIBP 676 * mode. 677 */ 678 if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP)) 679 mode = SPECTRE_V2_USER_NONE; 680 set_mode: 681 spectre_v2_user = mode; 682 /* Only print the STIBP mode when SMT possible */ 683 if (smt_possible) 684 pr_info("%s\n", spectre_v2_user_strings[mode]); 685 } 686 687 static const char * const spectre_v2_strings[] = { 688 [SPECTRE_V2_NONE] = "Vulnerable", 689 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", 690 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", 691 [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", 692 }; 693 694 static const struct { 695 const char *option; 696 enum spectre_v2_mitigation_cmd cmd; 697 bool secure; 698 } mitigation_options[] __initconst = { 699 { "off", SPECTRE_V2_CMD_NONE, false }, 700 { "on", SPECTRE_V2_CMD_FORCE, true }, 701 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 702 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, 703 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 704 { "auto", SPECTRE_V2_CMD_AUTO, false }, 705 }; 706 707 static void __init spec_v2_print_cond(const char *reason, bool secure) 708 { 709 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 710 pr_info("%s selected on command line.\n", reason); 711 } 712 713 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 714 { 715 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 716 char arg[20]; 717 int ret, i; 718 719 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 720 cpu_mitigations_off()) 721 return SPECTRE_V2_CMD_NONE; 722 723 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 724 if (ret < 0) 725 return SPECTRE_V2_CMD_AUTO; 726 727 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 728 if (!match_option(arg, ret, mitigation_options[i].option)) 729 continue; 730 cmd = mitigation_options[i].cmd; 731 break; 732 } 733 734 if (i >= ARRAY_SIZE(mitigation_options)) { 735 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 736 return SPECTRE_V2_CMD_AUTO; 737 } 738 739 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 740 cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || 741 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && 742 !IS_ENABLED(CONFIG_RETPOLINE)) { 743 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); 744 return SPECTRE_V2_CMD_AUTO; 745 } 746 747 if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && 748 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON && 749 boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { 750 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); 751 return SPECTRE_V2_CMD_AUTO; 752 } 753 754 spec_v2_print_cond(mitigation_options[i].option, 755 mitigation_options[i].secure); 756 return cmd; 757 } 758 759 static void __init spectre_v2_select_mitigation(void) 760 { 761 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 762 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 763 764 /* 765 * If the CPU is not affected and the command line mode is NONE or AUTO 766 * then nothing to do. 767 */ 768 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 769 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 770 return; 771 772 switch (cmd) { 773 case SPECTRE_V2_CMD_NONE: 774 return; 775 776 case SPECTRE_V2_CMD_FORCE: 777 case SPECTRE_V2_CMD_AUTO: 778 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 779 mode = SPECTRE_V2_IBRS_ENHANCED; 780 /* Force it so VMEXIT will restore correctly */ 781 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 782 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 783 goto specv2_set_mode; 784 } 785 if (IS_ENABLED(CONFIG_RETPOLINE)) 786 goto retpoline_auto; 787 break; 788 case SPECTRE_V2_CMD_RETPOLINE_AMD: 789 if (IS_ENABLED(CONFIG_RETPOLINE)) 790 goto retpoline_amd; 791 break; 792 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 793 if (IS_ENABLED(CONFIG_RETPOLINE)) 794 goto retpoline_generic; 795 break; 796 case SPECTRE_V2_CMD_RETPOLINE: 797 if (IS_ENABLED(CONFIG_RETPOLINE)) 798 goto retpoline_auto; 799 break; 800 } 801 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); 802 return; 803 804 retpoline_auto: 805 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 806 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 807 retpoline_amd: 808 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 809 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); 810 goto retpoline_generic; 811 } 812 mode = SPECTRE_V2_RETPOLINE_AMD; 813 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); 814 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 815 } else { 816 retpoline_generic: 817 mode = SPECTRE_V2_RETPOLINE_GENERIC; 818 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 819 } 820 821 specv2_set_mode: 822 spectre_v2_enabled = mode; 823 pr_info("%s\n", spectre_v2_strings[mode]); 824 825 /* 826 * If spectre v2 protection has been enabled, unconditionally fill 827 * RSB during a context switch; this protects against two independent 828 * issues: 829 * 830 * - RSB underflow (and switch to BTB) on Skylake+ 831 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs 832 */ 833 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 834 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 835 836 /* 837 * Retpoline means the kernel is safe because it has no indirect 838 * branches. Enhanced IBRS protects firmware too, so, enable restricted 839 * speculation around firmware calls only when Enhanced IBRS isn't 840 * supported. 841 * 842 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 843 * the user might select retpoline on the kernel command line and if 844 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 845 * enable IBRS around firmware calls. 846 */ 847 if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { 848 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 849 pr_info("Enabling Restricted Speculation for firmware calls\n"); 850 } 851 852 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 853 spectre_v2_user_select_mitigation(cmd); 854 } 855 856 static void update_stibp_msr(void * __unused) 857 { 858 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 859 } 860 861 /* Update x86_spec_ctrl_base in case SMT state changed. */ 862 static void update_stibp_strict(void) 863 { 864 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 865 866 if (sched_smt_active()) 867 mask |= SPEC_CTRL_STIBP; 868 869 if (mask == x86_spec_ctrl_base) 870 return; 871 872 pr_info("Update user space SMT mitigation: STIBP %s\n", 873 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 874 x86_spec_ctrl_base = mask; 875 on_each_cpu(update_stibp_msr, NULL, 1); 876 } 877 878 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 879 static void update_indir_branch_cond(void) 880 { 881 if (sched_smt_active()) 882 static_branch_enable(&switch_to_cond_stibp); 883 else 884 static_branch_disable(&switch_to_cond_stibp); 885 } 886 887 #undef pr_fmt 888 #define pr_fmt(fmt) fmt 889 890 /* Update the static key controlling the MDS CPU buffer clear in idle */ 891 static void update_mds_branch_idle(void) 892 { 893 /* 894 * Enable the idle clearing if SMT is active on CPUs which are 895 * affected only by MSBDS and not any other MDS variant. 896 * 897 * The other variants cannot be mitigated when SMT is enabled, so 898 * clearing the buffers on idle just to prevent the Store Buffer 899 * repartitioning leak would be a window dressing exercise. 900 */ 901 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 902 return; 903 904 if (sched_smt_active()) 905 static_branch_enable(&mds_idle_clear); 906 else 907 static_branch_disable(&mds_idle_clear); 908 } 909 910 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 911 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 912 913 void cpu_bugs_smt_update(void) 914 { 915 mutex_lock(&spec_ctrl_mutex); 916 917 switch (spectre_v2_user) { 918 case SPECTRE_V2_USER_NONE: 919 break; 920 case SPECTRE_V2_USER_STRICT: 921 case SPECTRE_V2_USER_STRICT_PREFERRED: 922 update_stibp_strict(); 923 break; 924 case SPECTRE_V2_USER_PRCTL: 925 case SPECTRE_V2_USER_SECCOMP: 926 update_indir_branch_cond(); 927 break; 928 } 929 930 switch (mds_mitigation) { 931 case MDS_MITIGATION_FULL: 932 case MDS_MITIGATION_VMWERV: 933 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 934 pr_warn_once(MDS_MSG_SMT); 935 update_mds_branch_idle(); 936 break; 937 case MDS_MITIGATION_OFF: 938 break; 939 } 940 941 switch (taa_mitigation) { 942 case TAA_MITIGATION_VERW: 943 case TAA_MITIGATION_UCODE_NEEDED: 944 if (sched_smt_active()) 945 pr_warn_once(TAA_MSG_SMT); 946 break; 947 case TAA_MITIGATION_TSX_DISABLED: 948 case TAA_MITIGATION_OFF: 949 break; 950 } 951 952 mutex_unlock(&spec_ctrl_mutex); 953 } 954 955 #undef pr_fmt 956 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 957 958 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 959 960 /* The kernel command line selection */ 961 enum ssb_mitigation_cmd { 962 SPEC_STORE_BYPASS_CMD_NONE, 963 SPEC_STORE_BYPASS_CMD_AUTO, 964 SPEC_STORE_BYPASS_CMD_ON, 965 SPEC_STORE_BYPASS_CMD_PRCTL, 966 SPEC_STORE_BYPASS_CMD_SECCOMP, 967 }; 968 969 static const char * const ssb_strings[] = { 970 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 971 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 972 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 973 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 974 }; 975 976 static const struct { 977 const char *option; 978 enum ssb_mitigation_cmd cmd; 979 } ssb_mitigation_options[] __initconst = { 980 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 981 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 982 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 983 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 984 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 985 }; 986 987 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 988 { 989 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 990 char arg[20]; 991 int ret, i; 992 993 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 994 cpu_mitigations_off()) { 995 return SPEC_STORE_BYPASS_CMD_NONE; 996 } else { 997 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 998 arg, sizeof(arg)); 999 if (ret < 0) 1000 return SPEC_STORE_BYPASS_CMD_AUTO; 1001 1002 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1003 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1004 continue; 1005 1006 cmd = ssb_mitigation_options[i].cmd; 1007 break; 1008 } 1009 1010 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1011 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1012 return SPEC_STORE_BYPASS_CMD_AUTO; 1013 } 1014 } 1015 1016 return cmd; 1017 } 1018 1019 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1020 { 1021 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1022 enum ssb_mitigation_cmd cmd; 1023 1024 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1025 return mode; 1026 1027 cmd = ssb_parse_cmdline(); 1028 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1029 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1030 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1031 return mode; 1032 1033 switch (cmd) { 1034 case SPEC_STORE_BYPASS_CMD_AUTO: 1035 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1036 /* 1037 * Choose prctl+seccomp as the default mode if seccomp is 1038 * enabled. 1039 */ 1040 if (IS_ENABLED(CONFIG_SECCOMP)) 1041 mode = SPEC_STORE_BYPASS_SECCOMP; 1042 else 1043 mode = SPEC_STORE_BYPASS_PRCTL; 1044 break; 1045 case SPEC_STORE_BYPASS_CMD_ON: 1046 mode = SPEC_STORE_BYPASS_DISABLE; 1047 break; 1048 case SPEC_STORE_BYPASS_CMD_PRCTL: 1049 mode = SPEC_STORE_BYPASS_PRCTL; 1050 break; 1051 case SPEC_STORE_BYPASS_CMD_NONE: 1052 break; 1053 } 1054 1055 /* 1056 * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper 1057 * bit in the mask to allow guests to use the mitigation even in the 1058 * case where the host does not enable it. 1059 */ 1060 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 1061 static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1062 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD; 1063 } 1064 1065 /* 1066 * We have three CPU feature flags that are in play here: 1067 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1068 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1069 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1070 */ 1071 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1072 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1073 /* 1074 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1075 * use a completely different MSR and bit dependent on family. 1076 */ 1077 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1078 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1079 x86_amd_ssb_disable(); 1080 } else { 1081 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1082 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1083 } 1084 } 1085 1086 return mode; 1087 } 1088 1089 static void ssb_select_mitigation(void) 1090 { 1091 ssb_mode = __ssb_select_mitigation(); 1092 1093 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1094 pr_info("%s\n", ssb_strings[ssb_mode]); 1095 } 1096 1097 #undef pr_fmt 1098 #define pr_fmt(fmt) "Speculation prctl: " fmt 1099 1100 static void task_update_spec_tif(struct task_struct *tsk) 1101 { 1102 /* Force the update of the real TIF bits */ 1103 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1104 1105 /* 1106 * Immediately update the speculation control MSRs for the current 1107 * task, but for a non-current task delay setting the CPU 1108 * mitigation until it is scheduled next. 1109 * 1110 * This can only happen for SECCOMP mitigation. For PRCTL it's 1111 * always the current task. 1112 */ 1113 if (tsk == current) 1114 speculation_ctrl_update_current(); 1115 } 1116 1117 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 1118 { 1119 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 1120 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 1121 return -ENXIO; 1122 1123 switch (ctrl) { 1124 case PR_SPEC_ENABLE: 1125 /* If speculation is force disabled, enable is not allowed */ 1126 if (task_spec_ssb_force_disable(task)) 1127 return -EPERM; 1128 task_clear_spec_ssb_disable(task); 1129 task_clear_spec_ssb_noexec(task); 1130 task_update_spec_tif(task); 1131 break; 1132 case PR_SPEC_DISABLE: 1133 task_set_spec_ssb_disable(task); 1134 task_clear_spec_ssb_noexec(task); 1135 task_update_spec_tif(task); 1136 break; 1137 case PR_SPEC_FORCE_DISABLE: 1138 task_set_spec_ssb_disable(task); 1139 task_set_spec_ssb_force_disable(task); 1140 task_clear_spec_ssb_noexec(task); 1141 task_update_spec_tif(task); 1142 break; 1143 case PR_SPEC_DISABLE_NOEXEC: 1144 if (task_spec_ssb_force_disable(task)) 1145 return -EPERM; 1146 task_set_spec_ssb_disable(task); 1147 task_set_spec_ssb_noexec(task); 1148 task_update_spec_tif(task); 1149 break; 1150 default: 1151 return -ERANGE; 1152 } 1153 return 0; 1154 } 1155 1156 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 1157 { 1158 switch (ctrl) { 1159 case PR_SPEC_ENABLE: 1160 if (spectre_v2_user == SPECTRE_V2_USER_NONE) 1161 return 0; 1162 /* 1163 * Indirect branch speculation is always disabled in strict 1164 * mode. 1165 */ 1166 if (spectre_v2_user == SPECTRE_V2_USER_STRICT || 1167 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) 1168 return -EPERM; 1169 task_clear_spec_ib_disable(task); 1170 task_update_spec_tif(task); 1171 break; 1172 case PR_SPEC_DISABLE: 1173 case PR_SPEC_FORCE_DISABLE: 1174 /* 1175 * Indirect branch speculation is always allowed when 1176 * mitigation is force disabled. 1177 */ 1178 if (spectre_v2_user == SPECTRE_V2_USER_NONE) 1179 return -EPERM; 1180 if (spectre_v2_user == SPECTRE_V2_USER_STRICT || 1181 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) 1182 return 0; 1183 task_set_spec_ib_disable(task); 1184 if (ctrl == PR_SPEC_FORCE_DISABLE) 1185 task_set_spec_ib_force_disable(task); 1186 task_update_spec_tif(task); 1187 break; 1188 default: 1189 return -ERANGE; 1190 } 1191 return 0; 1192 } 1193 1194 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 1195 unsigned long ctrl) 1196 { 1197 switch (which) { 1198 case PR_SPEC_STORE_BYPASS: 1199 return ssb_prctl_set(task, ctrl); 1200 case PR_SPEC_INDIRECT_BRANCH: 1201 return ib_prctl_set(task, ctrl); 1202 default: 1203 return -ENODEV; 1204 } 1205 } 1206 1207 #ifdef CONFIG_SECCOMP 1208 void arch_seccomp_spec_mitigate(struct task_struct *task) 1209 { 1210 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 1211 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1212 if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP) 1213 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1214 } 1215 #endif 1216 1217 static int ssb_prctl_get(struct task_struct *task) 1218 { 1219 switch (ssb_mode) { 1220 case SPEC_STORE_BYPASS_DISABLE: 1221 return PR_SPEC_DISABLE; 1222 case SPEC_STORE_BYPASS_SECCOMP: 1223 case SPEC_STORE_BYPASS_PRCTL: 1224 if (task_spec_ssb_force_disable(task)) 1225 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1226 if (task_spec_ssb_noexec(task)) 1227 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 1228 if (task_spec_ssb_disable(task)) 1229 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1230 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1231 default: 1232 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1233 return PR_SPEC_ENABLE; 1234 return PR_SPEC_NOT_AFFECTED; 1235 } 1236 } 1237 1238 static int ib_prctl_get(struct task_struct *task) 1239 { 1240 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 1241 return PR_SPEC_NOT_AFFECTED; 1242 1243 switch (spectre_v2_user) { 1244 case SPECTRE_V2_USER_NONE: 1245 return PR_SPEC_ENABLE; 1246 case SPECTRE_V2_USER_PRCTL: 1247 case SPECTRE_V2_USER_SECCOMP: 1248 if (task_spec_ib_force_disable(task)) 1249 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1250 if (task_spec_ib_disable(task)) 1251 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1252 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1253 case SPECTRE_V2_USER_STRICT: 1254 case SPECTRE_V2_USER_STRICT_PREFERRED: 1255 return PR_SPEC_DISABLE; 1256 default: 1257 return PR_SPEC_NOT_AFFECTED; 1258 } 1259 } 1260 1261 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 1262 { 1263 switch (which) { 1264 case PR_SPEC_STORE_BYPASS: 1265 return ssb_prctl_get(task); 1266 case PR_SPEC_INDIRECT_BRANCH: 1267 return ib_prctl_get(task); 1268 default: 1269 return -ENODEV; 1270 } 1271 } 1272 1273 void x86_spec_ctrl_setup_ap(void) 1274 { 1275 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 1276 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1277 1278 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 1279 x86_amd_ssb_disable(); 1280 } 1281 1282 bool itlb_multihit_kvm_mitigation; 1283 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 1284 1285 #undef pr_fmt 1286 #define pr_fmt(fmt) "L1TF: " fmt 1287 1288 /* Default mitigation for L1TF-affected CPUs */ 1289 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 1290 #if IS_ENABLED(CONFIG_KVM_INTEL) 1291 EXPORT_SYMBOL_GPL(l1tf_mitigation); 1292 #endif 1293 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 1294 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 1295 1296 /* 1297 * These CPUs all support 44bits physical address space internally in the 1298 * cache but CPUID can report a smaller number of physical address bits. 1299 * 1300 * The L1TF mitigation uses the top most address bit for the inversion of 1301 * non present PTEs. When the installed memory reaches into the top most 1302 * address bit due to memory holes, which has been observed on machines 1303 * which report 36bits physical address bits and have 32G RAM installed, 1304 * then the mitigation range check in l1tf_select_mitigation() triggers. 1305 * This is a false positive because the mitigation is still possible due to 1306 * the fact that the cache uses 44bit internally. Use the cache bits 1307 * instead of the reported physical bits and adjust them on the affected 1308 * machines to 44bit if the reported bits are less than 44. 1309 */ 1310 static void override_cache_bits(struct cpuinfo_x86 *c) 1311 { 1312 if (c->x86 != 6) 1313 return; 1314 1315 switch (c->x86_model) { 1316 case INTEL_FAM6_NEHALEM: 1317 case INTEL_FAM6_WESTMERE: 1318 case INTEL_FAM6_SANDYBRIDGE: 1319 case INTEL_FAM6_IVYBRIDGE: 1320 case INTEL_FAM6_HASWELL: 1321 case INTEL_FAM6_HASWELL_L: 1322 case INTEL_FAM6_HASWELL_G: 1323 case INTEL_FAM6_BROADWELL: 1324 case INTEL_FAM6_BROADWELL_G: 1325 case INTEL_FAM6_SKYLAKE_L: 1326 case INTEL_FAM6_SKYLAKE: 1327 case INTEL_FAM6_KABYLAKE_L: 1328 case INTEL_FAM6_KABYLAKE: 1329 if (c->x86_cache_bits < 44) 1330 c->x86_cache_bits = 44; 1331 break; 1332 } 1333 } 1334 1335 static void __init l1tf_select_mitigation(void) 1336 { 1337 u64 half_pa; 1338 1339 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1340 return; 1341 1342 if (cpu_mitigations_off()) 1343 l1tf_mitigation = L1TF_MITIGATION_OFF; 1344 else if (cpu_mitigations_auto_nosmt()) 1345 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1346 1347 override_cache_bits(&boot_cpu_data); 1348 1349 switch (l1tf_mitigation) { 1350 case L1TF_MITIGATION_OFF: 1351 case L1TF_MITIGATION_FLUSH_NOWARN: 1352 case L1TF_MITIGATION_FLUSH: 1353 break; 1354 case L1TF_MITIGATION_FLUSH_NOSMT: 1355 case L1TF_MITIGATION_FULL: 1356 cpu_smt_disable(false); 1357 break; 1358 case L1TF_MITIGATION_FULL_FORCE: 1359 cpu_smt_disable(true); 1360 break; 1361 } 1362 1363 #if CONFIG_PGTABLE_LEVELS == 2 1364 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 1365 return; 1366 #endif 1367 1368 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 1369 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 1370 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 1371 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 1372 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 1373 half_pa); 1374 pr_info("However, doing so will make a part of your RAM unusable.\n"); 1375 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 1376 return; 1377 } 1378 1379 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 1380 } 1381 1382 static int __init l1tf_cmdline(char *str) 1383 { 1384 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1385 return 0; 1386 1387 if (!str) 1388 return -EINVAL; 1389 1390 if (!strcmp(str, "off")) 1391 l1tf_mitigation = L1TF_MITIGATION_OFF; 1392 else if (!strcmp(str, "flush,nowarn")) 1393 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 1394 else if (!strcmp(str, "flush")) 1395 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 1396 else if (!strcmp(str, "flush,nosmt")) 1397 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1398 else if (!strcmp(str, "full")) 1399 l1tf_mitigation = L1TF_MITIGATION_FULL; 1400 else if (!strcmp(str, "full,force")) 1401 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 1402 1403 return 0; 1404 } 1405 early_param("l1tf", l1tf_cmdline); 1406 1407 #undef pr_fmt 1408 #define pr_fmt(fmt) fmt 1409 1410 #ifdef CONFIG_SYSFS 1411 1412 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 1413 1414 #if IS_ENABLED(CONFIG_KVM_INTEL) 1415 static const char * const l1tf_vmx_states[] = { 1416 [VMENTER_L1D_FLUSH_AUTO] = "auto", 1417 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 1418 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 1419 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 1420 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 1421 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 1422 }; 1423 1424 static ssize_t l1tf_show_state(char *buf) 1425 { 1426 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 1427 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1428 1429 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 1430 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 1431 sched_smt_active())) { 1432 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 1433 l1tf_vmx_states[l1tf_vmx_mitigation]); 1434 } 1435 1436 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 1437 l1tf_vmx_states[l1tf_vmx_mitigation], 1438 sched_smt_active() ? "vulnerable" : "disabled"); 1439 } 1440 1441 static ssize_t itlb_multihit_show_state(char *buf) 1442 { 1443 if (itlb_multihit_kvm_mitigation) 1444 return sprintf(buf, "KVM: Mitigation: Split huge pages\n"); 1445 else 1446 return sprintf(buf, "KVM: Vulnerable\n"); 1447 } 1448 #else 1449 static ssize_t l1tf_show_state(char *buf) 1450 { 1451 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1452 } 1453 1454 static ssize_t itlb_multihit_show_state(char *buf) 1455 { 1456 return sprintf(buf, "Processor vulnerable\n"); 1457 } 1458 #endif 1459 1460 static ssize_t mds_show_state(char *buf) 1461 { 1462 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1463 return sprintf(buf, "%s; SMT Host state unknown\n", 1464 mds_strings[mds_mitigation]); 1465 } 1466 1467 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 1468 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1469 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 1470 sched_smt_active() ? "mitigated" : "disabled")); 1471 } 1472 1473 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1474 sched_smt_active() ? "vulnerable" : "disabled"); 1475 } 1476 1477 static ssize_t tsx_async_abort_show_state(char *buf) 1478 { 1479 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 1480 (taa_mitigation == TAA_MITIGATION_OFF)) 1481 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]); 1482 1483 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1484 return sprintf(buf, "%s; SMT Host state unknown\n", 1485 taa_strings[taa_mitigation]); 1486 } 1487 1488 return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 1489 sched_smt_active() ? "vulnerable" : "disabled"); 1490 } 1491 1492 static char *stibp_state(void) 1493 { 1494 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) 1495 return ""; 1496 1497 switch (spectre_v2_user) { 1498 case SPECTRE_V2_USER_NONE: 1499 return ", STIBP: disabled"; 1500 case SPECTRE_V2_USER_STRICT: 1501 return ", STIBP: forced"; 1502 case SPECTRE_V2_USER_STRICT_PREFERRED: 1503 return ", STIBP: always-on"; 1504 case SPECTRE_V2_USER_PRCTL: 1505 case SPECTRE_V2_USER_SECCOMP: 1506 if (static_key_enabled(&switch_to_cond_stibp)) 1507 return ", STIBP: conditional"; 1508 } 1509 return ""; 1510 } 1511 1512 static char *ibpb_state(void) 1513 { 1514 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1515 if (static_key_enabled(&switch_mm_always_ibpb)) 1516 return ", IBPB: always-on"; 1517 if (static_key_enabled(&switch_mm_cond_ibpb)) 1518 return ", IBPB: conditional"; 1519 return ", IBPB: disabled"; 1520 } 1521 return ""; 1522 } 1523 1524 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 1525 char *buf, unsigned int bug) 1526 { 1527 if (!boot_cpu_has_bug(bug)) 1528 return sprintf(buf, "Not affected\n"); 1529 1530 switch (bug) { 1531 case X86_BUG_CPU_MELTDOWN: 1532 if (boot_cpu_has(X86_FEATURE_PTI)) 1533 return sprintf(buf, "Mitigation: PTI\n"); 1534 1535 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 1536 return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 1537 1538 break; 1539 1540 case X86_BUG_SPECTRE_V1: 1541 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1542 1543 case X86_BUG_SPECTRE_V2: 1544 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], 1545 ibpb_state(), 1546 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 1547 stibp_state(), 1548 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 1549 spectre_v2_module_string()); 1550 1551 case X86_BUG_SPEC_STORE_BYPASS: 1552 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); 1553 1554 case X86_BUG_L1TF: 1555 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 1556 return l1tf_show_state(buf); 1557 break; 1558 1559 case X86_BUG_MDS: 1560 return mds_show_state(buf); 1561 1562 case X86_BUG_TAA: 1563 return tsx_async_abort_show_state(buf); 1564 1565 case X86_BUG_ITLB_MULTIHIT: 1566 return itlb_multihit_show_state(buf); 1567 1568 default: 1569 break; 1570 } 1571 1572 return sprintf(buf, "Vulnerable\n"); 1573 } 1574 1575 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 1576 { 1577 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 1578 } 1579 1580 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 1581 { 1582 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 1583 } 1584 1585 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 1586 { 1587 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 1588 } 1589 1590 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 1591 { 1592 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 1593 } 1594 1595 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 1596 { 1597 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 1598 } 1599 1600 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 1601 { 1602 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 1603 } 1604 1605 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 1606 { 1607 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 1608 } 1609 1610 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 1611 { 1612 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 1613 } 1614 #endif 1615