1 /*- 2 * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com> 3 * Copyright 2014 Michal Meloun <meloun@miracle.cz> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/pcpu.h> 35 #include <sys/smp.h> 36 #include <sys/sysctl.h> 37 38 #include <machine/cpu.h> 39 #include <machine/cpuinfo.h> 40 #include <machine/elf.h> 41 #include <machine/md_var.h> 42 43 void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set); 44 45 int disable_bp_hardening; 46 int spectre_v2_safe = 1; 47 48 struct cpuinfo cpuinfo = 49 { 50 /* Use safe defaults for start */ 51 .dcache_line_size = 32, 52 .dcache_line_mask = 31, 53 .icache_line_size = 32, 54 .icache_line_mask = 31, 55 }; 56 57 static SYSCTL_NODE(_hw, OID_AUTO, cpu, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 58 "CPU"); 59 static SYSCTL_NODE(_hw_cpu, OID_AUTO, quirks, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 60 "CPU quirks"); 61 62 /* 63 * Tunable CPU quirks. 64 * Be careful, ACTRL cannot be changed if CPU is started in secure 65 * mode(world) and write to ACTRL can cause exception! 66 * These quirks are intended for optimizing CPU performance, not for 67 * applying errata workarounds. Nobody can expect that CPU with unfixed 68 * errata is stable enough to execute the kernel until quirks are applied. 69 */ 70 static uint32_t cpu_quirks_actlr_mask; 71 SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_mask, 72 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_mask, 0, 73 "Bits to be masked in ACTLR"); 74 75 static uint32_t cpu_quirks_actlr_set; 76 SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_set, 77 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_set, 0, 78 "Bits to be set in ACTLR"); 79 80 static int 81 sysctl_hw_cpu_quirks_actrl_value(SYSCTL_HANDLER_ARGS) 82 { 83 uint32_t reg; 84 85 reg = cp15_actlr_get(); 86 return (SYSCTL_OUT(req, ®, sizeof(reg))); 87 } 88 SYSCTL_PROC(_hw_cpu_quirks, OID_AUTO, actlr_value, 89 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 90 sysctl_hw_cpu_quirks_actrl_value, "IU", 91 "Value of ACTLR"); 92 93 /* Read and parse CPU id scheme */ 94 void 95 cpuinfo_init(void) 96 { 97 uint32_t tmp; 98 99 /* 100 * Prematurely fetch CPU quirks. Standard fetch for tunable 101 * sysctls is handled using SYSINIT, thus too late for boot CPU. 102 * Keep names in sync with sysctls. 103 */ 104 TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_mask", &cpu_quirks_actlr_mask); 105 TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_set", &cpu_quirks_actlr_set); 106 107 cpuinfo.midr = cp15_midr_get(); 108 /* Test old version id schemes first */ 109 if ((cpuinfo.midr & CPU_ID_IMPLEMENTOR_MASK) == CPU_ID_ARM_LTD) { 110 if (CPU_ID_ISOLD(cpuinfo.midr)) { 111 /* obsolete ARMv2 or ARMv3 CPU */ 112 cpuinfo.midr = 0; 113 return; 114 } 115 if (CPU_ID_IS7(cpuinfo.midr)) { 116 if ((cpuinfo.midr & (1 << 23)) == 0) { 117 /* obsolete ARMv3 CPU */ 118 cpuinfo.midr = 0; 119 return; 120 } 121 /* ARMv4T CPU */ 122 cpuinfo.architecture = 1; 123 cpuinfo.revision = (cpuinfo.midr >> 16) & 0x7F; 124 } else { 125 /* ARM new id scheme */ 126 cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 127 cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 128 } 129 } else { 130 /* non ARM -> must be new id scheme */ 131 cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 132 cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 133 } 134 /* Parse rest of MIDR */ 135 cpuinfo.implementer = (cpuinfo.midr >> 24) & 0xFF; 136 cpuinfo.part_number = (cpuinfo.midr >> 4) & 0xFFF; 137 cpuinfo.patch = cpuinfo.midr & 0x0F; 138 139 /* CP15 c0,c0 regs 0-7 exist on all CPUs (although aliased with MIDR) */ 140 cpuinfo.ctr = cp15_ctr_get(); 141 cpuinfo.tcmtr = cp15_tcmtr_get(); 142 cpuinfo.tlbtr = cp15_tlbtr_get(); 143 cpuinfo.mpidr = cp15_mpidr_get(); 144 cpuinfo.revidr = cp15_revidr_get(); 145 146 /* if CPU is not v7 cpu id scheme */ 147 if (cpuinfo.architecture != 0xF) 148 return; 149 cpuinfo.id_pfr0 = cp15_id_pfr0_get(); 150 cpuinfo.id_pfr1 = cp15_id_pfr1_get(); 151 cpuinfo.id_dfr0 = cp15_id_dfr0_get(); 152 cpuinfo.id_afr0 = cp15_id_afr0_get(); 153 cpuinfo.id_mmfr0 = cp15_id_mmfr0_get(); 154 cpuinfo.id_mmfr1 = cp15_id_mmfr1_get(); 155 cpuinfo.id_mmfr2 = cp15_id_mmfr2_get(); 156 cpuinfo.id_mmfr3 = cp15_id_mmfr3_get(); 157 cpuinfo.id_isar0 = cp15_id_isar0_get(); 158 cpuinfo.id_isar1 = cp15_id_isar1_get(); 159 cpuinfo.id_isar2 = cp15_id_isar2_get(); 160 cpuinfo.id_isar3 = cp15_id_isar3_get(); 161 cpuinfo.id_isar4 = cp15_id_isar4_get(); 162 cpuinfo.id_isar5 = cp15_id_isar5_get(); 163 164 /* Not yet - CBAR only exist on ARM SMP Cortex A CPUs 165 cpuinfo.cbar = cp15_cbar_get(); 166 */ 167 if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 168 cpuinfo.ccsidr = cp15_ccsidr_get(); 169 cpuinfo.clidr = cp15_clidr_get(); 170 } 171 172 /* Test if revidr is implemented */ 173 if (cpuinfo.revidr == cpuinfo.midr) 174 cpuinfo.revidr = 0; 175 176 /* parsed bits of above registers */ 177 /* id_mmfr0 */ 178 cpuinfo.outermost_shareability = (cpuinfo.id_mmfr0 >> 8) & 0xF; 179 cpuinfo.shareability_levels = (cpuinfo.id_mmfr0 >> 12) & 0xF; 180 cpuinfo.auxiliary_registers = (cpuinfo.id_mmfr0 >> 20) & 0xF; 181 cpuinfo.innermost_shareability = (cpuinfo.id_mmfr0 >> 28) & 0xF; 182 /* id_mmfr2 */ 183 cpuinfo.mem_barrier = (cpuinfo.id_mmfr2 >> 20) & 0xF; 184 /* id_mmfr3 */ 185 cpuinfo.coherent_walk = (cpuinfo.id_mmfr3 >> 20) & 0xF; 186 cpuinfo.maintenance_broadcast =(cpuinfo.id_mmfr3 >> 12) & 0xF; 187 /* id_pfr1 */ 188 cpuinfo.generic_timer_ext = (cpuinfo.id_pfr1 >> 16) & 0xF; 189 cpuinfo.virtualization_ext = (cpuinfo.id_pfr1 >> 12) & 0xF; 190 cpuinfo.security_ext = (cpuinfo.id_pfr1 >> 4) & 0xF; 191 /* mpidr */ 192 cpuinfo.mp_ext = (cpuinfo.mpidr >> 31u) & 0x1; 193 194 /* L1 Cache sizes */ 195 if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 196 cpuinfo.dcache_line_size = 197 1 << (CPU_CT_DMINLINE(cpuinfo.ctr) + 2); 198 cpuinfo.icache_line_size = 199 1 << (CPU_CT_IMINLINE(cpuinfo.ctr) + 2); 200 } else { 201 cpuinfo.dcache_line_size = 202 1 << (CPU_CT_xSIZE_LEN(CPU_CT_DSIZE(cpuinfo.ctr)) + 3); 203 cpuinfo.icache_line_size = 204 1 << (CPU_CT_xSIZE_LEN(CPU_CT_ISIZE(cpuinfo.ctr)) + 3); 205 } 206 cpuinfo.dcache_line_mask = cpuinfo.dcache_line_size - 1; 207 cpuinfo.icache_line_mask = cpuinfo.icache_line_size - 1; 208 209 /* Fill AT_HWCAP bits. */ 210 elf_hwcap |= HWCAP_HALF | HWCAP_FAST_MULT; /* Required for all CPUs */ 211 elf_hwcap |= HWCAP_TLS | HWCAP_EDSP; /* Required for v6+ CPUs */ 212 213 tmp = (cpuinfo.id_isar0 >> 24) & 0xF; /* Divide_instrs */ 214 if (tmp >= 1) 215 elf_hwcap |= HWCAP_IDIVT; 216 if (tmp >= 2) 217 elf_hwcap |= HWCAP_IDIVA; 218 219 tmp = (cpuinfo.id_pfr0 >> 4) & 0xF; /* State1 */ 220 if (tmp >= 1) 221 elf_hwcap |= HWCAP_THUMB; 222 223 tmp = (cpuinfo.id_pfr0 >> 12) & 0xF; /* State3 */ 224 if (tmp >= 1) 225 elf_hwcap |= HWCAP_THUMBEE; 226 227 tmp = (cpuinfo.id_mmfr0 >> 0) & 0xF; /* VMSA */ 228 if (tmp >= 5) 229 elf_hwcap |= HWCAP_LPAE; 230 231 /* Fill AT_HWCAP2 bits. */ 232 tmp = (cpuinfo.id_isar5 >> 4) & 0xF; /* AES */ 233 if (tmp >= 1) 234 elf_hwcap2 |= HWCAP2_AES; 235 if (tmp >= 2) 236 elf_hwcap2 |= HWCAP2_PMULL; 237 238 tmp = (cpuinfo.id_isar5 >> 8) & 0xF; /* SHA1 */ 239 if (tmp >= 1) 240 elf_hwcap2 |= HWCAP2_SHA1; 241 242 tmp = (cpuinfo.id_isar5 >> 12) & 0xF; /* SHA2 */ 243 if (tmp >= 1) 244 elf_hwcap2 |= HWCAP2_SHA2; 245 246 tmp = (cpuinfo.id_isar5 >> 16) & 0xF; /* CRC32 */ 247 if (tmp >= 1) 248 elf_hwcap2 |= HWCAP2_CRC32; 249 } 250 251 /* 252 * Get bits that must be set or cleared in ACLR register. 253 * Note: Bits in ACLR register are IMPLEMENTATION DEFINED. 254 * Its expected that SCU is in operational state before this 255 * function is called. 256 */ 257 static void 258 cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set) 259 { 260 261 *actlr_mask = 0; 262 *actlr_set = 0; 263 264 if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 265 switch (cpuinfo.part_number) { 266 case CPU_ARCH_CORTEX_A75: 267 case CPU_ARCH_CORTEX_A73: 268 case CPU_ARCH_CORTEX_A72: 269 case CPU_ARCH_CORTEX_A57: 270 case CPU_ARCH_CORTEX_A53: 271 /* Nothing to do for AArch32 */ 272 break; 273 case CPU_ARCH_CORTEX_A17: 274 case CPU_ARCH_CORTEX_A12: /* A12 is merged to A17 */ 275 /* 276 * Enable SMP mode 277 */ 278 *actlr_mask = (1 << 6); 279 *actlr_set = (1 << 6); 280 break; 281 case CPU_ARCH_CORTEX_A15: 282 /* 283 * Enable snoop-delayed exclusive handling 284 * Enable SMP mode 285 */ 286 *actlr_mask = (1U << 31) |(1 << 6); 287 *actlr_set = (1U << 31) |(1 << 6); 288 break; 289 case CPU_ARCH_CORTEX_A9: 290 /* 291 * Disable exclusive L1/L2 cache control 292 * Enable SMP mode 293 * Enable Cache and TLB maintenance broadcast 294 */ 295 *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 296 *actlr_set = (1 << 6) | (1 << 0); 297 break; 298 case CPU_ARCH_CORTEX_A8: 299 /* 300 * Enable L2 cache 301 * Enable L1 data cache hardware alias checks 302 */ 303 *actlr_mask = (1 << 1) | (1 << 0); 304 *actlr_set = (1 << 1); 305 break; 306 case CPU_ARCH_CORTEX_A7: 307 /* 308 * Enable SMP mode 309 */ 310 *actlr_mask = (1 << 6); 311 *actlr_set = (1 << 6); 312 break; 313 case CPU_ARCH_CORTEX_A5: 314 /* 315 * Disable exclusive L1/L2 cache control 316 * Enable SMP mode 317 * Enable Cache and TLB maintenance broadcast 318 */ 319 *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 320 *actlr_set = (1 << 6) | (1 << 0); 321 break; 322 case CPU_ARCH_ARM1176: 323 /* 324 * Restrict cache size to 16KB 325 * Enable the return stack 326 * Enable dynamic branch prediction 327 * Enable static branch prediction 328 */ 329 *actlr_mask = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 330 *actlr_set = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 331 break; 332 } 333 return; 334 } 335 } 336 337 /* Reinitialize MMU to final kernel mapping and apply all CPU quirks. */ 338 void 339 cpuinfo_reinit_mmu(uint32_t ttb) 340 { 341 uint32_t actlr_mask; 342 uint32_t actlr_set; 343 344 cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); 345 actlr_mask |= cpu_quirks_actlr_mask; 346 actlr_set |= cpu_quirks_actlr_set; 347 reinit_mmu(ttb, actlr_mask, actlr_set); 348 } 349 350 static bool 351 modify_actlr(uint32_t clear, uint32_t set) 352 { 353 uint32_t reg, newreg; 354 355 reg = cp15_actlr_get(); 356 newreg = reg; 357 newreg &= ~clear; 358 newreg |= set; 359 if (reg == newreg) 360 return (true); 361 cp15_actlr_set(newreg); 362 363 reg = cp15_actlr_get(); 364 if (reg == newreg) 365 return (true); 366 return (false); 367 } 368 369 /* Apply/restore BP hardening on current core. */ 370 static int 371 apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask) 372 { 373 if (enable) { 374 if (actrl && !modify_actlr(0, set_mask)) 375 return (-1); 376 PCPU_SET(bp_harden_kind, kind); 377 } else { 378 PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 379 if (actrl) 380 modify_actlr(~0, PCPU_GET(original_actlr)); 381 spectre_v2_safe = 0; 382 } 383 return (0); 384 } 385 386 static void 387 handle_bp_hardening(bool enable) 388 { 389 int kind; 390 char *kind_str; 391 392 kind = PCPU_BP_HARDEN_KIND_NONE; 393 /* 394 * Note: Access to ACTRL is locked to secure world on most boards. 395 * This means that full BP hardening depends on updated u-boot/firmware 396 * or is impossible at all (if secure monitor is in on-chip ROM). 397 */ 398 if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 399 switch (cpuinfo.part_number) { 400 case CPU_ARCH_CORTEX_A8: 401 /* 402 * For Cortex-A8, IBE bit must be set otherwise 403 * BPIALL is effectively NOP. 404 * Unfortunately, Cortex-A is also affected by 405 * ARM erratum 687067 which causes non-working 406 * BPIALL if IBE bit is set and 'Instruction L1 System 407 * Array Debug Register 0' is not 0. 408 * This register is not reset on power-up and is 409 * accessible only from secure world, so we cannot do 410 * nothing (nor detect) to fix this issue. 411 * I afraid that on chip ROM based secure monitor on 412 * AM335x (BeagleBone) doesn't reset this debug 413 * register. 414 */ 415 kind = PCPU_BP_HARDEN_KIND_BPIALL; 416 if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0) 417 goto actlr_err; 418 break; 419 break; 420 421 case CPU_ARCH_CORTEX_A9: 422 case CPU_ARCH_CORTEX_A12: 423 case CPU_ARCH_CORTEX_A17: 424 case CPU_ARCH_CORTEX_A57: 425 case CPU_ARCH_CORTEX_A72: 426 case CPU_ARCH_CORTEX_A73: 427 case CPU_ARCH_CORTEX_A75: 428 kind = PCPU_BP_HARDEN_KIND_BPIALL; 429 if (apply_bp_hardening(enable, kind, false, 0) != 0) 430 goto actlr_err; 431 break; 432 433 case CPU_ARCH_CORTEX_A15: 434 /* 435 * For Cortex-A15, set 'Enable invalidates of BTB' bit. 436 * Despite this, the BPIALL is still effectively NOP, 437 * but with this bit set, the ICIALLU also flushes 438 * branch predictor as side effect. 439 */ 440 kind = PCPU_BP_HARDEN_KIND_ICIALLU; 441 if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0) 442 goto actlr_err; 443 break; 444 445 default: 446 break; 447 } 448 } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) { 449 printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative " 450 "branch attacks. !!!\n" 451 "Qualcomm Krait cores are known (or believed) to be " 452 "vulnerable to \n" 453 "speculative branch attacks, no mitigation exists yet.\n", 454 PCPU_GET(cpuid)); 455 goto unkonown_mitigation; 456 } else { 457 goto unkonown_mitigation; 458 } 459 460 if (bootverbose) { 461 switch (kind) { 462 case PCPU_BP_HARDEN_KIND_NONE: 463 kind_str = "not necessary"; 464 break; 465 case PCPU_BP_HARDEN_KIND_BPIALL: 466 kind_str = "BPIALL"; 467 break; 468 case PCPU_BP_HARDEN_KIND_ICIALLU: 469 kind_str = "ICIALLU"; 470 break; 471 default: 472 panic("Unknown BP hardering kind (%d).", kind); 473 } 474 printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid), 475 kind_str); 476 } 477 478 return; 479 480 unkonown_mitigation: 481 PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 482 spectre_v2_safe = 0; 483 return; 484 485 actlr_err: 486 PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 487 spectre_v2_safe = 0; 488 printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch " 489 "attacks. !!!\n" 490 "We cannot enable required bit(s) in ACTRL register\n" 491 "because it's locked by secure monitor and/or firmware.\n", 492 PCPU_GET(cpuid)); 493 } 494 495 void 496 cpuinfo_init_bp_hardening(void) 497 { 498 499 /* 500 * Store original unmodified ACTRL, so we can restore it when 501 * BP hardening is disabled by sysctl. 502 */ 503 PCPU_SET(original_actlr, cp15_actlr_get()); 504 handle_bp_hardening(true); 505 } 506 507 static void 508 bp_hardening_action(void *arg) 509 { 510 511 handle_bp_hardening(disable_bp_hardening == 0); 512 } 513 514 static int 515 sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS) 516 { 517 int rv; 518 519 rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 520 521 if (!rv && req->newptr) { 522 spectre_v2_safe = 1; 523 dmb(); 524 #ifdef SMP 525 smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, 526 bp_hardening_action, NULL, NULL); 527 #else 528 bp_hardening_action(NULL); 529 #endif 530 } 531 532 return (rv); 533 } 534 535 SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening, 536 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 537 &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I", 538 "Disable BP hardening mitigation."); 539 540 SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD, 541 &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks"); 542