12b71562fSIan Lepore /*- 22b71562fSIan Lepore * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com> 32b71562fSIan Lepore * Copyright 2014 Michal Meloun <meloun@miracle.cz> 42b71562fSIan Lepore * All rights reserved. 52b71562fSIan Lepore * 62b71562fSIan Lepore * Redistribution and use in source and binary forms, with or without 72b71562fSIan Lepore * modification, are permitted provided that the following conditions 82b71562fSIan Lepore * are met: 92b71562fSIan Lepore * 1. Redistributions of source code must retain the above copyright 102b71562fSIan Lepore * notice, this list of conditions and the following disclaimer. 112b71562fSIan Lepore * 2. Redistributions in binary form must reproduce the above copyright 122b71562fSIan Lepore * notice, this list of conditions and the following disclaimer in the 132b71562fSIan Lepore * documentation and/or other materials provided with the distribution. 142b71562fSIan Lepore * 152b71562fSIan Lepore * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 162b71562fSIan Lepore * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 172b71562fSIan Lepore * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 182b71562fSIan Lepore * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 192b71562fSIan Lepore * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 202b71562fSIan Lepore * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 212b71562fSIan Lepore * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 222b71562fSIan Lepore * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 232b71562fSIan Lepore * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 242b71562fSIan Lepore * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 252b71562fSIan Lepore * SUCH DAMAGE. 262b71562fSIan Lepore */ 272b71562fSIan Lepore 282b71562fSIan Lepore #include <sys/cdefs.h> 292b71562fSIan Lepore __FBSDID("$FreeBSD$"); 302b71562fSIan Lepore 312b71562fSIan Lepore #include <sys/param.h> 322b71562fSIan Lepore #include <sys/systm.h> 33c40a5f8aSMichal Meloun #include <sys/kernel.h> 34a36b6ec0SMichal Meloun #include <sys/pcpu.h> 35a36b6ec0SMichal Meloun #include <sys/smp.h> 367bf5720aSMichal Meloun #include <sys/sysctl.h> 372b71562fSIan Lepore 383025d19dSMichal Meloun #include <machine/cpu.h> 392b71562fSIan Lepore #include <machine/cpuinfo.h> 400cbf724eSMichal Meloun #include <machine/elf.h> 410cbf724eSMichal Meloun #include <machine/md_var.h> 422b71562fSIan Lepore 437bf5720aSMichal Meloun void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set); 44a36b6ec0SMichal Meloun 45a36b6ec0SMichal Meloun int disable_bp_hardening; 46a36b6ec0SMichal Meloun int spectre_v2_safe = 1; 477bf5720aSMichal Meloun 48a286c311SIan Lepore struct cpuinfo cpuinfo = 49a286c311SIan Lepore { 50a286c311SIan Lepore /* Use safe defaults for start */ 51a286c311SIan Lepore .dcache_line_size = 32, 52a286c311SIan Lepore .dcache_line_mask = 31, 53a286c311SIan Lepore .icache_line_size = 32, 54a286c311SIan Lepore .icache_line_mask = 31, 55a286c311SIan Lepore }; 562b71562fSIan Lepore 577029da5cSPawel Biernacki static SYSCTL_NODE(_hw, OID_AUTO, cpu, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 587bf5720aSMichal Meloun "CPU"); 597029da5cSPawel Biernacki static SYSCTL_NODE(_hw_cpu, OID_AUTO, quirks, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 607bf5720aSMichal Meloun "CPU quirks"); 617bf5720aSMichal Meloun 627bf5720aSMichal Meloun /* 637bf5720aSMichal Meloun * Tunable CPU quirks. 647bf5720aSMichal Meloun * Be careful, ACTRL cannot be changed if CPU is started in secure 657bf5720aSMichal Meloun * mode(world) and write to ACTRL can cause exception! 667bf5720aSMichal Meloun * These quirks are intended for optimizing CPU performance, not for 677bf5720aSMichal Meloun * applying errata workarounds. Nobody can expect that CPU with unfixed 687bf5720aSMichal Meloun * errata is stable enough to execute the kernel until quirks are applied. 697bf5720aSMichal Meloun */ 707bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_mask; 717bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_mask, 727bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_mask, 0, 737bf5720aSMichal Meloun "Bits to be masked in ACTLR"); 747bf5720aSMichal Meloun 757bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_set; 767bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_set, 777bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_set, 0, 787bf5720aSMichal Meloun "Bits to be set in ACTLR"); 797bf5720aSMichal Meloun 8080b1995bSMateusz Guzik static int 8180b1995bSMateusz Guzik sysctl_hw_cpu_quirks_actrl_value(SYSCTL_HANDLER_ARGS) 8280b1995bSMateusz Guzik { 8380b1995bSMateusz Guzik uint32_t reg; 8480b1995bSMateusz Guzik 8580b1995bSMateusz Guzik reg = cp15_actlr_get(); 8680b1995bSMateusz Guzik return (SYSCTL_OUT(req, ®, sizeof(reg))); 8780b1995bSMateusz Guzik } 8880b1995bSMateusz Guzik SYSCTL_PROC(_hw_cpu_quirks, OID_AUTO, actlr_value, 8980b1995bSMateusz Guzik CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 9080b1995bSMateusz Guzik sysctl_hw_cpu_quirks_actrl_value, "IU", 9180b1995bSMateusz Guzik "Value of ACTLR"); 9280b1995bSMateusz Guzik 932b71562fSIan Lepore /* Read and parse CPU id scheme */ 942b71562fSIan Lepore void 952b71562fSIan Lepore cpuinfo_init(void) 962b71562fSIan Lepore { 970cbf724eSMichal Meloun uint32_t tmp; 982b71562fSIan Lepore 99c40a5f8aSMichal Meloun /* 100c40a5f8aSMichal Meloun * Prematurely fetch CPU quirks. Standard fetch for tunable 101c40a5f8aSMichal Meloun * sysctls is handled using SYSINIT, thus too late for boot CPU. 102c40a5f8aSMichal Meloun * Keep names in sync with sysctls. 103c40a5f8aSMichal Meloun */ 104c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_mask", &cpu_quirks_actlr_mask); 105c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_set", &cpu_quirks_actlr_set); 106c40a5f8aSMichal Meloun 1072b71562fSIan Lepore cpuinfo.midr = cp15_midr_get(); 1082b71562fSIan Lepore /* Test old version id schemes first */ 1092b71562fSIan Lepore if ((cpuinfo.midr & CPU_ID_IMPLEMENTOR_MASK) == CPU_ID_ARM_LTD) { 1102b71562fSIan Lepore if (CPU_ID_ISOLD(cpuinfo.midr)) { 1112b71562fSIan Lepore /* obsolete ARMv2 or ARMv3 CPU */ 1122b71562fSIan Lepore cpuinfo.midr = 0; 1132b71562fSIan Lepore return; 1142b71562fSIan Lepore } 1152b71562fSIan Lepore if (CPU_ID_IS7(cpuinfo.midr)) { 1162b71562fSIan Lepore if ((cpuinfo.midr & (1 << 23)) == 0) { 1172b71562fSIan Lepore /* obsolete ARMv3 CPU */ 1182b71562fSIan Lepore cpuinfo.midr = 0; 1192b71562fSIan Lepore return; 1202b71562fSIan Lepore } 1212b71562fSIan Lepore /* ARMv4T CPU */ 1222b71562fSIan Lepore cpuinfo.architecture = 1; 1232b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 16) & 0x7F; 1248e6dd301SIan Lepore } else { 1258e6dd301SIan Lepore /* ARM new id scheme */ 1268e6dd301SIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 1278e6dd301SIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 1282b71562fSIan Lepore } 1292b71562fSIan Lepore } else { 1308e6dd301SIan Lepore /* non ARM -> must be new id scheme */ 1312b71562fSIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 1322b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 1332b71562fSIan Lepore } 1342b71562fSIan Lepore /* Parse rest of MIDR */ 1352b71562fSIan Lepore cpuinfo.implementer = (cpuinfo.midr >> 24) & 0xFF; 1362b71562fSIan Lepore cpuinfo.part_number = (cpuinfo.midr >> 4) & 0xFFF; 1372b71562fSIan Lepore cpuinfo.patch = cpuinfo.midr & 0x0F; 1382b71562fSIan Lepore 1392b71562fSIan Lepore /* CP15 c0,c0 regs 0-7 exist on all CPUs (although aliased with MIDR) */ 1402b71562fSIan Lepore cpuinfo.ctr = cp15_ctr_get(); 1412b71562fSIan Lepore cpuinfo.tcmtr = cp15_tcmtr_get(); 1422b71562fSIan Lepore cpuinfo.tlbtr = cp15_tlbtr_get(); 1432b71562fSIan Lepore cpuinfo.mpidr = cp15_mpidr_get(); 1442b71562fSIan Lepore cpuinfo.revidr = cp15_revidr_get(); 1452b71562fSIan Lepore 1462b71562fSIan Lepore /* if CPU is not v7 cpu id scheme */ 1472b71562fSIan Lepore if (cpuinfo.architecture != 0xF) 1482b71562fSIan Lepore return; 1492b71562fSIan Lepore cpuinfo.id_pfr0 = cp15_id_pfr0_get(); 1502b71562fSIan Lepore cpuinfo.id_pfr1 = cp15_id_pfr1_get(); 1512b71562fSIan Lepore cpuinfo.id_dfr0 = cp15_id_dfr0_get(); 1522b71562fSIan Lepore cpuinfo.id_afr0 = cp15_id_afr0_get(); 1532b71562fSIan Lepore cpuinfo.id_mmfr0 = cp15_id_mmfr0_get(); 1542b71562fSIan Lepore cpuinfo.id_mmfr1 = cp15_id_mmfr1_get(); 1552b71562fSIan Lepore cpuinfo.id_mmfr2 = cp15_id_mmfr2_get(); 1562b71562fSIan Lepore cpuinfo.id_mmfr3 = cp15_id_mmfr3_get(); 1572b71562fSIan Lepore cpuinfo.id_isar0 = cp15_id_isar0_get(); 1582b71562fSIan Lepore cpuinfo.id_isar1 = cp15_id_isar1_get(); 1592b71562fSIan Lepore cpuinfo.id_isar2 = cp15_id_isar2_get(); 1602b71562fSIan Lepore cpuinfo.id_isar3 = cp15_id_isar3_get(); 1612b71562fSIan Lepore cpuinfo.id_isar4 = cp15_id_isar4_get(); 1622b71562fSIan Lepore cpuinfo.id_isar5 = cp15_id_isar5_get(); 1632b71562fSIan Lepore 1642b71562fSIan Lepore /* Not yet - CBAR only exist on ARM SMP Cortex A CPUs 1652b71562fSIan Lepore cpuinfo.cbar = cp15_cbar_get(); 1662b71562fSIan Lepore */ 167ba0bb206SMichal Meloun if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 168ba0bb206SMichal Meloun cpuinfo.ccsidr = cp15_ccsidr_get(); 169ba0bb206SMichal Meloun cpuinfo.clidr = cp15_clidr_get(); 170ba0bb206SMichal Meloun } 1712b71562fSIan Lepore 1722b71562fSIan Lepore /* Test if revidr is implemented */ 1732b71562fSIan Lepore if (cpuinfo.revidr == cpuinfo.midr) 1742b71562fSIan Lepore cpuinfo.revidr = 0; 1752b71562fSIan Lepore 1762b71562fSIan Lepore /* parsed bits of above registers */ 1772b71562fSIan Lepore /* id_mmfr0 */ 1782b71562fSIan Lepore cpuinfo.outermost_shareability = (cpuinfo.id_mmfr0 >> 8) & 0xF; 1792b71562fSIan Lepore cpuinfo.shareability_levels = (cpuinfo.id_mmfr0 >> 12) & 0xF; 1802b71562fSIan Lepore cpuinfo.auxiliary_registers = (cpuinfo.id_mmfr0 >> 20) & 0xF; 1812b71562fSIan Lepore cpuinfo.innermost_shareability = (cpuinfo.id_mmfr0 >> 28) & 0xF; 1822b71562fSIan Lepore /* id_mmfr2 */ 1832b71562fSIan Lepore cpuinfo.mem_barrier = (cpuinfo.id_mmfr2 >> 20) & 0xF; 1842b71562fSIan Lepore /* id_mmfr3 */ 1852b71562fSIan Lepore cpuinfo.coherent_walk = (cpuinfo.id_mmfr3 >> 20) & 0xF; 1862b71562fSIan Lepore cpuinfo.maintenance_broadcast =(cpuinfo.id_mmfr3 >> 12) & 0xF; 1872b71562fSIan Lepore /* id_pfr1 */ 1882b71562fSIan Lepore cpuinfo.generic_timer_ext = (cpuinfo.id_pfr1 >> 16) & 0xF; 1892b71562fSIan Lepore cpuinfo.virtualization_ext = (cpuinfo.id_pfr1 >> 12) & 0xF; 1902b71562fSIan Lepore cpuinfo.security_ext = (cpuinfo.id_pfr1 >> 4) & 0xF; 191d029cb61SAndrew Turner /* mpidr */ 192d029cb61SAndrew Turner cpuinfo.mp_ext = (cpuinfo.mpidr >> 31u) & 0x1; 193a286c311SIan Lepore 194a286c311SIan Lepore /* L1 Cache sizes */ 195a22f8196SIan Lepore if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 196a22f8196SIan Lepore cpuinfo.dcache_line_size = 197a22f8196SIan Lepore 1 << (CPU_CT_DMINLINE(cpuinfo.ctr) + 2); 198a22f8196SIan Lepore cpuinfo.icache_line_size = 199a22f8196SIan Lepore 1 << (CPU_CT_IMINLINE(cpuinfo.ctr) + 2); 200a22f8196SIan Lepore } else { 201a22f8196SIan Lepore cpuinfo.dcache_line_size = 202a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_DSIZE(cpuinfo.ctr)) + 3); 203a22f8196SIan Lepore cpuinfo.icache_line_size = 204a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_ISIZE(cpuinfo.ctr)) + 3); 205a22f8196SIan Lepore } 206a286c311SIan Lepore cpuinfo.dcache_line_mask = cpuinfo.dcache_line_size - 1; 207a286c311SIan Lepore cpuinfo.icache_line_mask = cpuinfo.icache_line_size - 1; 2080cbf724eSMichal Meloun 2090cbf724eSMichal Meloun /* Fill AT_HWCAP bits. */ 21032c48d07SMichal Meloun elf_hwcap |= HWCAP_HALF | HWCAP_FAST_MULT; /* Required for all CPUs */ 21132c48d07SMichal Meloun elf_hwcap |= HWCAP_TLS | HWCAP_EDSP; /* Required for v6+ CPUs */ 2120cbf724eSMichal Meloun 2130cbf724eSMichal Meloun tmp = (cpuinfo.id_isar0 >> 24) & 0xF; /* Divide_instrs */ 2140cbf724eSMichal Meloun if (tmp >= 1) 2150cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVT; 2160cbf724eSMichal Meloun if (tmp >= 2) 2170cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVA; 2180cbf724eSMichal Meloun 2190cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 4) & 0xF; /* State1 */ 2200cbf724eSMichal Meloun if (tmp >= 1) 2210cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMB; 2220cbf724eSMichal Meloun 2230cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 12) & 0xF; /* State3 */ 2240cbf724eSMichal Meloun if (tmp >= 1) 2250cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMBEE; 2260cbf724eSMichal Meloun 2270cbf724eSMichal Meloun tmp = (cpuinfo.id_mmfr0 >> 0) & 0xF; /* VMSA */ 2280cbf724eSMichal Meloun if (tmp >= 5) 2290cbf724eSMichal Meloun elf_hwcap |= HWCAP_LPAE; 2300cbf724eSMichal Meloun 2310cbf724eSMichal Meloun /* Fill AT_HWCAP2 bits. */ 2320cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 4) & 0xF; /* AES */ 2330cbf724eSMichal Meloun if (tmp >= 1) 2340cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_AES; 2350cbf724eSMichal Meloun if (tmp >= 2) 2360cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_PMULL; 2370cbf724eSMichal Meloun 2380cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 8) & 0xF; /* SHA1 */ 2390cbf724eSMichal Meloun if (tmp >= 1) 2400cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA1; 2410cbf724eSMichal Meloun 2420cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 12) & 0xF; /* SHA2 */ 2430cbf724eSMichal Meloun if (tmp >= 1) 2440cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA2; 2450cbf724eSMichal Meloun 2460cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 16) & 0xF; /* CRC32 */ 2470cbf724eSMichal Meloun if (tmp >= 1) 2480cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_CRC32; 2492b71562fSIan Lepore } 250935c21a1SIan Lepore 251935c21a1SIan Lepore /* 252935c21a1SIan Lepore * Get bits that must be set or cleared in ACLR register. 253935c21a1SIan Lepore * Note: Bits in ACLR register are IMPLEMENTATION DEFINED. 254935c21a1SIan Lepore * Its expected that SCU is in operational state before this 255935c21a1SIan Lepore * function is called. 256935c21a1SIan Lepore */ 2577bf5720aSMichal Meloun static void 258935c21a1SIan Lepore cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set) 259935c21a1SIan Lepore { 2607bf5720aSMichal Meloun 261935c21a1SIan Lepore *actlr_mask = 0; 262935c21a1SIan Lepore *actlr_set = 0; 263935c21a1SIan Lepore 264935c21a1SIan Lepore if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 265935c21a1SIan Lepore switch (cpuinfo.part_number) { 266a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75: 267ba0bb206SMichal Meloun case CPU_ARCH_CORTEX_A73: 26855e447c9SMichal Meloun case CPU_ARCH_CORTEX_A72: 26955e447c9SMichal Meloun case CPU_ARCH_CORTEX_A57: 27055e447c9SMichal Meloun case CPU_ARCH_CORTEX_A53: 27155e447c9SMichal Meloun /* Nothing to do for AArch32 */ 27255e447c9SMichal Meloun break; 273935c21a1SIan Lepore case CPU_ARCH_CORTEX_A17: 274935c21a1SIan Lepore case CPU_ARCH_CORTEX_A12: /* A12 is merged to A17 */ 275935c21a1SIan Lepore /* 276935c21a1SIan Lepore * Enable SMP mode 277935c21a1SIan Lepore */ 278935c21a1SIan Lepore *actlr_mask = (1 << 6); 279935c21a1SIan Lepore *actlr_set = (1 << 6); 280935c21a1SIan Lepore break; 281935c21a1SIan Lepore case CPU_ARCH_CORTEX_A15: 282935c21a1SIan Lepore /* 283935c21a1SIan Lepore * Enable snoop-delayed exclusive handling 284935c21a1SIan Lepore * Enable SMP mode 285935c21a1SIan Lepore */ 286935c21a1SIan Lepore *actlr_mask = (1U << 31) |(1 << 6); 287935c21a1SIan Lepore *actlr_set = (1U << 31) |(1 << 6); 288935c21a1SIan Lepore break; 289935c21a1SIan Lepore case CPU_ARCH_CORTEX_A9: 290935c21a1SIan Lepore /* 291935c21a1SIan Lepore * Disable exclusive L1/L2 cache control 292935c21a1SIan Lepore * Enable SMP mode 293935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast 294935c21a1SIan Lepore */ 295935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 296935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0); 297935c21a1SIan Lepore break; 298935c21a1SIan Lepore case CPU_ARCH_CORTEX_A8: 299935c21a1SIan Lepore /* 300935c21a1SIan Lepore * Enable L2 cache 301935c21a1SIan Lepore * Enable L1 data cache hardware alias checks 302935c21a1SIan Lepore */ 303935c21a1SIan Lepore *actlr_mask = (1 << 1) | (1 << 0); 304935c21a1SIan Lepore *actlr_set = (1 << 1); 305935c21a1SIan Lepore break; 306935c21a1SIan Lepore case CPU_ARCH_CORTEX_A7: 307935c21a1SIan Lepore /* 308935c21a1SIan Lepore * Enable SMP mode 309935c21a1SIan Lepore */ 310935c21a1SIan Lepore *actlr_mask = (1 << 6); 311935c21a1SIan Lepore *actlr_set = (1 << 6); 312935c21a1SIan Lepore break; 313935c21a1SIan Lepore case CPU_ARCH_CORTEX_A5: 314935c21a1SIan Lepore /* 315935c21a1SIan Lepore * Disable exclusive L1/L2 cache control 316935c21a1SIan Lepore * Enable SMP mode 317935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast 318935c21a1SIan Lepore */ 319935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 320935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0); 321935c21a1SIan Lepore break; 322935c21a1SIan Lepore case CPU_ARCH_ARM1176: 323935c21a1SIan Lepore /* 324935c21a1SIan Lepore * Restrict cache size to 16KB 325935c21a1SIan Lepore * Enable the return stack 326935c21a1SIan Lepore * Enable dynamic branch prediction 327935c21a1SIan Lepore * Enable static branch prediction 328935c21a1SIan Lepore */ 329935c21a1SIan Lepore *actlr_mask = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 330935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 331935c21a1SIan Lepore break; 332935c21a1SIan Lepore } 333935c21a1SIan Lepore return; 334935c21a1SIan Lepore } 335935c21a1SIan Lepore } 3367bf5720aSMichal Meloun 3377bf5720aSMichal Meloun /* Reinitialize MMU to final kernel mapping and apply all CPU quirks. */ 3387bf5720aSMichal Meloun void 3397bf5720aSMichal Meloun cpuinfo_reinit_mmu(uint32_t ttb) 3407bf5720aSMichal Meloun { 3417bf5720aSMichal Meloun uint32_t actlr_mask; 3427bf5720aSMichal Meloun uint32_t actlr_set; 3437bf5720aSMichal Meloun 3447bf5720aSMichal Meloun cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); 3457bf5720aSMichal Meloun actlr_mask |= cpu_quirks_actlr_mask; 3467bf5720aSMichal Meloun actlr_set |= cpu_quirks_actlr_set; 3477bf5720aSMichal Meloun reinit_mmu(ttb, actlr_mask, actlr_set); 3487bf5720aSMichal Meloun } 3497bf5720aSMichal Meloun 350a36b6ec0SMichal Meloun static bool 351a36b6ec0SMichal Meloun modify_actlr(uint32_t clear, uint32_t set) 352a36b6ec0SMichal Meloun { 353a36b6ec0SMichal Meloun uint32_t reg, newreg; 354a36b6ec0SMichal Meloun 355a36b6ec0SMichal Meloun reg = cp15_actlr_get(); 356a36b6ec0SMichal Meloun newreg = reg; 357a36b6ec0SMichal Meloun newreg &= ~clear; 358a36b6ec0SMichal Meloun newreg |= set; 359a36b6ec0SMichal Meloun if (reg == newreg) 360a36b6ec0SMichal Meloun return (true); 361a36b6ec0SMichal Meloun cp15_actlr_set(newreg); 362a36b6ec0SMichal Meloun 363a36b6ec0SMichal Meloun reg = cp15_actlr_get(); 364a36b6ec0SMichal Meloun if (reg == newreg) 365a36b6ec0SMichal Meloun return (true); 366a36b6ec0SMichal Meloun return (false); 367a36b6ec0SMichal Meloun } 368a36b6ec0SMichal Meloun 369a36b6ec0SMichal Meloun /* Apply/restore BP hardening on current core. */ 370a36b6ec0SMichal Meloun static int 371a36b6ec0SMichal Meloun apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask) 372a36b6ec0SMichal Meloun { 373a36b6ec0SMichal Meloun if (enable) { 374a36b6ec0SMichal Meloun if (actrl && !modify_actlr(0, set_mask)) 375a36b6ec0SMichal Meloun return (-1); 376a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, kind); 377a36b6ec0SMichal Meloun } else { 378a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 379a36b6ec0SMichal Meloun if (actrl) 380a36b6ec0SMichal Meloun modify_actlr(~0, PCPU_GET(original_actlr)); 381a36b6ec0SMichal Meloun spectre_v2_safe = 0; 382a36b6ec0SMichal Meloun } 383a36b6ec0SMichal Meloun return (0); 384a36b6ec0SMichal Meloun } 385a36b6ec0SMichal Meloun 386a36b6ec0SMichal Meloun static void 387a36b6ec0SMichal Meloun handle_bp_hardening(bool enable) 388a36b6ec0SMichal Meloun { 389a36b6ec0SMichal Meloun int kind; 390a36b6ec0SMichal Meloun char *kind_str; 391a36b6ec0SMichal Meloun 392a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_NONE; 393a36b6ec0SMichal Meloun /* 394a36b6ec0SMichal Meloun * Note: Access to ACTRL is locked to secure world on most boards. 395a36b6ec0SMichal Meloun * This means that full BP hardening depends on updated u-boot/firmware 396a36b6ec0SMichal Meloun * or is impossible at all (if secure monitor is in on-chip ROM). 397a36b6ec0SMichal Meloun */ 398a36b6ec0SMichal Meloun if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 399a36b6ec0SMichal Meloun switch (cpuinfo.part_number) { 400a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A8: 401a36b6ec0SMichal Meloun /* 402a36b6ec0SMichal Meloun * For Cortex-A8, IBE bit must be set otherwise 403a36b6ec0SMichal Meloun * BPIALL is effectively NOP. 404a36b6ec0SMichal Meloun * Unfortunately, Cortex-A is also affected by 405a36b6ec0SMichal Meloun * ARM erratum 687067 which causes non-working 406a36b6ec0SMichal Meloun * BPIALL if IBE bit is set and 'Instruction L1 System 407a36b6ec0SMichal Meloun * Array Debug Register 0' is not 0. 408a36b6ec0SMichal Meloun * This register is not reset on power-up and is 409a36b6ec0SMichal Meloun * accessible only from secure world, so we cannot do 410a36b6ec0SMichal Meloun * nothing (nor detect) to fix this issue. 411a36b6ec0SMichal Meloun * I afraid that on chip ROM based secure monitor on 412a36b6ec0SMichal Meloun * AM335x (BeagleBone) doesn't reset this debug 413a36b6ec0SMichal Meloun * register. 414a36b6ec0SMichal Meloun */ 415a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL; 416a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0) 417a36b6ec0SMichal Meloun goto actlr_err; 418a36b6ec0SMichal Meloun break; 419a36b6ec0SMichal Meloun break; 420a36b6ec0SMichal Meloun 421a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A9: 422a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A12: 423a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A17: 424a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A57: 425a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A72: 426a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A73: 427a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75: 428a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL; 429a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, false, 0) != 0) 430a36b6ec0SMichal Meloun goto actlr_err; 431a36b6ec0SMichal Meloun break; 432a36b6ec0SMichal Meloun 433a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A15: 434a36b6ec0SMichal Meloun /* 435a36b6ec0SMichal Meloun * For Cortex-A15, set 'Enable invalidates of BTB' bit. 436a36b6ec0SMichal Meloun * Despite this, the BPIALL is still effectively NOP, 437a36b6ec0SMichal Meloun * but with this bit set, the ICIALLU also flushes 438a36b6ec0SMichal Meloun * branch predictor as side effect. 439a36b6ec0SMichal Meloun */ 440a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_ICIALLU; 441a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0) 442a36b6ec0SMichal Meloun goto actlr_err; 443a36b6ec0SMichal Meloun break; 444a36b6ec0SMichal Meloun 445a36b6ec0SMichal Meloun default: 446a36b6ec0SMichal Meloun break; 447a36b6ec0SMichal Meloun } 448a36b6ec0SMichal Meloun } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) { 449a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative " 450a36b6ec0SMichal Meloun "branch attacks. !!!\n" 451a36b6ec0SMichal Meloun "Qualcomm Krait cores are known (or believed) to be " 452a36b6ec0SMichal Meloun "vulnerable to \n" 453a36b6ec0SMichal Meloun "speculative branch attacks, no mitigation exists yet.\n", 454a36b6ec0SMichal Meloun PCPU_GET(cpuid)); 455a36b6ec0SMichal Meloun goto unkonown_mitigation; 456a36b6ec0SMichal Meloun } else { 457a36b6ec0SMichal Meloun goto unkonown_mitigation; 458a36b6ec0SMichal Meloun } 459a36b6ec0SMichal Meloun 460a36b6ec0SMichal Meloun if (bootverbose) { 461a36b6ec0SMichal Meloun switch (kind) { 462a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_NONE: 463a36b6ec0SMichal Meloun kind_str = "not necessary"; 464a36b6ec0SMichal Meloun break; 465a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_BPIALL: 466a36b6ec0SMichal Meloun kind_str = "BPIALL"; 467a36b6ec0SMichal Meloun break; 468a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_ICIALLU: 469a36b6ec0SMichal Meloun kind_str = "ICIALLU"; 470a36b6ec0SMichal Meloun break; 471a36b6ec0SMichal Meloun default: 472a36b6ec0SMichal Meloun panic("Unknown BP hardering kind (%d).", kind); 473a36b6ec0SMichal Meloun } 474a36b6ec0SMichal Meloun printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid), 475a36b6ec0SMichal Meloun kind_str); 476a36b6ec0SMichal Meloun } 477a36b6ec0SMichal Meloun 478a36b6ec0SMichal Meloun return; 479a36b6ec0SMichal Meloun 480a36b6ec0SMichal Meloun unkonown_mitigation: 481a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 482a36b6ec0SMichal Meloun spectre_v2_safe = 0; 483a36b6ec0SMichal Meloun return; 484a36b6ec0SMichal Meloun 485a36b6ec0SMichal Meloun actlr_err: 486a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 487a36b6ec0SMichal Meloun spectre_v2_safe = 0; 488a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch " 489a36b6ec0SMichal Meloun "attacks. !!!\n" 490a36b6ec0SMichal Meloun "We cannot enable required bit(s) in ACTRL register\n" 491a36b6ec0SMichal Meloun "because it's locked by secure monitor and/or firmware.\n", 492a36b6ec0SMichal Meloun PCPU_GET(cpuid)); 493a36b6ec0SMichal Meloun } 494a36b6ec0SMichal Meloun 495a36b6ec0SMichal Meloun void 496a36b6ec0SMichal Meloun cpuinfo_init_bp_hardening(void) 497a36b6ec0SMichal Meloun { 498a36b6ec0SMichal Meloun 499a36b6ec0SMichal Meloun /* 500a36b6ec0SMichal Meloun * Store original unmodified ACTRL, so we can restore it when 501a36b6ec0SMichal Meloun * BP hardening is disabled by sysctl. 502a36b6ec0SMichal Meloun */ 503a36b6ec0SMichal Meloun PCPU_SET(original_actlr, cp15_actlr_get()); 504a36b6ec0SMichal Meloun handle_bp_hardening(true); 505a36b6ec0SMichal Meloun } 506a36b6ec0SMichal Meloun 507a36b6ec0SMichal Meloun static void 508a36b6ec0SMichal Meloun bp_hardening_action(void *arg) 509a36b6ec0SMichal Meloun { 510a36b6ec0SMichal Meloun 511a36b6ec0SMichal Meloun handle_bp_hardening(disable_bp_hardening == 0); 512a36b6ec0SMichal Meloun } 513a36b6ec0SMichal Meloun 514a36b6ec0SMichal Meloun static int 515a36b6ec0SMichal Meloun sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS) 516a36b6ec0SMichal Meloun { 517a36b6ec0SMichal Meloun int rv; 518a36b6ec0SMichal Meloun 519a36b6ec0SMichal Meloun rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 520a36b6ec0SMichal Meloun 521a36b6ec0SMichal Meloun if (!rv && req->newptr) { 522a36b6ec0SMichal Meloun spectre_v2_safe = 1; 523a36b6ec0SMichal Meloun dmb(); 524a36b6ec0SMichal Meloun #ifdef SMP 525a36b6ec0SMichal Meloun smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, 526a36b6ec0SMichal Meloun bp_hardening_action, NULL, NULL); 527a36b6ec0SMichal Meloun #else 528a36b6ec0SMichal Meloun bp_hardening_action(NULL); 529a36b6ec0SMichal Meloun #endif 530a36b6ec0SMichal Meloun } 531a36b6ec0SMichal Meloun 532a36b6ec0SMichal Meloun return (rv); 533a36b6ec0SMichal Meloun } 534a36b6ec0SMichal Meloun 535a36b6ec0SMichal Meloun SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening, 536a36b6ec0SMichal Meloun CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 537a36b6ec0SMichal Meloun &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I", 538a36b6ec0SMichal Meloun "Disable BP hardening mitigation."); 539a36b6ec0SMichal Meloun 540a36b6ec0SMichal Meloun SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD, 541a36b6ec0SMichal Meloun &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks"); 542