12b71562fSIan Lepore /*- 22b71562fSIan Lepore * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com> 32b71562fSIan Lepore * Copyright 2014 Michal Meloun <meloun@miracle.cz> 42b71562fSIan Lepore * All rights reserved. 52b71562fSIan Lepore * 62b71562fSIan Lepore * Redistribution and use in source and binary forms, with or without 72b71562fSIan Lepore * modification, are permitted provided that the following conditions 82b71562fSIan Lepore * are met: 92b71562fSIan Lepore * 1. Redistributions of source code must retain the above copyright 102b71562fSIan Lepore * notice, this list of conditions and the following disclaimer. 112b71562fSIan Lepore * 2. Redistributions in binary form must reproduce the above copyright 122b71562fSIan Lepore * notice, this list of conditions and the following disclaimer in the 132b71562fSIan Lepore * documentation and/or other materials provided with the distribution. 142b71562fSIan Lepore * 152b71562fSIan Lepore * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 162b71562fSIan Lepore * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 172b71562fSIan Lepore * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 182b71562fSIan Lepore * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 192b71562fSIan Lepore * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 202b71562fSIan Lepore * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 212b71562fSIan Lepore * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 222b71562fSIan Lepore * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 232b71562fSIan Lepore * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 242b71562fSIan Lepore * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 252b71562fSIan Lepore * SUCH DAMAGE. 262b71562fSIan Lepore */ 272b71562fSIan Lepore 282b71562fSIan Lepore #include <sys/cdefs.h> 292b71562fSIan Lepore __FBSDID("$FreeBSD$"); 302b71562fSIan Lepore 312b71562fSIan Lepore #include <sys/param.h> 322b71562fSIan Lepore #include <sys/systm.h> 33c40a5f8aSMichal Meloun #include <sys/kernel.h> 34a36b6ec0SMichal Meloun #include <sys/pcpu.h> 35a36b6ec0SMichal Meloun #include <sys/smp.h> 367bf5720aSMichal Meloun #include <sys/sysctl.h> 372b71562fSIan Lepore 383025d19dSMichal Meloun #include <machine/cpu.h> 392b71562fSIan Lepore #include <machine/cpuinfo.h> 400cbf724eSMichal Meloun #include <machine/elf.h> 410cbf724eSMichal Meloun #include <machine/md_var.h> 422b71562fSIan Lepore 437bf5720aSMichal Meloun #if __ARM_ARCH >= 6 447bf5720aSMichal Meloun void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set); 45a36b6ec0SMichal Meloun 46a36b6ec0SMichal Meloun int disable_bp_hardening; 47a36b6ec0SMichal Meloun int spectre_v2_safe = 1; 487bf5720aSMichal Meloun #endif 497bf5720aSMichal Meloun 50a286c311SIan Lepore struct cpuinfo cpuinfo = 51a286c311SIan Lepore { 52a286c311SIan Lepore /* Use safe defaults for start */ 53a286c311SIan Lepore .dcache_line_size = 32, 54a286c311SIan Lepore .dcache_line_mask = 31, 55a286c311SIan Lepore .icache_line_size = 32, 56a286c311SIan Lepore .icache_line_mask = 31, 57a286c311SIan Lepore }; 582b71562fSIan Lepore 597bf5720aSMichal Meloun static SYSCTL_NODE(_hw, OID_AUTO, cpu, CTLFLAG_RD, 0, 607bf5720aSMichal Meloun "CPU"); 617bf5720aSMichal Meloun static SYSCTL_NODE(_hw_cpu, OID_AUTO, quirks, CTLFLAG_RD, 0, 627bf5720aSMichal Meloun "CPU quirks"); 637bf5720aSMichal Meloun 647bf5720aSMichal Meloun /* 657bf5720aSMichal Meloun * Tunable CPU quirks. 667bf5720aSMichal Meloun * Be careful, ACTRL cannot be changed if CPU is started in secure 677bf5720aSMichal Meloun * mode(world) and write to ACTRL can cause exception! 687bf5720aSMichal Meloun * These quirks are intended for optimizing CPU performance, not for 697bf5720aSMichal Meloun * applying errata workarounds. Nobody can expect that CPU with unfixed 707bf5720aSMichal Meloun * errata is stable enough to execute the kernel until quirks are applied. 717bf5720aSMichal Meloun */ 727bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_mask; 737bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_mask, 747bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_mask, 0, 757bf5720aSMichal Meloun "Bits to be masked in ACTLR"); 767bf5720aSMichal Meloun 777bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_set; 787bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_set, 797bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_set, 0, 807bf5720aSMichal Meloun "Bits to be set in ACTLR"); 817bf5720aSMichal Meloun 827bf5720aSMichal Meloun 832b71562fSIan Lepore /* Read and parse CPU id scheme */ 842b71562fSIan Lepore void 852b71562fSIan Lepore cpuinfo_init(void) 862b71562fSIan Lepore { 870cbf724eSMichal Meloun #if __ARM_ARCH >= 6 880cbf724eSMichal Meloun uint32_t tmp; 890cbf724eSMichal Meloun #endif 902b71562fSIan Lepore 91c40a5f8aSMichal Meloun /* 92c40a5f8aSMichal Meloun * Prematurely fetch CPU quirks. Standard fetch for tunable 93c40a5f8aSMichal Meloun * sysctls is handled using SYSINIT, thus too late for boot CPU. 94c40a5f8aSMichal Meloun * Keep names in sync with sysctls. 95c40a5f8aSMichal Meloun */ 96c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_mask", &cpu_quirks_actlr_mask); 97c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_set", &cpu_quirks_actlr_set); 98c40a5f8aSMichal Meloun 992b71562fSIan Lepore cpuinfo.midr = cp15_midr_get(); 1002b71562fSIan Lepore /* Test old version id schemes first */ 1012b71562fSIan Lepore if ((cpuinfo.midr & CPU_ID_IMPLEMENTOR_MASK) == CPU_ID_ARM_LTD) { 1022b71562fSIan Lepore if (CPU_ID_ISOLD(cpuinfo.midr)) { 1032b71562fSIan Lepore /* obsolete ARMv2 or ARMv3 CPU */ 1042b71562fSIan Lepore cpuinfo.midr = 0; 1052b71562fSIan Lepore return; 1062b71562fSIan Lepore } 1072b71562fSIan Lepore if (CPU_ID_IS7(cpuinfo.midr)) { 1082b71562fSIan Lepore if ((cpuinfo.midr & (1 << 23)) == 0) { 1092b71562fSIan Lepore /* obsolete ARMv3 CPU */ 1102b71562fSIan Lepore cpuinfo.midr = 0; 1112b71562fSIan Lepore return; 1122b71562fSIan Lepore } 1132b71562fSIan Lepore /* ARMv4T CPU */ 1142b71562fSIan Lepore cpuinfo.architecture = 1; 1152b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 16) & 0x7F; 1168e6dd301SIan Lepore } else { 1178e6dd301SIan Lepore /* ARM new id scheme */ 1188e6dd301SIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 1198e6dd301SIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 1202b71562fSIan Lepore } 1212b71562fSIan Lepore } else { 1228e6dd301SIan Lepore /* non ARM -> must be new id scheme */ 1232b71562fSIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F; 1242b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F; 1252b71562fSIan Lepore } 1262b71562fSIan Lepore /* Parse rest of MIDR */ 1272b71562fSIan Lepore cpuinfo.implementer = (cpuinfo.midr >> 24) & 0xFF; 1282b71562fSIan Lepore cpuinfo.part_number = (cpuinfo.midr >> 4) & 0xFFF; 1292b71562fSIan Lepore cpuinfo.patch = cpuinfo.midr & 0x0F; 1302b71562fSIan Lepore 1312b71562fSIan Lepore /* CP15 c0,c0 regs 0-7 exist on all CPUs (although aliased with MIDR) */ 1322b71562fSIan Lepore cpuinfo.ctr = cp15_ctr_get(); 1332b71562fSIan Lepore cpuinfo.tcmtr = cp15_tcmtr_get(); 1343025d19dSMichal Meloun #if __ARM_ARCH >= 6 1352b71562fSIan Lepore cpuinfo.tlbtr = cp15_tlbtr_get(); 1362b71562fSIan Lepore cpuinfo.mpidr = cp15_mpidr_get(); 1372b71562fSIan Lepore cpuinfo.revidr = cp15_revidr_get(); 1383025d19dSMichal Meloun #endif 1392b71562fSIan Lepore 1402b71562fSIan Lepore /* if CPU is not v7 cpu id scheme */ 1412b71562fSIan Lepore if (cpuinfo.architecture != 0xF) 1422b71562fSIan Lepore return; 1433025d19dSMichal Meloun #if __ARM_ARCH >= 6 1442b71562fSIan Lepore cpuinfo.id_pfr0 = cp15_id_pfr0_get(); 1452b71562fSIan Lepore cpuinfo.id_pfr1 = cp15_id_pfr1_get(); 1462b71562fSIan Lepore cpuinfo.id_dfr0 = cp15_id_dfr0_get(); 1472b71562fSIan Lepore cpuinfo.id_afr0 = cp15_id_afr0_get(); 1482b71562fSIan Lepore cpuinfo.id_mmfr0 = cp15_id_mmfr0_get(); 1492b71562fSIan Lepore cpuinfo.id_mmfr1 = cp15_id_mmfr1_get(); 1502b71562fSIan Lepore cpuinfo.id_mmfr2 = cp15_id_mmfr2_get(); 1512b71562fSIan Lepore cpuinfo.id_mmfr3 = cp15_id_mmfr3_get(); 1522b71562fSIan Lepore cpuinfo.id_isar0 = cp15_id_isar0_get(); 1532b71562fSIan Lepore cpuinfo.id_isar1 = cp15_id_isar1_get(); 1542b71562fSIan Lepore cpuinfo.id_isar2 = cp15_id_isar2_get(); 1552b71562fSIan Lepore cpuinfo.id_isar3 = cp15_id_isar3_get(); 1562b71562fSIan Lepore cpuinfo.id_isar4 = cp15_id_isar4_get(); 1572b71562fSIan Lepore cpuinfo.id_isar5 = cp15_id_isar5_get(); 1582b71562fSIan Lepore 1592b71562fSIan Lepore /* Not yet - CBAR only exist on ARM SMP Cortex A CPUs 1602b71562fSIan Lepore cpuinfo.cbar = cp15_cbar_get(); 1612b71562fSIan Lepore */ 162ba0bb206SMichal Meloun if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 163ba0bb206SMichal Meloun cpuinfo.ccsidr = cp15_ccsidr_get(); 164ba0bb206SMichal Meloun cpuinfo.clidr = cp15_clidr_get(); 165ba0bb206SMichal Meloun } 1662b71562fSIan Lepore 1672b71562fSIan Lepore /* Test if revidr is implemented */ 1682b71562fSIan Lepore if (cpuinfo.revidr == cpuinfo.midr) 1692b71562fSIan Lepore cpuinfo.revidr = 0; 1702b71562fSIan Lepore 1712b71562fSIan Lepore /* parsed bits of above registers */ 1722b71562fSIan Lepore /* id_mmfr0 */ 1732b71562fSIan Lepore cpuinfo.outermost_shareability = (cpuinfo.id_mmfr0 >> 8) & 0xF; 1742b71562fSIan Lepore cpuinfo.shareability_levels = (cpuinfo.id_mmfr0 >> 12) & 0xF; 1752b71562fSIan Lepore cpuinfo.auxiliary_registers = (cpuinfo.id_mmfr0 >> 20) & 0xF; 1762b71562fSIan Lepore cpuinfo.innermost_shareability = (cpuinfo.id_mmfr0 >> 28) & 0xF; 1772b71562fSIan Lepore /* id_mmfr2 */ 1782b71562fSIan Lepore cpuinfo.mem_barrier = (cpuinfo.id_mmfr2 >> 20) & 0xF; 1792b71562fSIan Lepore /* id_mmfr3 */ 1802b71562fSIan Lepore cpuinfo.coherent_walk = (cpuinfo.id_mmfr3 >> 20) & 0xF; 1812b71562fSIan Lepore cpuinfo.maintenance_broadcast =(cpuinfo.id_mmfr3 >> 12) & 0xF; 1822b71562fSIan Lepore /* id_pfr1 */ 1832b71562fSIan Lepore cpuinfo.generic_timer_ext = (cpuinfo.id_pfr1 >> 16) & 0xF; 1842b71562fSIan Lepore cpuinfo.virtualization_ext = (cpuinfo.id_pfr1 >> 12) & 0xF; 1852b71562fSIan Lepore cpuinfo.security_ext = (cpuinfo.id_pfr1 >> 4) & 0xF; 186d029cb61SAndrew Turner /* mpidr */ 187d029cb61SAndrew Turner cpuinfo.mp_ext = (cpuinfo.mpidr >> 31u) & 0x1; 188a286c311SIan Lepore 189a286c311SIan Lepore /* L1 Cache sizes */ 190a22f8196SIan Lepore if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) { 191a22f8196SIan Lepore cpuinfo.dcache_line_size = 192a22f8196SIan Lepore 1 << (CPU_CT_DMINLINE(cpuinfo.ctr) + 2); 193a22f8196SIan Lepore cpuinfo.icache_line_size = 194a22f8196SIan Lepore 1 << (CPU_CT_IMINLINE(cpuinfo.ctr) + 2); 195a22f8196SIan Lepore } else { 196a22f8196SIan Lepore cpuinfo.dcache_line_size = 197a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_DSIZE(cpuinfo.ctr)) + 3); 198a22f8196SIan Lepore cpuinfo.icache_line_size = 199a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_ISIZE(cpuinfo.ctr)) + 3); 200a22f8196SIan Lepore } 201a286c311SIan Lepore cpuinfo.dcache_line_mask = cpuinfo.dcache_line_size - 1; 202a286c311SIan Lepore cpuinfo.icache_line_mask = cpuinfo.icache_line_size - 1; 2030cbf724eSMichal Meloun 2040cbf724eSMichal Meloun /* Fill AT_HWCAP bits. */ 20532c48d07SMichal Meloun elf_hwcap |= HWCAP_HALF | HWCAP_FAST_MULT; /* Required for all CPUs */ 20632c48d07SMichal Meloun elf_hwcap |= HWCAP_TLS | HWCAP_EDSP; /* Required for v6+ CPUs */ 2070cbf724eSMichal Meloun 2080cbf724eSMichal Meloun tmp = (cpuinfo.id_isar0 >> 24) & 0xF; /* Divide_instrs */ 2090cbf724eSMichal Meloun if (tmp >= 1) 2100cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVT; 2110cbf724eSMichal Meloun if (tmp >= 2) 2120cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVA; 2130cbf724eSMichal Meloun 2140cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 4) & 0xF; /* State1 */ 2150cbf724eSMichal Meloun if (tmp >= 1) 2160cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMB; 2170cbf724eSMichal Meloun 2180cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 12) & 0xF; /* State3 */ 2190cbf724eSMichal Meloun if (tmp >= 1) 2200cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMBEE; 2210cbf724eSMichal Meloun 2220cbf724eSMichal Meloun tmp = (cpuinfo.id_mmfr0 >> 0) & 0xF; /* VMSA */ 2230cbf724eSMichal Meloun if (tmp >= 5) 2240cbf724eSMichal Meloun elf_hwcap |= HWCAP_LPAE; 2250cbf724eSMichal Meloun 2260cbf724eSMichal Meloun /* Fill AT_HWCAP2 bits. */ 2270cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 4) & 0xF; /* AES */ 2280cbf724eSMichal Meloun if (tmp >= 1) 2290cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_AES; 2300cbf724eSMichal Meloun if (tmp >= 2) 2310cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_PMULL; 2320cbf724eSMichal Meloun 2330cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 8) & 0xF; /* SHA1 */ 2340cbf724eSMichal Meloun if (tmp >= 1) 2350cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA1; 2360cbf724eSMichal Meloun 2370cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 12) & 0xF; /* SHA2 */ 2380cbf724eSMichal Meloun if (tmp >= 1) 2390cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA2; 2400cbf724eSMichal Meloun 2410cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 16) & 0xF; /* CRC32 */ 2420cbf724eSMichal Meloun if (tmp >= 1) 2430cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_CRC32; 2443025d19dSMichal Meloun #endif 2452b71562fSIan Lepore } 246935c21a1SIan Lepore 2477bf5720aSMichal Meloun #if __ARM_ARCH >= 6 248935c21a1SIan Lepore /* 249935c21a1SIan Lepore * Get bits that must be set or cleared in ACLR register. 250935c21a1SIan Lepore * Note: Bits in ACLR register are IMPLEMENTATION DEFINED. 251935c21a1SIan Lepore * Its expected that SCU is in operational state before this 252935c21a1SIan Lepore * function is called. 253935c21a1SIan Lepore */ 2547bf5720aSMichal Meloun static void 255935c21a1SIan Lepore cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set) 256935c21a1SIan Lepore { 2577bf5720aSMichal Meloun 258935c21a1SIan Lepore *actlr_mask = 0; 259935c21a1SIan Lepore *actlr_set = 0; 260935c21a1SIan Lepore 261935c21a1SIan Lepore if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 262935c21a1SIan Lepore switch (cpuinfo.part_number) { 263a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75: 264ba0bb206SMichal Meloun case CPU_ARCH_CORTEX_A73: 26555e447c9SMichal Meloun case CPU_ARCH_CORTEX_A72: 26655e447c9SMichal Meloun case CPU_ARCH_CORTEX_A57: 26755e447c9SMichal Meloun case CPU_ARCH_CORTEX_A53: 26855e447c9SMichal Meloun /* Nothing to do for AArch32 */ 26955e447c9SMichal Meloun break; 270935c21a1SIan Lepore case CPU_ARCH_CORTEX_A17: 271935c21a1SIan Lepore case CPU_ARCH_CORTEX_A12: /* A12 is merged to A17 */ 272935c21a1SIan Lepore /* 273935c21a1SIan Lepore * Enable SMP mode 274935c21a1SIan Lepore */ 275935c21a1SIan Lepore *actlr_mask = (1 << 6); 276935c21a1SIan Lepore *actlr_set = (1 << 6); 277935c21a1SIan Lepore break; 278935c21a1SIan Lepore case CPU_ARCH_CORTEX_A15: 279935c21a1SIan Lepore /* 280935c21a1SIan Lepore * Enable snoop-delayed exclusive handling 281935c21a1SIan Lepore * Enable SMP mode 282935c21a1SIan Lepore */ 283935c21a1SIan Lepore *actlr_mask = (1U << 31) |(1 << 6); 284935c21a1SIan Lepore *actlr_set = (1U << 31) |(1 << 6); 285935c21a1SIan Lepore break; 286935c21a1SIan Lepore case CPU_ARCH_CORTEX_A9: 287935c21a1SIan Lepore /* 288935c21a1SIan Lepore * Disable exclusive L1/L2 cache control 289935c21a1SIan Lepore * Enable SMP mode 290935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast 291935c21a1SIan Lepore */ 292935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 293935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0); 294935c21a1SIan Lepore break; 295935c21a1SIan Lepore case CPU_ARCH_CORTEX_A8: 296935c21a1SIan Lepore /* 297935c21a1SIan Lepore * Enable L2 cache 298935c21a1SIan Lepore * Enable L1 data cache hardware alias checks 299935c21a1SIan Lepore */ 300935c21a1SIan Lepore *actlr_mask = (1 << 1) | (1 << 0); 301935c21a1SIan Lepore *actlr_set = (1 << 1); 302935c21a1SIan Lepore break; 303935c21a1SIan Lepore case CPU_ARCH_CORTEX_A7: 304935c21a1SIan Lepore /* 305935c21a1SIan Lepore * Enable SMP mode 306935c21a1SIan Lepore */ 307935c21a1SIan Lepore *actlr_mask = (1 << 6); 308935c21a1SIan Lepore *actlr_set = (1 << 6); 309935c21a1SIan Lepore break; 310935c21a1SIan Lepore case CPU_ARCH_CORTEX_A5: 311935c21a1SIan Lepore /* 312935c21a1SIan Lepore * Disable exclusive L1/L2 cache control 313935c21a1SIan Lepore * Enable SMP mode 314935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast 315935c21a1SIan Lepore */ 316935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0); 317935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0); 318935c21a1SIan Lepore break; 319935c21a1SIan Lepore case CPU_ARCH_ARM1176: 320935c21a1SIan Lepore /* 321935c21a1SIan Lepore * Restrict cache size to 16KB 322935c21a1SIan Lepore * Enable the return stack 323935c21a1SIan Lepore * Enable dynamic branch prediction 324935c21a1SIan Lepore * Enable static branch prediction 325935c21a1SIan Lepore */ 326935c21a1SIan Lepore *actlr_mask = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 327935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0); 328935c21a1SIan Lepore break; 329935c21a1SIan Lepore } 330935c21a1SIan Lepore return; 331935c21a1SIan Lepore } 332935c21a1SIan Lepore } 3337bf5720aSMichal Meloun 3347bf5720aSMichal Meloun /* Reinitialize MMU to final kernel mapping and apply all CPU quirks. */ 3357bf5720aSMichal Meloun void 3367bf5720aSMichal Meloun cpuinfo_reinit_mmu(uint32_t ttb) 3377bf5720aSMichal Meloun { 3387bf5720aSMichal Meloun uint32_t actlr_mask; 3397bf5720aSMichal Meloun uint32_t actlr_set; 3407bf5720aSMichal Meloun 3417bf5720aSMichal Meloun cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); 3427bf5720aSMichal Meloun actlr_mask |= cpu_quirks_actlr_mask; 3437bf5720aSMichal Meloun actlr_set |= cpu_quirks_actlr_set; 3447bf5720aSMichal Meloun reinit_mmu(ttb, actlr_mask, actlr_set); 3457bf5720aSMichal Meloun } 3467bf5720aSMichal Meloun 347a36b6ec0SMichal Meloun static bool 348a36b6ec0SMichal Meloun modify_actlr(uint32_t clear, uint32_t set) 349a36b6ec0SMichal Meloun { 350a36b6ec0SMichal Meloun uint32_t reg, newreg; 351a36b6ec0SMichal Meloun 352a36b6ec0SMichal Meloun reg = cp15_actlr_get(); 353a36b6ec0SMichal Meloun newreg = reg; 354a36b6ec0SMichal Meloun newreg &= ~clear; 355a36b6ec0SMichal Meloun newreg |= set; 356a36b6ec0SMichal Meloun if (reg == newreg) 357a36b6ec0SMichal Meloun return (true); 358a36b6ec0SMichal Meloun cp15_actlr_set(newreg); 359a36b6ec0SMichal Meloun 360a36b6ec0SMichal Meloun reg = cp15_actlr_get(); 361a36b6ec0SMichal Meloun if (reg == newreg) 362a36b6ec0SMichal Meloun return (true); 363a36b6ec0SMichal Meloun return (false); 364a36b6ec0SMichal Meloun } 365a36b6ec0SMichal Meloun 366a36b6ec0SMichal Meloun /* Apply/restore BP hardening on current core. */ 367a36b6ec0SMichal Meloun static int 368a36b6ec0SMichal Meloun apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask) 369a36b6ec0SMichal Meloun { 370a36b6ec0SMichal Meloun if (enable) { 371a36b6ec0SMichal Meloun if (actrl && !modify_actlr(0, set_mask)) 372a36b6ec0SMichal Meloun return (-1); 373a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, kind); 374a36b6ec0SMichal Meloun } else { 375a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 376a36b6ec0SMichal Meloun if (actrl) 377a36b6ec0SMichal Meloun modify_actlr(~0, PCPU_GET(original_actlr)); 378a36b6ec0SMichal Meloun spectre_v2_safe = 0; 379a36b6ec0SMichal Meloun } 380a36b6ec0SMichal Meloun return (0); 381a36b6ec0SMichal Meloun } 382a36b6ec0SMichal Meloun 383a36b6ec0SMichal Meloun static void 384a36b6ec0SMichal Meloun handle_bp_hardening(bool enable) 385a36b6ec0SMichal Meloun { 386a36b6ec0SMichal Meloun int kind; 387a36b6ec0SMichal Meloun char *kind_str; 388a36b6ec0SMichal Meloun 389a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_NONE; 390a36b6ec0SMichal Meloun /* 391a36b6ec0SMichal Meloun * Note: Access to ACTRL is locked to secure world on most boards. 392a36b6ec0SMichal Meloun * This means that full BP hardening depends on updated u-boot/firmware 393a36b6ec0SMichal Meloun * or is impossible at all (if secure monitor is in on-chip ROM). 394a36b6ec0SMichal Meloun */ 395a36b6ec0SMichal Meloun if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { 396a36b6ec0SMichal Meloun switch (cpuinfo.part_number) { 397a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A8: 398a36b6ec0SMichal Meloun /* 399a36b6ec0SMichal Meloun * For Cortex-A8, IBE bit must be set otherwise 400a36b6ec0SMichal Meloun * BPIALL is effectively NOP. 401a36b6ec0SMichal Meloun * Unfortunately, Cortex-A is also affected by 402a36b6ec0SMichal Meloun * ARM erratum 687067 which causes non-working 403a36b6ec0SMichal Meloun * BPIALL if IBE bit is set and 'Instruction L1 System 404a36b6ec0SMichal Meloun * Array Debug Register 0' is not 0. 405a36b6ec0SMichal Meloun * This register is not reset on power-up and is 406a36b6ec0SMichal Meloun * accessible only from secure world, so we cannot do 407a36b6ec0SMichal Meloun * nothing (nor detect) to fix this issue. 408a36b6ec0SMichal Meloun * I afraid that on chip ROM based secure monitor on 409a36b6ec0SMichal Meloun * AM335x (BeagleBone) doesn't reset this debug 410a36b6ec0SMichal Meloun * register. 411a36b6ec0SMichal Meloun */ 412a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL; 413a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0) 414a36b6ec0SMichal Meloun goto actlr_err; 415a36b6ec0SMichal Meloun break; 416a36b6ec0SMichal Meloun break; 417a36b6ec0SMichal Meloun 418a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A9: 419a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A12: 420a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A17: 421a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A57: 422a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A72: 423a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A73: 424a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75: 425a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL; 426a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, false, 0) != 0) 427a36b6ec0SMichal Meloun goto actlr_err; 428a36b6ec0SMichal Meloun break; 429a36b6ec0SMichal Meloun 430a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A15: 431a36b6ec0SMichal Meloun /* 432a36b6ec0SMichal Meloun * For Cortex-A15, set 'Enable invalidates of BTB' bit. 433a36b6ec0SMichal Meloun * Despite this, the BPIALL is still effectively NOP, 434a36b6ec0SMichal Meloun * but with this bit set, the ICIALLU also flushes 435a36b6ec0SMichal Meloun * branch predictor as side effect. 436a36b6ec0SMichal Meloun */ 437a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_ICIALLU; 438a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0) 439a36b6ec0SMichal Meloun goto actlr_err; 440a36b6ec0SMichal Meloun break; 441a36b6ec0SMichal Meloun 442a36b6ec0SMichal Meloun default: 443a36b6ec0SMichal Meloun break; 444a36b6ec0SMichal Meloun } 445a36b6ec0SMichal Meloun } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) { 446a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative " 447a36b6ec0SMichal Meloun "branch attacks. !!!\n" 448a36b6ec0SMichal Meloun "Qualcomm Krait cores are known (or believed) to be " 449a36b6ec0SMichal Meloun "vulnerable to \n" 450a36b6ec0SMichal Meloun "speculative branch attacks, no mitigation exists yet.\n", 451a36b6ec0SMichal Meloun PCPU_GET(cpuid)); 452a36b6ec0SMichal Meloun goto unkonown_mitigation; 453a36b6ec0SMichal Meloun } else { 454a36b6ec0SMichal Meloun goto unkonown_mitigation; 455a36b6ec0SMichal Meloun } 456a36b6ec0SMichal Meloun 457a36b6ec0SMichal Meloun if (bootverbose) { 458a36b6ec0SMichal Meloun switch (kind) { 459a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_NONE: 460a36b6ec0SMichal Meloun kind_str = "not necessary"; 461a36b6ec0SMichal Meloun break; 462a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_BPIALL: 463a36b6ec0SMichal Meloun kind_str = "BPIALL"; 464a36b6ec0SMichal Meloun break; 465a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_ICIALLU: 466a36b6ec0SMichal Meloun kind_str = "ICIALLU"; 467a36b6ec0SMichal Meloun break; 468a36b6ec0SMichal Meloun default: 469a36b6ec0SMichal Meloun panic("Unknown BP hardering kind (%d).", kind); 470a36b6ec0SMichal Meloun } 471a36b6ec0SMichal Meloun printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid), 472a36b6ec0SMichal Meloun kind_str); 473a36b6ec0SMichal Meloun } 474a36b6ec0SMichal Meloun 475a36b6ec0SMichal Meloun return; 476a36b6ec0SMichal Meloun 477a36b6ec0SMichal Meloun unkonown_mitigation: 478a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 479a36b6ec0SMichal Meloun spectre_v2_safe = 0; 480a36b6ec0SMichal Meloun return; 481a36b6ec0SMichal Meloun 482a36b6ec0SMichal Meloun actlr_err: 483a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); 484a36b6ec0SMichal Meloun spectre_v2_safe = 0; 485a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch " 486a36b6ec0SMichal Meloun "attacks. !!!\n" 487a36b6ec0SMichal Meloun "We cannot enable required bit(s) in ACTRL register\n" 488a36b6ec0SMichal Meloun "because it's locked by secure monitor and/or firmware.\n", 489a36b6ec0SMichal Meloun PCPU_GET(cpuid)); 490a36b6ec0SMichal Meloun } 491a36b6ec0SMichal Meloun 492a36b6ec0SMichal Meloun void 493a36b6ec0SMichal Meloun cpuinfo_init_bp_hardening(void) 494a36b6ec0SMichal Meloun { 495a36b6ec0SMichal Meloun 496a36b6ec0SMichal Meloun /* 497a36b6ec0SMichal Meloun * Store original unmodified ACTRL, so we can restore it when 498a36b6ec0SMichal Meloun * BP hardening is disabled by sysctl. 499a36b6ec0SMichal Meloun */ 500a36b6ec0SMichal Meloun PCPU_SET(original_actlr, cp15_actlr_get()); 501a36b6ec0SMichal Meloun handle_bp_hardening(true); 502a36b6ec0SMichal Meloun } 503a36b6ec0SMichal Meloun 504a36b6ec0SMichal Meloun static void 505a36b6ec0SMichal Meloun bp_hardening_action(void *arg) 506a36b6ec0SMichal Meloun { 507a36b6ec0SMichal Meloun 508a36b6ec0SMichal Meloun handle_bp_hardening(disable_bp_hardening == 0); 509a36b6ec0SMichal Meloun } 510a36b6ec0SMichal Meloun 511a36b6ec0SMichal Meloun static int 512a36b6ec0SMichal Meloun sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS) 513a36b6ec0SMichal Meloun { 514a36b6ec0SMichal Meloun int rv; 515a36b6ec0SMichal Meloun 516a36b6ec0SMichal Meloun rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 517a36b6ec0SMichal Meloun 518a36b6ec0SMichal Meloun if (!rv && req->newptr) { 519a36b6ec0SMichal Meloun spectre_v2_safe = 1; 520a36b6ec0SMichal Meloun dmb(); 521a36b6ec0SMichal Meloun #ifdef SMP 522a36b6ec0SMichal Meloun smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, 523a36b6ec0SMichal Meloun bp_hardening_action, NULL, NULL); 524a36b6ec0SMichal Meloun #else 525a36b6ec0SMichal Meloun bp_hardening_action(NULL); 526a36b6ec0SMichal Meloun #endif 527a36b6ec0SMichal Meloun } 528a36b6ec0SMichal Meloun 529a36b6ec0SMichal Meloun return (rv); 530a36b6ec0SMichal Meloun } 531a36b6ec0SMichal Meloun 532a36b6ec0SMichal Meloun SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening, 533a36b6ec0SMichal Meloun CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 534a36b6ec0SMichal Meloun &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I", 535a36b6ec0SMichal Meloun "Disable BP hardening mitigation."); 536a36b6ec0SMichal Meloun 537a36b6ec0SMichal Meloun SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD, 538a36b6ec0SMichal Meloun &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks"); 539a36b6ec0SMichal Meloun 5407bf5720aSMichal Meloun #endif /* __ARM_ARCH >= 6 */ 541