12b71562fSIan Lepore /*-
22b71562fSIan Lepore * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com>
32b71562fSIan Lepore * Copyright 2014 Michal Meloun <meloun@miracle.cz>
42b71562fSIan Lepore * All rights reserved.
52b71562fSIan Lepore *
62b71562fSIan Lepore * Redistribution and use in source and binary forms, with or without
72b71562fSIan Lepore * modification, are permitted provided that the following conditions
82b71562fSIan Lepore * are met:
92b71562fSIan Lepore * 1. Redistributions of source code must retain the above copyright
102b71562fSIan Lepore * notice, this list of conditions and the following disclaimer.
112b71562fSIan Lepore * 2. Redistributions in binary form must reproduce the above copyright
122b71562fSIan Lepore * notice, this list of conditions and the following disclaimer in the
132b71562fSIan Lepore * documentation and/or other materials provided with the distribution.
142b71562fSIan Lepore *
152b71562fSIan Lepore * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
162b71562fSIan Lepore * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
172b71562fSIan Lepore * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
182b71562fSIan Lepore * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
192b71562fSIan Lepore * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
202b71562fSIan Lepore * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
212b71562fSIan Lepore * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
222b71562fSIan Lepore * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
232b71562fSIan Lepore * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
242b71562fSIan Lepore * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
252b71562fSIan Lepore * SUCH DAMAGE.
262b71562fSIan Lepore */
272b71562fSIan Lepore
282b71562fSIan Lepore #include <sys/param.h>
292b71562fSIan Lepore #include <sys/systm.h>
30c40a5f8aSMichal Meloun #include <sys/kernel.h>
31a36b6ec0SMichal Meloun #include <sys/pcpu.h>
32a36b6ec0SMichal Meloun #include <sys/smp.h>
337bf5720aSMichal Meloun #include <sys/sysctl.h>
342b71562fSIan Lepore
353025d19dSMichal Meloun #include <machine/cpu.h>
362b71562fSIan Lepore #include <machine/cpuinfo.h>
370cbf724eSMichal Meloun #include <machine/elf.h>
380cbf724eSMichal Meloun #include <machine/md_var.h>
392b71562fSIan Lepore
407bf5720aSMichal Meloun void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set);
41a36b6ec0SMichal Meloun
42a36b6ec0SMichal Meloun int disable_bp_hardening;
43a36b6ec0SMichal Meloun int spectre_v2_safe = 1;
447bf5720aSMichal Meloun
45a286c311SIan Lepore struct cpuinfo cpuinfo =
46a286c311SIan Lepore {
47a286c311SIan Lepore /* Use safe defaults for start */
48a286c311SIan Lepore .dcache_line_size = 32,
49a286c311SIan Lepore .dcache_line_mask = 31,
50a286c311SIan Lepore .icache_line_size = 32,
51a286c311SIan Lepore .icache_line_mask = 31,
52a286c311SIan Lepore };
532b71562fSIan Lepore
547029da5cSPawel Biernacki static SYSCTL_NODE(_hw, OID_AUTO, cpu, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
557bf5720aSMichal Meloun "CPU");
567029da5cSPawel Biernacki static SYSCTL_NODE(_hw_cpu, OID_AUTO, quirks, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
577bf5720aSMichal Meloun "CPU quirks");
587bf5720aSMichal Meloun
597bf5720aSMichal Meloun /*
607bf5720aSMichal Meloun * Tunable CPU quirks.
617bf5720aSMichal Meloun * Be careful, ACTRL cannot be changed if CPU is started in secure
627bf5720aSMichal Meloun * mode(world) and write to ACTRL can cause exception!
637bf5720aSMichal Meloun * These quirks are intended for optimizing CPU performance, not for
647bf5720aSMichal Meloun * applying errata workarounds. Nobody can expect that CPU with unfixed
657bf5720aSMichal Meloun * errata is stable enough to execute the kernel until quirks are applied.
667bf5720aSMichal Meloun */
677bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_mask;
687bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_mask,
697bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_mask, 0,
707bf5720aSMichal Meloun "Bits to be masked in ACTLR");
717bf5720aSMichal Meloun
727bf5720aSMichal Meloun static uint32_t cpu_quirks_actlr_set;
737bf5720aSMichal Meloun SYSCTL_INT(_hw_cpu_quirks, OID_AUTO, actlr_set,
747bf5720aSMichal Meloun CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &cpu_quirks_actlr_set, 0,
757bf5720aSMichal Meloun "Bits to be set in ACTLR");
767bf5720aSMichal Meloun
7780b1995bSMateusz Guzik static int
sysctl_hw_cpu_quirks_actrl_value(SYSCTL_HANDLER_ARGS)7880b1995bSMateusz Guzik sysctl_hw_cpu_quirks_actrl_value(SYSCTL_HANDLER_ARGS)
7980b1995bSMateusz Guzik {
8080b1995bSMateusz Guzik uint32_t reg;
8180b1995bSMateusz Guzik
8280b1995bSMateusz Guzik reg = cp15_actlr_get();
8380b1995bSMateusz Guzik return (SYSCTL_OUT(req, ®, sizeof(reg)));
8480b1995bSMateusz Guzik }
8580b1995bSMateusz Guzik SYSCTL_PROC(_hw_cpu_quirks, OID_AUTO, actlr_value,
8680b1995bSMateusz Guzik CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
8780b1995bSMateusz Guzik sysctl_hw_cpu_quirks_actrl_value, "IU",
8880b1995bSMateusz Guzik "Value of ACTLR");
8980b1995bSMateusz Guzik
902b71562fSIan Lepore /* Read and parse CPU id scheme */
912b71562fSIan Lepore void
cpuinfo_init(void)922b71562fSIan Lepore cpuinfo_init(void)
932b71562fSIan Lepore {
940cbf724eSMichal Meloun uint32_t tmp;
952b71562fSIan Lepore
96c40a5f8aSMichal Meloun /*
97c40a5f8aSMichal Meloun * Prematurely fetch CPU quirks. Standard fetch for tunable
98c40a5f8aSMichal Meloun * sysctls is handled using SYSINIT, thus too late for boot CPU.
99c40a5f8aSMichal Meloun * Keep names in sync with sysctls.
100c40a5f8aSMichal Meloun */
101c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_mask", &cpu_quirks_actlr_mask);
102c40a5f8aSMichal Meloun TUNABLE_INT_FETCH("hw.cpu.quirks.actlr_set", &cpu_quirks_actlr_set);
103c40a5f8aSMichal Meloun
1042b71562fSIan Lepore cpuinfo.midr = cp15_midr_get();
1052b71562fSIan Lepore /* Test old version id schemes first */
1062b71562fSIan Lepore if ((cpuinfo.midr & CPU_ID_IMPLEMENTOR_MASK) == CPU_ID_ARM_LTD) {
1072b71562fSIan Lepore if (CPU_ID_ISOLD(cpuinfo.midr)) {
1082b71562fSIan Lepore /* obsolete ARMv2 or ARMv3 CPU */
1092b71562fSIan Lepore cpuinfo.midr = 0;
1102b71562fSIan Lepore return;
1112b71562fSIan Lepore }
1122b71562fSIan Lepore if (CPU_ID_IS7(cpuinfo.midr)) {
1132b71562fSIan Lepore if ((cpuinfo.midr & (1 << 23)) == 0) {
1142b71562fSIan Lepore /* obsolete ARMv3 CPU */
1152b71562fSIan Lepore cpuinfo.midr = 0;
1162b71562fSIan Lepore return;
1172b71562fSIan Lepore }
1182b71562fSIan Lepore /* ARMv4T CPU */
1192b71562fSIan Lepore cpuinfo.architecture = 1;
1202b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 16) & 0x7F;
1218e6dd301SIan Lepore } else {
1228e6dd301SIan Lepore /* ARM new id scheme */
1238e6dd301SIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F;
1248e6dd301SIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F;
1252b71562fSIan Lepore }
1262b71562fSIan Lepore } else {
1278e6dd301SIan Lepore /* non ARM -> must be new id scheme */
1282b71562fSIan Lepore cpuinfo.architecture = (cpuinfo.midr >> 16) & 0x0F;
1292b71562fSIan Lepore cpuinfo.revision = (cpuinfo.midr >> 20) & 0x0F;
1302b71562fSIan Lepore }
1312b71562fSIan Lepore /* Parse rest of MIDR */
1322b71562fSIan Lepore cpuinfo.implementer = (cpuinfo.midr >> 24) & 0xFF;
1332b71562fSIan Lepore cpuinfo.part_number = (cpuinfo.midr >> 4) & 0xFFF;
1342b71562fSIan Lepore cpuinfo.patch = cpuinfo.midr & 0x0F;
1352b71562fSIan Lepore
1362b71562fSIan Lepore /* CP15 c0,c0 regs 0-7 exist on all CPUs (although aliased with MIDR) */
1372b71562fSIan Lepore cpuinfo.ctr = cp15_ctr_get();
1382b71562fSIan Lepore cpuinfo.tcmtr = cp15_tcmtr_get();
1392b71562fSIan Lepore cpuinfo.tlbtr = cp15_tlbtr_get();
1402b71562fSIan Lepore cpuinfo.mpidr = cp15_mpidr_get();
1412b71562fSIan Lepore cpuinfo.revidr = cp15_revidr_get();
1422b71562fSIan Lepore
1432b71562fSIan Lepore /* if CPU is not v7 cpu id scheme */
1442b71562fSIan Lepore if (cpuinfo.architecture != 0xF)
1452b71562fSIan Lepore return;
1462b71562fSIan Lepore cpuinfo.id_pfr0 = cp15_id_pfr0_get();
1472b71562fSIan Lepore cpuinfo.id_pfr1 = cp15_id_pfr1_get();
1482b71562fSIan Lepore cpuinfo.id_dfr0 = cp15_id_dfr0_get();
1492b71562fSIan Lepore cpuinfo.id_afr0 = cp15_id_afr0_get();
1502b71562fSIan Lepore cpuinfo.id_mmfr0 = cp15_id_mmfr0_get();
1512b71562fSIan Lepore cpuinfo.id_mmfr1 = cp15_id_mmfr1_get();
1522b71562fSIan Lepore cpuinfo.id_mmfr2 = cp15_id_mmfr2_get();
1532b71562fSIan Lepore cpuinfo.id_mmfr3 = cp15_id_mmfr3_get();
1542b71562fSIan Lepore cpuinfo.id_isar0 = cp15_id_isar0_get();
1552b71562fSIan Lepore cpuinfo.id_isar1 = cp15_id_isar1_get();
1562b71562fSIan Lepore cpuinfo.id_isar2 = cp15_id_isar2_get();
1572b71562fSIan Lepore cpuinfo.id_isar3 = cp15_id_isar3_get();
1582b71562fSIan Lepore cpuinfo.id_isar4 = cp15_id_isar4_get();
1592b71562fSIan Lepore cpuinfo.id_isar5 = cp15_id_isar5_get();
1602b71562fSIan Lepore
1612b71562fSIan Lepore /* Not yet - CBAR only exist on ARM SMP Cortex A CPUs
1622b71562fSIan Lepore cpuinfo.cbar = cp15_cbar_get();
1632b71562fSIan Lepore */
164ba0bb206SMichal Meloun if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) {
165ba0bb206SMichal Meloun cpuinfo.ccsidr = cp15_ccsidr_get();
166ba0bb206SMichal Meloun cpuinfo.clidr = cp15_clidr_get();
167ba0bb206SMichal Meloun }
1682b71562fSIan Lepore
1692b71562fSIan Lepore /* Test if revidr is implemented */
1702b71562fSIan Lepore if (cpuinfo.revidr == cpuinfo.midr)
1712b71562fSIan Lepore cpuinfo.revidr = 0;
1722b71562fSIan Lepore
1732b71562fSIan Lepore /* parsed bits of above registers */
1742b71562fSIan Lepore /* id_mmfr0 */
1752b71562fSIan Lepore cpuinfo.outermost_shareability = (cpuinfo.id_mmfr0 >> 8) & 0xF;
1762b71562fSIan Lepore cpuinfo.shareability_levels = (cpuinfo.id_mmfr0 >> 12) & 0xF;
1772b71562fSIan Lepore cpuinfo.auxiliary_registers = (cpuinfo.id_mmfr0 >> 20) & 0xF;
1782b71562fSIan Lepore cpuinfo.innermost_shareability = (cpuinfo.id_mmfr0 >> 28) & 0xF;
1792b71562fSIan Lepore /* id_mmfr2 */
1802b71562fSIan Lepore cpuinfo.mem_barrier = (cpuinfo.id_mmfr2 >> 20) & 0xF;
1812b71562fSIan Lepore /* id_mmfr3 */
1822b71562fSIan Lepore cpuinfo.coherent_walk = (cpuinfo.id_mmfr3 >> 20) & 0xF;
1832b71562fSIan Lepore cpuinfo.maintenance_broadcast =(cpuinfo.id_mmfr3 >> 12) & 0xF;
1842b71562fSIan Lepore /* id_pfr1 */
1852b71562fSIan Lepore cpuinfo.generic_timer_ext = (cpuinfo.id_pfr1 >> 16) & 0xF;
1862b71562fSIan Lepore cpuinfo.virtualization_ext = (cpuinfo.id_pfr1 >> 12) & 0xF;
1872b71562fSIan Lepore cpuinfo.security_ext = (cpuinfo.id_pfr1 >> 4) & 0xF;
188d029cb61SAndrew Turner /* mpidr */
189d029cb61SAndrew Turner cpuinfo.mp_ext = (cpuinfo.mpidr >> 31u) & 0x1;
190a286c311SIan Lepore
191a286c311SIan Lepore /* L1 Cache sizes */
192a22f8196SIan Lepore if (CPU_CT_FORMAT(cpuinfo.ctr) == CPU_CT_ARMV7) {
193a22f8196SIan Lepore cpuinfo.dcache_line_size =
194a22f8196SIan Lepore 1 << (CPU_CT_DMINLINE(cpuinfo.ctr) + 2);
195a22f8196SIan Lepore cpuinfo.icache_line_size =
196a22f8196SIan Lepore 1 << (CPU_CT_IMINLINE(cpuinfo.ctr) + 2);
197a22f8196SIan Lepore } else {
198a22f8196SIan Lepore cpuinfo.dcache_line_size =
199a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_DSIZE(cpuinfo.ctr)) + 3);
200a22f8196SIan Lepore cpuinfo.icache_line_size =
201a22f8196SIan Lepore 1 << (CPU_CT_xSIZE_LEN(CPU_CT_ISIZE(cpuinfo.ctr)) + 3);
202a22f8196SIan Lepore }
203a286c311SIan Lepore cpuinfo.dcache_line_mask = cpuinfo.dcache_line_size - 1;
204a286c311SIan Lepore cpuinfo.icache_line_mask = cpuinfo.icache_line_size - 1;
2050cbf724eSMichal Meloun
2060cbf724eSMichal Meloun /* Fill AT_HWCAP bits. */
20732c48d07SMichal Meloun elf_hwcap |= HWCAP_HALF | HWCAP_FAST_MULT; /* Required for all CPUs */
20832c48d07SMichal Meloun elf_hwcap |= HWCAP_TLS | HWCAP_EDSP; /* Required for v6+ CPUs */
2090cbf724eSMichal Meloun
2100cbf724eSMichal Meloun tmp = (cpuinfo.id_isar0 >> 24) & 0xF; /* Divide_instrs */
2110cbf724eSMichal Meloun if (tmp >= 1)
2120cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVT;
2130cbf724eSMichal Meloun if (tmp >= 2)
2140cbf724eSMichal Meloun elf_hwcap |= HWCAP_IDIVA;
2150cbf724eSMichal Meloun
2160cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 4) & 0xF; /* State1 */
2170cbf724eSMichal Meloun if (tmp >= 1)
2180cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMB;
2190cbf724eSMichal Meloun
2200cbf724eSMichal Meloun tmp = (cpuinfo.id_pfr0 >> 12) & 0xF; /* State3 */
2210cbf724eSMichal Meloun if (tmp >= 1)
2220cbf724eSMichal Meloun elf_hwcap |= HWCAP_THUMBEE;
2230cbf724eSMichal Meloun
2240cbf724eSMichal Meloun tmp = (cpuinfo.id_mmfr0 >> 0) & 0xF; /* VMSA */
2250cbf724eSMichal Meloun if (tmp >= 5)
2260cbf724eSMichal Meloun elf_hwcap |= HWCAP_LPAE;
2270cbf724eSMichal Meloun
2280cbf724eSMichal Meloun /* Fill AT_HWCAP2 bits. */
2290cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 4) & 0xF; /* AES */
2300cbf724eSMichal Meloun if (tmp >= 1)
2310cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_AES;
2320cbf724eSMichal Meloun if (tmp >= 2)
2330cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_PMULL;
2340cbf724eSMichal Meloun
2350cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 8) & 0xF; /* SHA1 */
2360cbf724eSMichal Meloun if (tmp >= 1)
2370cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA1;
2380cbf724eSMichal Meloun
2390cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 12) & 0xF; /* SHA2 */
2400cbf724eSMichal Meloun if (tmp >= 1)
2410cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_SHA2;
2420cbf724eSMichal Meloun
2430cbf724eSMichal Meloun tmp = (cpuinfo.id_isar5 >> 16) & 0xF; /* CRC32 */
2440cbf724eSMichal Meloun if (tmp >= 1)
2450cbf724eSMichal Meloun elf_hwcap2 |= HWCAP2_CRC32;
2462b71562fSIan Lepore }
247935c21a1SIan Lepore
248935c21a1SIan Lepore /*
249935c21a1SIan Lepore * Get bits that must be set or cleared in ACLR register.
250935c21a1SIan Lepore * Note: Bits in ACLR register are IMPLEMENTATION DEFINED.
251935c21a1SIan Lepore * Its expected that SCU is in operational state before this
252935c21a1SIan Lepore * function is called.
253935c21a1SIan Lepore */
2547bf5720aSMichal Meloun static void
cpuinfo_get_actlr_modifier(uint32_t * actlr_mask,uint32_t * actlr_set)255935c21a1SIan Lepore cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set)
256935c21a1SIan Lepore {
2577bf5720aSMichal Meloun
258935c21a1SIan Lepore *actlr_mask = 0;
259935c21a1SIan Lepore *actlr_set = 0;
260935c21a1SIan Lepore
261935c21a1SIan Lepore if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
262935c21a1SIan Lepore switch (cpuinfo.part_number) {
263a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75:
264ba0bb206SMichal Meloun case CPU_ARCH_CORTEX_A73:
26555e447c9SMichal Meloun case CPU_ARCH_CORTEX_A72:
26655e447c9SMichal Meloun case CPU_ARCH_CORTEX_A57:
26755e447c9SMichal Meloun case CPU_ARCH_CORTEX_A53:
26855e447c9SMichal Meloun /* Nothing to do for AArch32 */
26955e447c9SMichal Meloun break;
270935c21a1SIan Lepore case CPU_ARCH_CORTEX_A17:
271935c21a1SIan Lepore case CPU_ARCH_CORTEX_A12: /* A12 is merged to A17 */
272935c21a1SIan Lepore /*
273935c21a1SIan Lepore * Enable SMP mode
274935c21a1SIan Lepore */
275935c21a1SIan Lepore *actlr_mask = (1 << 6);
276935c21a1SIan Lepore *actlr_set = (1 << 6);
277935c21a1SIan Lepore break;
278935c21a1SIan Lepore case CPU_ARCH_CORTEX_A15:
279935c21a1SIan Lepore /*
280935c21a1SIan Lepore * Enable snoop-delayed exclusive handling
281935c21a1SIan Lepore * Enable SMP mode
282935c21a1SIan Lepore */
283935c21a1SIan Lepore *actlr_mask = (1U << 31) |(1 << 6);
284935c21a1SIan Lepore *actlr_set = (1U << 31) |(1 << 6);
285935c21a1SIan Lepore break;
286935c21a1SIan Lepore case CPU_ARCH_CORTEX_A9:
287935c21a1SIan Lepore /*
288935c21a1SIan Lepore * Disable exclusive L1/L2 cache control
289935c21a1SIan Lepore * Enable SMP mode
290935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast
291935c21a1SIan Lepore */
292935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0);
293935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0);
294935c21a1SIan Lepore break;
295935c21a1SIan Lepore case CPU_ARCH_CORTEX_A8:
296935c21a1SIan Lepore /*
297935c21a1SIan Lepore * Enable L2 cache
298935c21a1SIan Lepore * Enable L1 data cache hardware alias checks
299935c21a1SIan Lepore */
300935c21a1SIan Lepore *actlr_mask = (1 << 1) | (1 << 0);
301935c21a1SIan Lepore *actlr_set = (1 << 1);
302935c21a1SIan Lepore break;
303935c21a1SIan Lepore case CPU_ARCH_CORTEX_A7:
304935c21a1SIan Lepore /*
305935c21a1SIan Lepore * Enable SMP mode
306935c21a1SIan Lepore */
307935c21a1SIan Lepore *actlr_mask = (1 << 6);
308935c21a1SIan Lepore *actlr_set = (1 << 6);
309935c21a1SIan Lepore break;
310935c21a1SIan Lepore case CPU_ARCH_CORTEX_A5:
311935c21a1SIan Lepore /*
312935c21a1SIan Lepore * Disable exclusive L1/L2 cache control
313935c21a1SIan Lepore * Enable SMP mode
314935c21a1SIan Lepore * Enable Cache and TLB maintenance broadcast
315935c21a1SIan Lepore */
316935c21a1SIan Lepore *actlr_mask = (1 << 7) | (1 << 6) | (1 << 0);
317935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 0);
318935c21a1SIan Lepore break;
319935c21a1SIan Lepore case CPU_ARCH_ARM1176:
320935c21a1SIan Lepore /*
321935c21a1SIan Lepore * Restrict cache size to 16KB
322935c21a1SIan Lepore * Enable the return stack
323935c21a1SIan Lepore * Enable dynamic branch prediction
324935c21a1SIan Lepore * Enable static branch prediction
325935c21a1SIan Lepore */
326935c21a1SIan Lepore *actlr_mask = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0);
327935c21a1SIan Lepore *actlr_set = (1 << 6) | (1 << 2) | (1 << 1) | (1 << 0);
328935c21a1SIan Lepore break;
329935c21a1SIan Lepore }
330935c21a1SIan Lepore return;
331935c21a1SIan Lepore }
332935c21a1SIan Lepore }
3337bf5720aSMichal Meloun
3347bf5720aSMichal Meloun /* Reinitialize MMU to final kernel mapping and apply all CPU quirks. */
3357bf5720aSMichal Meloun void
cpuinfo_reinit_mmu(uint32_t ttb)3367bf5720aSMichal Meloun cpuinfo_reinit_mmu(uint32_t ttb)
3377bf5720aSMichal Meloun {
3387bf5720aSMichal Meloun uint32_t actlr_mask;
3397bf5720aSMichal Meloun uint32_t actlr_set;
3407bf5720aSMichal Meloun
3417bf5720aSMichal Meloun cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set);
3427bf5720aSMichal Meloun actlr_mask |= cpu_quirks_actlr_mask;
3437bf5720aSMichal Meloun actlr_set |= cpu_quirks_actlr_set;
3447bf5720aSMichal Meloun reinit_mmu(ttb, actlr_mask, actlr_set);
3457bf5720aSMichal Meloun }
3467bf5720aSMichal Meloun
347a36b6ec0SMichal Meloun static bool
modify_actlr(uint32_t clear,uint32_t set)348a36b6ec0SMichal Meloun modify_actlr(uint32_t clear, uint32_t set)
349a36b6ec0SMichal Meloun {
350a36b6ec0SMichal Meloun uint32_t reg, newreg;
351a36b6ec0SMichal Meloun
352a36b6ec0SMichal Meloun reg = cp15_actlr_get();
353a36b6ec0SMichal Meloun newreg = reg;
354a36b6ec0SMichal Meloun newreg &= ~clear;
355a36b6ec0SMichal Meloun newreg |= set;
356a36b6ec0SMichal Meloun if (reg == newreg)
357a36b6ec0SMichal Meloun return (true);
358a36b6ec0SMichal Meloun cp15_actlr_set(newreg);
359a36b6ec0SMichal Meloun
360a36b6ec0SMichal Meloun reg = cp15_actlr_get();
361a36b6ec0SMichal Meloun if (reg == newreg)
362a36b6ec0SMichal Meloun return (true);
363a36b6ec0SMichal Meloun return (false);
364a36b6ec0SMichal Meloun }
365a36b6ec0SMichal Meloun
366a36b6ec0SMichal Meloun /* Apply/restore BP hardening on current core. */
367a36b6ec0SMichal Meloun static int
apply_bp_hardening(bool enable,int kind,bool actrl,uint32_t set_mask)368a36b6ec0SMichal Meloun apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask)
369a36b6ec0SMichal Meloun {
370a36b6ec0SMichal Meloun if (enable) {
371a36b6ec0SMichal Meloun if (actrl && !modify_actlr(0, set_mask))
372a36b6ec0SMichal Meloun return (-1);
373a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, kind);
374a36b6ec0SMichal Meloun } else {
375a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
376a36b6ec0SMichal Meloun if (actrl)
377a36b6ec0SMichal Meloun modify_actlr(~0, PCPU_GET(original_actlr));
378a36b6ec0SMichal Meloun spectre_v2_safe = 0;
379a36b6ec0SMichal Meloun }
380a36b6ec0SMichal Meloun return (0);
381a36b6ec0SMichal Meloun }
382a36b6ec0SMichal Meloun
383a36b6ec0SMichal Meloun static void
handle_bp_hardening(bool enable)384a36b6ec0SMichal Meloun handle_bp_hardening(bool enable)
385a36b6ec0SMichal Meloun {
386a36b6ec0SMichal Meloun int kind;
387a36b6ec0SMichal Meloun char *kind_str;
388a36b6ec0SMichal Meloun
389a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_NONE;
390a36b6ec0SMichal Meloun /*
391a36b6ec0SMichal Meloun * Note: Access to ACTRL is locked to secure world on most boards.
392a36b6ec0SMichal Meloun * This means that full BP hardening depends on updated u-boot/firmware
393a36b6ec0SMichal Meloun * or is impossible at all (if secure monitor is in on-chip ROM).
394a36b6ec0SMichal Meloun */
395a36b6ec0SMichal Meloun if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
396a36b6ec0SMichal Meloun switch (cpuinfo.part_number) {
397a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A8:
398a36b6ec0SMichal Meloun /*
399a36b6ec0SMichal Meloun * For Cortex-A8, IBE bit must be set otherwise
400a36b6ec0SMichal Meloun * BPIALL is effectively NOP.
401a36b6ec0SMichal Meloun * Unfortunately, Cortex-A is also affected by
402a36b6ec0SMichal Meloun * ARM erratum 687067 which causes non-working
403a36b6ec0SMichal Meloun * BPIALL if IBE bit is set and 'Instruction L1 System
404a36b6ec0SMichal Meloun * Array Debug Register 0' is not 0.
405a36b6ec0SMichal Meloun * This register is not reset on power-up and is
406a36b6ec0SMichal Meloun * accessible only from secure world, so we cannot do
407a36b6ec0SMichal Meloun * nothing (nor detect) to fix this issue.
408a36b6ec0SMichal Meloun * I afraid that on chip ROM based secure monitor on
409a36b6ec0SMichal Meloun * AM335x (BeagleBone) doesn't reset this debug
410a36b6ec0SMichal Meloun * register.
411a36b6ec0SMichal Meloun */
412a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL;
413a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0)
414a36b6ec0SMichal Meloun goto actlr_err;
415a36b6ec0SMichal Meloun break;
416a36b6ec0SMichal Meloun break;
417a36b6ec0SMichal Meloun
418a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A9:
419a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A12:
420a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A17:
421a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A57:
422a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A72:
423a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A73:
424a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A75:
425a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_BPIALL;
426a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, false, 0) != 0)
427a36b6ec0SMichal Meloun goto actlr_err;
428a36b6ec0SMichal Meloun break;
429a36b6ec0SMichal Meloun
430a36b6ec0SMichal Meloun case CPU_ARCH_CORTEX_A15:
431a36b6ec0SMichal Meloun /*
432a36b6ec0SMichal Meloun * For Cortex-A15, set 'Enable invalidates of BTB' bit.
433a36b6ec0SMichal Meloun * Despite this, the BPIALL is still effectively NOP,
434a36b6ec0SMichal Meloun * but with this bit set, the ICIALLU also flushes
435a36b6ec0SMichal Meloun * branch predictor as side effect.
436a36b6ec0SMichal Meloun */
437a36b6ec0SMichal Meloun kind = PCPU_BP_HARDEN_KIND_ICIALLU;
438a36b6ec0SMichal Meloun if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0)
439a36b6ec0SMichal Meloun goto actlr_err;
440a36b6ec0SMichal Meloun break;
441a36b6ec0SMichal Meloun
442a36b6ec0SMichal Meloun default:
443a36b6ec0SMichal Meloun break;
444a36b6ec0SMichal Meloun }
445a36b6ec0SMichal Meloun } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) {
446a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative "
447a36b6ec0SMichal Meloun "branch attacks. !!!\n"
448a36b6ec0SMichal Meloun "Qualcomm Krait cores are known (or believed) to be "
449a36b6ec0SMichal Meloun "vulnerable to \n"
450a36b6ec0SMichal Meloun "speculative branch attacks, no mitigation exists yet.\n",
451a36b6ec0SMichal Meloun PCPU_GET(cpuid));
452a36b6ec0SMichal Meloun goto unkonown_mitigation;
453a36b6ec0SMichal Meloun } else {
454a36b6ec0SMichal Meloun goto unkonown_mitigation;
455a36b6ec0SMichal Meloun }
456a36b6ec0SMichal Meloun
457a36b6ec0SMichal Meloun if (bootverbose) {
458a36b6ec0SMichal Meloun switch (kind) {
459a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_NONE:
460a36b6ec0SMichal Meloun kind_str = "not necessary";
461a36b6ec0SMichal Meloun break;
462a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_BPIALL:
463a36b6ec0SMichal Meloun kind_str = "BPIALL";
464a36b6ec0SMichal Meloun break;
465a36b6ec0SMichal Meloun case PCPU_BP_HARDEN_KIND_ICIALLU:
466a36b6ec0SMichal Meloun kind_str = "ICIALLU";
467a36b6ec0SMichal Meloun break;
468a36b6ec0SMichal Meloun default:
469a36b6ec0SMichal Meloun panic("Unknown BP hardering kind (%d).", kind);
470a36b6ec0SMichal Meloun }
471a36b6ec0SMichal Meloun printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid),
472a36b6ec0SMichal Meloun kind_str);
473a36b6ec0SMichal Meloun }
474a36b6ec0SMichal Meloun
475a36b6ec0SMichal Meloun return;
476a36b6ec0SMichal Meloun
477a36b6ec0SMichal Meloun unkonown_mitigation:
478a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
479a36b6ec0SMichal Meloun spectre_v2_safe = 0;
480a36b6ec0SMichal Meloun return;
481a36b6ec0SMichal Meloun
482a36b6ec0SMichal Meloun actlr_err:
483a36b6ec0SMichal Meloun PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
484a36b6ec0SMichal Meloun spectre_v2_safe = 0;
485a36b6ec0SMichal Meloun printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch "
486a36b6ec0SMichal Meloun "attacks. !!!\n"
487a36b6ec0SMichal Meloun "We cannot enable required bit(s) in ACTRL register\n"
488a36b6ec0SMichal Meloun "because it's locked by secure monitor and/or firmware.\n",
489a36b6ec0SMichal Meloun PCPU_GET(cpuid));
490a36b6ec0SMichal Meloun }
491a36b6ec0SMichal Meloun
492a36b6ec0SMichal Meloun void
cpuinfo_init_bp_hardening(void)493a36b6ec0SMichal Meloun cpuinfo_init_bp_hardening(void)
494a36b6ec0SMichal Meloun {
495a36b6ec0SMichal Meloun
496a36b6ec0SMichal Meloun /*
497a36b6ec0SMichal Meloun * Store original unmodified ACTRL, so we can restore it when
498a36b6ec0SMichal Meloun * BP hardening is disabled by sysctl.
499a36b6ec0SMichal Meloun */
500a36b6ec0SMichal Meloun PCPU_SET(original_actlr, cp15_actlr_get());
501a36b6ec0SMichal Meloun handle_bp_hardening(true);
502a36b6ec0SMichal Meloun }
503a36b6ec0SMichal Meloun
504a36b6ec0SMichal Meloun static void
bp_hardening_action(void * arg)505a36b6ec0SMichal Meloun bp_hardening_action(void *arg)
506a36b6ec0SMichal Meloun {
507a36b6ec0SMichal Meloun
508a36b6ec0SMichal Meloun handle_bp_hardening(disable_bp_hardening == 0);
509a36b6ec0SMichal Meloun }
510a36b6ec0SMichal Meloun
511a36b6ec0SMichal Meloun static int
sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS)512a36b6ec0SMichal Meloun sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS)
513a36b6ec0SMichal Meloun {
514a36b6ec0SMichal Meloun int rv;
515a36b6ec0SMichal Meloun
516a36b6ec0SMichal Meloun rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
517a36b6ec0SMichal Meloun
518a36b6ec0SMichal Meloun if (!rv && req->newptr) {
519a36b6ec0SMichal Meloun spectre_v2_safe = 1;
520a36b6ec0SMichal Meloun dmb();
521a36b6ec0SMichal Meloun #ifdef SMP
522a36b6ec0SMichal Meloun smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier,
523a36b6ec0SMichal Meloun bp_hardening_action, NULL, NULL);
524a36b6ec0SMichal Meloun #else
525a36b6ec0SMichal Meloun bp_hardening_action(NULL);
526a36b6ec0SMichal Meloun #endif
527a36b6ec0SMichal Meloun }
528a36b6ec0SMichal Meloun
529a36b6ec0SMichal Meloun return (rv);
530a36b6ec0SMichal Meloun }
531a36b6ec0SMichal Meloun
532a36b6ec0SMichal Meloun SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening,
533a36b6ec0SMichal Meloun CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
534a36b6ec0SMichal Meloun &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I",
535a36b6ec0SMichal Meloun "Disable BP hardening mitigation.");
536a36b6ec0SMichal Meloun
537a36b6ec0SMichal Meloun SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD,
538a36b6ec0SMichal Meloun &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks");
539