1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21d18c47cSCatalin Marinas /*
31d18c47cSCatalin Marinas * Based on arch/arm/mm/fault.c
41d18c47cSCatalin Marinas *
51d18c47cSCatalin Marinas * Copyright (C) 1995 Linus Torvalds
61d18c47cSCatalin Marinas * Copyright (C) 1995-2004 Russell King
71d18c47cSCatalin Marinas * Copyright (C) 2012 ARM Ltd.
81d18c47cSCatalin Marinas */
91d18c47cSCatalin Marinas
10d44f1b8dSJames Morse #include <linux/acpi.h>
1142f91093SWill Deacon #include <linux/bitfield.h>
120edfa839SPaul Gortmaker #include <linux/extable.h>
13840b2398SMarco Elver #include <linux/kfence.h>
141d18c47cSCatalin Marinas #include <linux/signal.h>
151d18c47cSCatalin Marinas #include <linux/mm.h>
161d18c47cSCatalin Marinas #include <linux/hardirq.h>
171d18c47cSCatalin Marinas #include <linux/init.h>
184291e9eeSAndrey Konovalov #include <linux/kasan.h>
191d18c47cSCatalin Marinas #include <linux/kprobes.h>
201d18c47cSCatalin Marinas #include <linux/uaccess.h>
211d18c47cSCatalin Marinas #include <linux/page-flags.h>
223f07c014SIngo Molnar #include <linux/sched/signal.h>
23b17b0153SIngo Molnar #include <linux/sched/debug.h>
241d18c47cSCatalin Marinas #include <linux/highmem.h>
251d18c47cSCatalin Marinas #include <linux/perf_event.h>
267209c868SJames Morse #include <linux/preempt.h>
27e7c600f1SJonathan (Zhixiong) Zhang #include <linux/hugetlb.h>
281d18c47cSCatalin Marinas
29d44f1b8dSJames Morse #include <asm/acpi.h>
307209c868SJames Morse #include <asm/bug.h>
313bbf7157SCatalin Marinas #include <asm/cmpxchg.h>
32338d4f49SJames Morse #include <asm/cpufeature.h>
33e8dfdf31SArd Biesheuvel #include <asm/efi.h>
341d18c47cSCatalin Marinas #include <asm/exception.h>
359a0c0328SJulien Thierry #include <asm/daifflags.h>
361d18c47cSCatalin Marinas #include <asm/debug-monitors.h>
379141300aSCatalin Marinas #include <asm/esr.h>
38b6e43c0eSJames Morse #include <asm/kprobes.h>
3998c970daSVincenzo Frascino #include <asm/mte.h>
40bfe29874SJames Morse #include <asm/processor.h>
41338d4f49SJames Morse #include <asm/sysreg.h>
421d18c47cSCatalin Marinas #include <asm/system_misc.h>
431d18c47cSCatalin Marinas #include <asm/tlbflush.h>
4492ff0674SWill Deacon #include <asm/traps.h>
451d18c47cSCatalin Marinas
4609a6adf5SVictor Kamensky struct fault_info {
478d56e5c5SAlexandru Elisei int (*fn)(unsigned long far, unsigned long esr,
4809a6adf5SVictor Kamensky struct pt_regs *regs);
4909a6adf5SVictor Kamensky int sig;
5009a6adf5SVictor Kamensky int code;
5109a6adf5SVictor Kamensky const char *name;
5209a6adf5SVictor Kamensky };
5309a6adf5SVictor Kamensky
5409a6adf5SVictor Kamensky static const struct fault_info fault_info[];
55359048f9SAnshuman Khandual static struct fault_info debug_fault_info[];
5609a6adf5SVictor Kamensky
esr_to_fault_info(unsigned long esr)578d56e5c5SAlexandru Elisei static inline const struct fault_info *esr_to_fault_info(unsigned long esr)
5809a6adf5SVictor Kamensky {
5900bbd5d9SAnshuman Khandual return fault_info + (esr & ESR_ELx_FSC);
6009a6adf5SVictor Kamensky }
613495386bSCatalin Marinas
esr_to_debug_fault_info(unsigned long esr)628d56e5c5SAlexandru Elisei static inline const struct fault_info *esr_to_debug_fault_info(unsigned long esr)
63359048f9SAnshuman Khandual {
64359048f9SAnshuman Khandual return debug_fault_info + DBG_ESR_EVT(esr);
65359048f9SAnshuman Khandual }
66359048f9SAnshuman Khandual
data_abort_decode(unsigned long esr)678d56e5c5SAlexandru Elisei static void data_abort_decode(unsigned long esr)
681f9b8936SJulien Thierry {
691f9d4ba6SMark Brown unsigned long iss2 = ESR_ELx_ISS2(esr);
701f9d4ba6SMark Brown
711f9b8936SJulien Thierry pr_alert("Data abort info:\n");
721f9b8936SJulien Thierry
731f9b8936SJulien Thierry if (esr & ESR_ELx_ISV) {
741f9b8936SJulien Thierry pr_alert(" Access size = %u byte(s)\n",
751f9b8936SJulien Thierry 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
761f9b8936SJulien Thierry pr_alert(" SSE = %lu, SRT = %lu\n",
771f9b8936SJulien Thierry (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
781f9b8936SJulien Thierry (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
791f9b8936SJulien Thierry pr_alert(" SF = %lu, AR = %lu\n",
801f9b8936SJulien Thierry (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
811f9b8936SJulien Thierry (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
821f9b8936SJulien Thierry } else {
831f9d4ba6SMark Brown pr_alert(" ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n",
841f9d4ba6SMark Brown esr & ESR_ELx_ISS_MASK, iss2);
851f9b8936SJulien Thierry }
861f9b8936SJulien Thierry
871f9d4ba6SMark Brown pr_alert(" CM = %lu, WnR = %lu, TnD = %lu, TagAccess = %lu\n",
881f9b8936SJulien Thierry (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
891f9d4ba6SMark Brown (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT,
901f9d4ba6SMark Brown (iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT,
911f9d4ba6SMark Brown (iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT);
921f9d4ba6SMark Brown
931f9d4ba6SMark Brown pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu, Xs = %llu\n",
941f9d4ba6SMark Brown (iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT,
951f9d4ba6SMark Brown (iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT,
961f9d4ba6SMark Brown (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT,
971f9d4ba6SMark Brown (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT);
981f9b8936SJulien Thierry }
991f9b8936SJulien Thierry
mem_abort_decode(unsigned long esr)1008d56e5c5SAlexandru Elisei static void mem_abort_decode(unsigned long esr)
1011f9b8936SJulien Thierry {
1021f9b8936SJulien Thierry pr_alert("Mem abort info:\n");
1031f9b8936SJulien Thierry
1048d56e5c5SAlexandru Elisei pr_alert(" ESR = 0x%016lx\n", esr);
1052951d5efSMiles Chen pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
1062951d5efSMiles Chen ESR_ELx_EC(esr), esr_get_class_string(esr),
1071f9b8936SJulien Thierry (esr & ESR_ELx_IL) ? 32 : 16);
1081f9b8936SJulien Thierry pr_alert(" SET = %lu, FnV = %lu\n",
1091f9b8936SJulien Thierry (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
1101f9b8936SJulien Thierry (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
1111f9b8936SJulien Thierry pr_alert(" EA = %lu, S1PTW = %lu\n",
1121f9b8936SJulien Thierry (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
1131f9b8936SJulien Thierry (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
1148d56e5c5SAlexandru Elisei pr_alert(" FSC = 0x%02lx: %s\n", (esr & ESR_ELx_FSC),
115e0e3903fSMark Rutland esr_to_fault_info(esr)->name);
1161f9b8936SJulien Thierry
1171f9b8936SJulien Thierry if (esr_is_data_abort(esr))
1181f9b8936SJulien Thierry data_abort_decode(esr);
1191f9b8936SJulien Thierry }
1201f9b8936SJulien Thierry
mm_to_pgd_phys(struct mm_struct * mm)121e4365f96SMark Rutland static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
122e4365f96SMark Rutland {
123e4365f96SMark Rutland /* Either init_pg_dir or swapper_pg_dir */
124e4365f96SMark Rutland if (mm == &init_mm)
125e4365f96SMark Rutland return __pa_symbol(mm->pgd);
126e4365f96SMark Rutland
127e4365f96SMark Rutland return (unsigned long)virt_to_phys(mm->pgd);
128e4365f96SMark Rutland }
129e4365f96SMark Rutland
1301d18c47cSCatalin Marinas /*
13167ce16ecSKristina Martsenko * Dump out the page tables associated with 'addr' in the currently active mm.
1321d18c47cSCatalin Marinas */
show_pte(unsigned long addr)1337048a597SWill Deacon static void show_pte(unsigned long addr)
1341d18c47cSCatalin Marinas {
13567ce16ecSKristina Martsenko struct mm_struct *mm;
13620a004e7SWill Deacon pgd_t *pgdp;
13720a004e7SWill Deacon pgd_t pgd;
1381d18c47cSCatalin Marinas
139356607f2SAndrey Konovalov if (is_ttbr0_addr(addr)) {
14067ce16ecSKristina Martsenko /* TTBR0 */
14167ce16ecSKristina Martsenko mm = current->active_mm;
14267ce16ecSKristina Martsenko if (mm == &init_mm) {
14367ce16ecSKristina Martsenko pr_alert("[%016lx] user address but active_mm is swapper\n",
14467ce16ecSKristina Martsenko addr);
14567ce16ecSKristina Martsenko return;
14667ce16ecSKristina Martsenko }
147356607f2SAndrey Konovalov } else if (is_ttbr1_addr(addr)) {
14867ce16ecSKristina Martsenko /* TTBR1 */
1491d18c47cSCatalin Marinas mm = &init_mm;
15067ce16ecSKristina Martsenko } else {
15167ce16ecSKristina Martsenko pr_alert("[%016lx] address between user and kernel address ranges\n",
15267ce16ecSKristina Martsenko addr);
15367ce16ecSKristina Martsenko return;
15467ce16ecSKristina Martsenko }
1551d18c47cSCatalin Marinas
1565383cc6eSSteve Capper pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
1571eb34b6eSWill Deacon mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
158e4365f96SMark Rutland vabits_actual, mm_to_pgd_phys(mm));
15920a004e7SWill Deacon pgdp = pgd_offset(mm, addr);
16020a004e7SWill Deacon pgd = READ_ONCE(*pgdp);
16120a004e7SWill Deacon pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
1621d18c47cSCatalin Marinas
1631d18c47cSCatalin Marinas do {
164e9f63768SMike Rapoport p4d_t *p4dp, p4d;
16520a004e7SWill Deacon pud_t *pudp, pud;
16620a004e7SWill Deacon pmd_t *pmdp, pmd;
16720a004e7SWill Deacon pte_t *ptep, pte;
1681d18c47cSCatalin Marinas
16920a004e7SWill Deacon if (pgd_none(pgd) || pgd_bad(pgd))
1701d18c47cSCatalin Marinas break;
1711d18c47cSCatalin Marinas
172e9f63768SMike Rapoport p4dp = p4d_offset(pgdp, addr);
173e9f63768SMike Rapoport p4d = READ_ONCE(*p4dp);
174e9f63768SMike Rapoport pr_cont(", p4d=%016llx", p4d_val(p4d));
175e9f63768SMike Rapoport if (p4d_none(p4d) || p4d_bad(p4d))
176e9f63768SMike Rapoport break;
177e9f63768SMike Rapoport
178e9f63768SMike Rapoport pudp = pud_offset(p4dp, addr);
17920a004e7SWill Deacon pud = READ_ONCE(*pudp);
18020a004e7SWill Deacon pr_cont(", pud=%016llx", pud_val(pud));
18120a004e7SWill Deacon if (pud_none(pud) || pud_bad(pud))
1821d18c47cSCatalin Marinas break;
1831d18c47cSCatalin Marinas
18420a004e7SWill Deacon pmdp = pmd_offset(pudp, addr);
18520a004e7SWill Deacon pmd = READ_ONCE(*pmdp);
18620a004e7SWill Deacon pr_cont(", pmd=%016llx", pmd_val(pmd));
18720a004e7SWill Deacon if (pmd_none(pmd) || pmd_bad(pmd))
1881d18c47cSCatalin Marinas break;
1891d18c47cSCatalin Marinas
19020a004e7SWill Deacon ptep = pte_offset_map(pmdp, addr);
19152924726SHugh Dickins if (!ptep)
19252924726SHugh Dickins break;
19352924726SHugh Dickins
194*5a00bfd6SRyan Roberts pte = __ptep_get(ptep);
19520a004e7SWill Deacon pr_cont(", pte=%016llx", pte_val(pte));
19620a004e7SWill Deacon pte_unmap(ptep);
1971d18c47cSCatalin Marinas } while(0);
1981d18c47cSCatalin Marinas
1996ef4fb38SMark Rutland pr_cont("\n");
2001d18c47cSCatalin Marinas }
2011d18c47cSCatalin Marinas
20266dbd6e6SCatalin Marinas /*
20366dbd6e6SCatalin Marinas * This function sets the access flags (dirty, accessed), as well as write
20466dbd6e6SCatalin Marinas * permission, and only to a more permissive setting.
20566dbd6e6SCatalin Marinas *
20666dbd6e6SCatalin Marinas * It needs to cope with hardware update of the accessed/dirty state by other
20766dbd6e6SCatalin Marinas * agents in the system and can safely skip the __sync_icache_dcache() call as,
208*5a00bfd6SRyan Roberts * like __set_ptes(), the PTE is never changed from no-exec to exec here.
20966dbd6e6SCatalin Marinas *
21066dbd6e6SCatalin Marinas * Returns whether or not the PTE actually changed.
21166dbd6e6SCatalin Marinas */
__ptep_set_access_flags(struct vm_area_struct * vma,unsigned long address,pte_t * ptep,pte_t entry,int dirty)212*5a00bfd6SRyan Roberts int __ptep_set_access_flags(struct vm_area_struct *vma,
21366dbd6e6SCatalin Marinas unsigned long address, pte_t *ptep,
21466dbd6e6SCatalin Marinas pte_t entry, int dirty)
21566dbd6e6SCatalin Marinas {
2163bbf7157SCatalin Marinas pteval_t old_pteval, pteval;
217*5a00bfd6SRyan Roberts pte_t pte = __ptep_get(ptep);
21866dbd6e6SCatalin Marinas
21920a004e7SWill Deacon if (pte_same(pte, entry))
22066dbd6e6SCatalin Marinas return 0;
22166dbd6e6SCatalin Marinas
22266dbd6e6SCatalin Marinas /* only preserve the access flags and write permission */
22373e86cb0SCatalin Marinas pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
22466dbd6e6SCatalin Marinas
22566dbd6e6SCatalin Marinas /*
22666dbd6e6SCatalin Marinas * Setting the flags must be done atomically to avoid racing with the
2276d332747SCatalin Marinas * hardware update of the access/dirty state. The PTE_RDONLY bit must
2286d332747SCatalin Marinas * be set to the most permissive (lowest value) of *ptep and entry
2296d332747SCatalin Marinas * (calculated as: a & b == ~(~a | ~b)).
23066dbd6e6SCatalin Marinas */
2316d332747SCatalin Marinas pte_val(entry) ^= PTE_RDONLY;
23220a004e7SWill Deacon pteval = pte_val(pte);
2333bbf7157SCatalin Marinas do {
2343bbf7157SCatalin Marinas old_pteval = pteval;
2353bbf7157SCatalin Marinas pteval ^= PTE_RDONLY;
2363bbf7157SCatalin Marinas pteval |= pte_val(entry);
2373bbf7157SCatalin Marinas pteval ^= PTE_RDONLY;
2383bbf7157SCatalin Marinas pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
2393bbf7157SCatalin Marinas } while (pteval != old_pteval);
24066dbd6e6SCatalin Marinas
2416a1bdb17SWill Deacon /* Invalidate a stale read-only entry */
2426a1bdb17SWill Deacon if (dirty)
2436a1bdb17SWill Deacon flush_tlb_page(vma, address);
24466dbd6e6SCatalin Marinas return 1;
24566dbd6e6SCatalin Marinas }
24666dbd6e6SCatalin Marinas
is_el1_instruction_abort(unsigned long esr)2478d56e5c5SAlexandru Elisei static bool is_el1_instruction_abort(unsigned long esr)
2489adeb8e7SLaura Abbott {
2499adeb8e7SLaura Abbott return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
2509adeb8e7SLaura Abbott }
2519adeb8e7SLaura Abbott
is_el1_data_abort(unsigned long esr)2528d56e5c5SAlexandru Elisei static bool is_el1_data_abort(unsigned long esr)
253fcf9dc02SKefeng Wang {
254fcf9dc02SKefeng Wang return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
255fcf9dc02SKefeng Wang }
256fcf9dc02SKefeng Wang
is_el1_permission_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)2578d56e5c5SAlexandru Elisei static inline bool is_el1_permission_fault(unsigned long addr, unsigned long esr,
258dbfe3828SAnshuman Khandual struct pt_regs *regs)
259b824b930SStephen Boyd {
2608d56e5c5SAlexandru Elisei if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
261b824b930SStephen Boyd return false;
262fcf9dc02SKefeng Wang
263b824b930SStephen Boyd if (esr_fsc_is_permission_fault(esr))
264b824b930SStephen Boyd return true;
265b824b930SStephen Boyd
266b824b930SStephen Boyd if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
267b824b930SStephen Boyd return esr_fsc_is_translation_fault(esr) &&
268356607f2SAndrey Konovalov (regs->pstate & PSR_PAN_BIT);
269b824b930SStephen Boyd
270b824b930SStephen Boyd return false;
271b824b930SStephen Boyd }
272b824b930SStephen Boyd
is_spurious_el1_translation_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)273b824b930SStephen Boyd static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
274b824b930SStephen Boyd unsigned long esr,
27542f91093SWill Deacon struct pt_regs *regs)
2768d56e5c5SAlexandru Elisei {
27742f91093SWill Deacon unsigned long flags;
27842f91093SWill Deacon u64 par, dfsc;
27942f91093SWill Deacon
28042f91093SWill Deacon if (!is_el1_data_abort(esr) || !esr_fsc_is_translation_fault(esr))
28142f91093SWill Deacon return false;
282fcf9dc02SKefeng Wang
28342f91093SWill Deacon local_irq_save(flags);
28442f91093SWill Deacon asm volatile("at s1e1r, %0" :: "r" (addr));
28542f91093SWill Deacon isb();
28642f91093SWill Deacon par = read_sysreg_par();
28742f91093SWill Deacon local_irq_restore(flags);
28842f91093SWill Deacon
28996d389caSRob Herring /*
29042f91093SWill Deacon * If we now have a valid translation, treat the translation fault as
29142f91093SWill Deacon * spurious.
29238137335SMark Rutland */
29338137335SMark Rutland if (!(par & SYS_PAR_EL1_F))
29438137335SMark Rutland return true;
29538137335SMark Rutland
29642f91093SWill Deacon /*
29738137335SMark Rutland * If we got a different type of fault from the AT instruction,
29842f91093SWill Deacon * treat the translation fault as spurious.
29942f91093SWill Deacon */
30042f91093SWill Deacon dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
30142f91093SWill Deacon return !esr_fsc_is_translation_fault(dfsc);
30242f91093SWill Deacon }
303308c5156SMark Rutland
die_kernel_fault(const char * msg,unsigned long addr,unsigned long esr,struct pt_regs * regs)30442f91093SWill Deacon static void die_kernel_fault(const char *msg, unsigned long addr,
30542f91093SWill Deacon unsigned long esr, struct pt_regs *regs)
30642f91093SWill Deacon {
307c870f14eSMark Rutland bust_spinlocks(1);
3088d56e5c5SAlexandru Elisei
309c870f14eSMark Rutland pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
310c870f14eSMark Rutland addr);
311c870f14eSMark Rutland
312c870f14eSMark Rutland kasan_non_canonical_hook(addr);
313c870f14eSMark Rutland
314c870f14eSMark Rutland mem_abort_decode(esr);
31507b742a4SMark Rutland
31607b742a4SMark Rutland show_pte(addr);
317c870f14eSMark Rutland die("Oops", regs, esr);
318c870f14eSMark Rutland bust_spinlocks(0);
319c870f14eSMark Rutland make_task_dead(SIGKILL);
320c870f14eSMark Rutland }
321c870f14eSMark Rutland
3220e25498fSEric W. Biederman #ifdef CONFIG_KASAN_HW_TAGS
report_tag_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)323c870f14eSMark Rutland static void report_tag_fault(unsigned long addr, unsigned long esr,
324c870f14eSMark Rutland struct pt_regs *regs)
3254291e9eeSAndrey Konovalov {
3268d56e5c5SAlexandru Elisei /*
32798c970daSVincenzo Frascino * SAS bits aren't set for all faults reported in EL1, so we can't
32898c970daSVincenzo Frascino * find out access size.
3294291e9eeSAndrey Konovalov */
3304291e9eeSAndrey Konovalov bool is_write = !!(esr & ESR_ELx_WNR);
3314291e9eeSAndrey Konovalov kasan_report((void *)addr, 0, is_write, regs->pc);
3324291e9eeSAndrey Konovalov }
33376721503SMark Rutland #else
334bb6e04a1SArnd Bergmann /* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
report_tag_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)33598c970daSVincenzo Frascino static inline void report_tag_fault(unsigned long addr, unsigned long esr,
3364291e9eeSAndrey Konovalov struct pt_regs *regs) { }
3374291e9eeSAndrey Konovalov #endif
3388d56e5c5SAlexandru Elisei
do_tag_recovery(unsigned long addr,unsigned long esr,struct pt_regs * regs)3394291e9eeSAndrey Konovalov static void do_tag_recovery(unsigned long addr, unsigned long esr,
3404291e9eeSAndrey Konovalov struct pt_regs *regs)
34198c970daSVincenzo Frascino {
3428d56e5c5SAlexandru Elisei
34398c970daSVincenzo Frascino report_tag_fault(addr, esr, regs);
34498c970daSVincenzo Frascino
34598c970daSVincenzo Frascino /*
34698c970daSVincenzo Frascino * Disable MTE Tag Checking on the local CPU for the current EL.
34798c970daSVincenzo Frascino * It will be done lazily on the other CPUs when they will hit a
34898c970daSVincenzo Frascino * tag fault.
34998c970daSVincenzo Frascino */
35098c970daSVincenzo Frascino sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
35198c970daSVincenzo Frascino SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE));
35298c970daSVincenzo Frascino isb();
353bc249e37SMark Brown }
354bc249e37SMark Brown
is_el1_mte_sync_tag_check_fault(unsigned long esr)35598c970daSVincenzo Frascino static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
35698c970daSVincenzo Frascino {
35798c970daSVincenzo Frascino unsigned long fsc = esr & ESR_ELx_FSC;
3588d56e5c5SAlexandru Elisei
35998c970daSVincenzo Frascino if (!is_el1_data_abort(esr))
3608d56e5c5SAlexandru Elisei return false;
36198c970daSVincenzo Frascino
362fcf9dc02SKefeng Wang if (fsc == ESR_ELx_FSC_MTE)
36398c970daSVincenzo Frascino return true;
36498c970daSVincenzo Frascino
36598c970daSVincenzo Frascino return false;
36698c970daSVincenzo Frascino }
36798c970daSVincenzo Frascino
__do_kernel_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)36898c970daSVincenzo Frascino static void __do_kernel_fault(unsigned long addr, unsigned long esr,
36998c970daSVincenzo Frascino struct pt_regs *regs)
37098c970daSVincenzo Frascino {
3710bb1fbffSMark Rutland const char *msg;
3720bb1fbffSMark Rutland
3730bb1fbffSMark Rutland /*
3740bb1fbffSMark Rutland * Are we prepared to handle this kernel fault?
3750bb1fbffSMark Rutland * We are almost certainly not prepared to handle instruction faults.
3768d56e5c5SAlexandru Elisei */
37767ce16ecSKristina Martsenko if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
3781d18c47cSCatalin Marinas return;
379b824b930SStephen Boyd
380b824b930SStephen Boyd if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
3811d18c47cSCatalin Marinas "Ignoring spurious kernel translation fault at virtual address %016lx\n", addr))
3821d18c47cSCatalin Marinas return;
3839adeb8e7SLaura Abbott
3841d18c47cSCatalin Marinas if (is_el1_mte_sync_tag_check_fault(esr)) {
3859adeb8e7SLaura Abbott do_tag_recovery(addr, esr, regs);
3861d18c47cSCatalin Marinas
3871d18c47cSCatalin Marinas return;
38842f91093SWill Deacon }
38942f91093SWill Deacon
39042f91093SWill Deacon if (is_el1_permission_fault(addr, esr, regs)) {
39142f91093SWill Deacon if (esr & ESR_ELx_WNR)
39298c970daSVincenzo Frascino msg = "write to read-only memory";
39398c970daSVincenzo Frascino else if (is_el1_instruction_abort(esr))
39498c970daSVincenzo Frascino msg = "execute from non-executable memory";
39598c970daSVincenzo Frascino else
39698c970daSVincenzo Frascino msg = "read from unreadable memory";
39798c970daSVincenzo Frascino } else if (addr < PAGE_SIZE) {
398dbfe3828SAnshuman Khandual msg = "NULL pointer dereference";
399b824b930SStephen Boyd } else {
400b824b930SStephen Boyd if (esr_fsc_is_translation_fault(esr) &&
401e44ec4a3SXiang Zheng kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
402e44ec4a3SXiang Zheng return;
403b824b930SStephen Boyd
404b824b930SStephen Boyd msg = "paging request";
405b824b930SStephen Boyd }
406b824b930SStephen Boyd
407b824b930SStephen Boyd if (efi_runtime_fixup_exception(regs, msg))
4080bb1fbffSMark Rutland return;
4090bb1fbffSMark Rutland
410840b2398SMarco Elver die_kernel_fault(msg, addr, esr, regs);
411840b2398SMarco Elver }
412b824b930SStephen Boyd
set_thread_esr(unsigned long address,unsigned long esr)413b824b930SStephen Boyd static void set_thread_esr(unsigned long address, unsigned long esr)
414b824b930SStephen Boyd {
415e8dfdf31SArd Biesheuvel current->thread.fault_address = address;
416e8dfdf31SArd Biesheuvel
417e8dfdf31SArd Biesheuvel /*
418c870f14eSMark Rutland * If the faulting address is in the kernel, we must sanitize the ESR.
4191d18c47cSCatalin Marinas * From userspace's point of view, kernel-only mappings don't exist
4201d18c47cSCatalin Marinas * at all, so we report them as level 0 translation faults.
4218d56e5c5SAlexandru Elisei * (This is not quite the way that "no mapping there at all" behaves:
4221d18c47cSCatalin Marinas * an alignment fault not caused by the memory type would take
423f29ad209SEric W. Biederman * precedence over translation fault for a real access to empty
424cc198460SPeter Maydell * space. Unfortunately we can't easily distinguish "alignment fault
425cc198460SPeter Maydell * not caused by memory type" from "alignment fault caused by memory
426cc198460SPeter Maydell * type", so we ignore this wrinkle and just return the translation
427cc198460SPeter Maydell * fault.)
428cc198460SPeter Maydell */
429cc198460SPeter Maydell if (!is_ttbr0_addr(current->thread.fault_address)) {
430cc198460SPeter Maydell switch (ESR_ELx_EC(esr)) {
431cc198460SPeter Maydell case ESR_ELx_EC_DABT_LOW:
432cc198460SPeter Maydell /*
433cc198460SPeter Maydell * These bits provide only information about the
434cc198460SPeter Maydell * faulting instruction, which userspace knows already.
435cc198460SPeter Maydell * We explicitly clear bits which are architecturally
436cc198460SPeter Maydell * RES0 in case they are given meanings in future.
437356607f2SAndrey Konovalov * We always report the ESR as if the fault was taken
438cc198460SPeter Maydell * to EL1 and so ISV and the bits in ISS[23:14] are
439cc198460SPeter Maydell * clear. (In fact it always will be a fault to EL1.)
440cc198460SPeter Maydell */
441cc198460SPeter Maydell esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
442cc198460SPeter Maydell ESR_ELx_CM | ESR_ELx_WNR;
443cc198460SPeter Maydell esr |= ESR_ELx_FSC_FAULT;
444cc198460SPeter Maydell break;
445cc198460SPeter Maydell case ESR_ELx_EC_IABT_LOW:
446cc198460SPeter Maydell /*
447cc198460SPeter Maydell * Claim a level 0 translation fault.
448cc198460SPeter Maydell * All other bits are architecturally RES0 for faults
449cc198460SPeter Maydell * reported with that DFSC value, so we clear them.
450cc198460SPeter Maydell */
451cc198460SPeter Maydell esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
452cc198460SPeter Maydell esr |= ESR_ELx_FSC_FAULT;
453cc198460SPeter Maydell break;
454cc198460SPeter Maydell default:
455cc198460SPeter Maydell /*
456cc198460SPeter Maydell * This should never happen (entry.S only brings us
457cc198460SPeter Maydell * into this code for insn and data aborts from a lower
458cc198460SPeter Maydell * exception level). Fail safe by not providing an ESR
459cc198460SPeter Maydell * context record at all.
460cc198460SPeter Maydell */
461cc198460SPeter Maydell WARN(1, "ESR 0x%lx is not DABT or IABT from EL0\n", esr);
462cc198460SPeter Maydell esr = 0;
463cc198460SPeter Maydell break;
464cc198460SPeter Maydell }
465cc198460SPeter Maydell }
466cc198460SPeter Maydell
467cc198460SPeter Maydell current->thread.fault_code = esr;
468cc198460SPeter Maydell }
4698d56e5c5SAlexandru Elisei
do_bad_area(unsigned long far,unsigned long esr,struct pt_regs * regs)470cc198460SPeter Maydell static void do_bad_area(unsigned long far, unsigned long esr,
471cc198460SPeter Maydell struct pt_regs *regs)
472cc198460SPeter Maydell {
473cc198460SPeter Maydell unsigned long addr = untagged_addr(far);
474cc198460SPeter Maydell
47592ff0674SWill Deacon /*
4761d18c47cSCatalin Marinas * If we are in kernel mode at this point, we have no context to
4771d18c47cSCatalin Marinas * handle this fault with.
4788d56e5c5SAlexandru Elisei */
479dceec3ffSPeter Collingbourne if (user_mode(regs)) {
4801d18c47cSCatalin Marinas const struct fault_info *inf = esr_to_fault_info(esr);
481dceec3ffSPeter Collingbourne
482dceec3ffSPeter Collingbourne set_thread_esr(addr, esr);
4831d18c47cSCatalin Marinas arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
4841d18c47cSCatalin Marinas } else {
4851d18c47cSCatalin Marinas __do_kernel_fault(addr, esr, regs);
4861d18c47cSCatalin Marinas }
48709a6adf5SVictor Kamensky }
48892ff0674SWill Deacon
4893eb0f519SEric W. Biederman #define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000)
490effb093aSEric W. Biederman #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000)
491dceec3ffSPeter Collingbourne
__do_page_fault(struct mm_struct * mm,struct vm_area_struct * vma,unsigned long addr,unsigned int mm_flags,unsigned long vm_flags,struct pt_regs * regs)49292ff0674SWill Deacon static vm_fault_t __do_page_fault(struct mm_struct *mm,
49367ce16ecSKristina Martsenko struct vm_area_struct *vma, unsigned long addr,
4941d18c47cSCatalin Marinas unsigned int mm_flags, unsigned long vm_flags,
49592ff0674SWill Deacon struct pt_regs *regs)
4961d18c47cSCatalin Marinas {
497d91d5808SMin-Hua Chen /*
498d91d5808SMin-Hua Chen * Ok, we have a good vm_area for this memory access, so we can handle
4991d18c47cSCatalin Marinas * it.
500ae870a68SLinus Torvalds * Check that the permissions on the VMA allow for the fault which
501ae870a68SLinus Torvalds * occurred.
5026a1bb025SPeter Xu */
5036a1bb025SPeter Xu if (!(vma->vm_flags & vm_flags))
5041d18c47cSCatalin Marinas return VM_FAULT_BADACCESS;
5051d18c47cSCatalin Marinas return handle_mm_fault(vma, addr, mm_flags, regs);
5061d18c47cSCatalin Marinas }
5071d18c47cSCatalin Marinas
is_el0_instruction_abort(unsigned long esr)508db6f4106SWill Deacon static bool is_el0_instruction_abort(unsigned long esr)
509cab15ce6SCatalin Marinas {
510db6f4106SWill Deacon return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
5114745224bSAnshuman Khandual }
5124745224bSAnshuman Khandual
51384c5e23eSGavin Shan /*
5141d18c47cSCatalin Marinas * Note: not valid for EL1 DC IVAC, but we never use that such that it
5151d18c47cSCatalin Marinas * should fault. EL0 cannot issue DC IVAC (undef).
5168d56e5c5SAlexandru Elisei */
is_write_abort(unsigned long esr)517541ec870SMark Rutland static bool is_write_abort(unsigned long esr)
518541ec870SMark Rutland {
519541ec870SMark Rutland return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
520541ec870SMark Rutland }
521c49bd02fSAnshuman Khandual
do_page_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)522c49bd02fSAnshuman Khandual static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
523c49bd02fSAnshuman Khandual struct pt_regs *regs)
524c49bd02fSAnshuman Khandual {
5258d56e5c5SAlexandru Elisei const struct fault_info *inf;
526c49bd02fSAnshuman Khandual struct mm_struct *mm = current->mm;
527c49bd02fSAnshuman Khandual vm_fault_t fault;
528c49bd02fSAnshuman Khandual unsigned long vm_flags;
529c49bd02fSAnshuman Khandual unsigned int mm_flags = FAULT_FLAG_DEFAULT;
5308d56e5c5SAlexandru Elisei unsigned long addr = untagged_addr(far);
5311d18c47cSCatalin Marinas struct vm_area_struct *vma;
5321d18c47cSCatalin Marinas
5332d2837faSEric W. Biederman if (kprobe_page_fault(regs, esr))
53461681036SAnshuman Khandual return 0;
5356a1bb025SPeter Xu
53618107f8aSVladimir Murzin /*
537dde16072SPeter Xu * If we're in an interrupt or have no user context, we must not take
538dceec3ffSPeter Collingbourne * the fault.
539cd7f176aSSuren Baghdasaryan */
540db6f4106SWill Deacon if (faulthandler_disabled() || !mm)
541b98cca44SAnshuman Khandual goto no_context;
5422dd0e8d2SSandeepa Prabhu
5432dd0e8d2SSandeepa Prabhu if (user_mode(regs))
5441d18c47cSCatalin Marinas mm_flags |= FAULT_FLAG_USER;
5451d18c47cSCatalin Marinas
5461d18c47cSCatalin Marinas /*
5471d18c47cSCatalin Marinas * vm_flags tells us what bits we must have in vma->vm_flags
54870ffdb93SDavid Hildenbrand * for the fault to be benign, __do_page_fault() would check
5491d18c47cSCatalin Marinas * vma->vm_flags & vm_flags and returns an error if the
5501d18c47cSCatalin Marinas * intersection is empty
551759496baSJohannes Weiner */
552759496baSJohannes Weiner if (is_el0_instruction_abort(esr)) {
553759496baSJohannes Weiner /* It was exec fault */
55418107f8aSVladimir Murzin vm_flags = VM_EXEC;
55518107f8aSVladimir Murzin mm_flags |= FAULT_FLAG_INSTRUCTION;
55618107f8aSVladimir Murzin } else if (is_write_abort(esr)) {
55718107f8aSVladimir Murzin /* It was write fault */
55818107f8aSVladimir Murzin vm_flags = VM_WRITE;
55918107f8aSVladimir Murzin mm_flags |= FAULT_FLAG_WRITE;
560541ec870SMark Rutland } else {
56118107f8aSVladimir Murzin /* It was read fault */
562759496baSJohannes Weiner vm_flags = VM_READ;
56301de1776SAnshuman Khandual /* Write implies read */
564c49bd02fSAnshuman Khandual vm_flags |= VM_WRITE;
56518107f8aSVladimir Murzin /* If EPAN is absent then exec implies read */
566759496baSJohannes Weiner if (!alternative_has_cap_unlikely(ARM64_HAS_EPAN))
567759496baSJohannes Weiner vm_flags |= VM_EXEC;
56818107f8aSVladimir Murzin }
56918107f8aSVladimir Murzin
57018107f8aSVladimir Murzin if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
57118107f8aSVladimir Murzin if (is_el1_instruction_abort(esr))
57218107f8aSVladimir Murzin die_kernel_fault("execution of user memory",
57318107f8aSVladimir Murzin addr, esr, regs);
5744e00f1d9SMark Rutland
57518107f8aSVladimir Murzin if (!search_exception_tables(regs->pc))
576759496baSJohannes Weiner die_kernel_fault("access to user memory outside uaccess routines",
577759496baSJohannes Weiner addr, esr, regs);
578356607f2SAndrey Konovalov }
5799adeb8e7SLaura Abbott
580c870f14eSMark Rutland perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
581c870f14eSMark Rutland
5829adeb8e7SLaura Abbott if (!(mm_flags & FAULT_FLAG_USER))
58357f4959bSJames Morse goto lock_mmap;
584c870f14eSMark Rutland
585c870f14eSMark Rutland vma = lock_vma_under_rcu(mm, addr);
58657f4959bSJames Morse if (!vma)
587338d4f49SJames Morse goto lock_mmap;
5880e3a9026SPunit Agrawal
5890e3a9026SPunit Agrawal if (!(vma->vm_flags & vm_flags)) {
590cd7f176aSSuren Baghdasaryan vma_end_read(vma);
591cd7f176aSSuren Baghdasaryan goto lock_mmap;
592cd7f176aSSuren Baghdasaryan }
593cd7f176aSSuren Baghdasaryan fault = handle_mm_fault(vma, addr, mm_flags | FAULT_FLAG_VMA_LOCK, regs);
594cd7f176aSSuren Baghdasaryan if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
595cd7f176aSSuren Baghdasaryan vma_end_read(vma);
596cd7f176aSSuren Baghdasaryan
597cd7f176aSSuren Baghdasaryan if (!(fault & VM_FAULT_RETRY)) {
598cd7f176aSSuren Baghdasaryan count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
599cd7f176aSSuren Baghdasaryan goto done;
600cd7f176aSSuren Baghdasaryan }
6010e2aba69SJisheng Zhang count_vm_vma_lock_event(VMA_LOCK_RETRY);
6024089eef0SSuren Baghdasaryan if (fault & VM_FAULT_MAJOR)
603cd7f176aSSuren Baghdasaryan mm_flags |= FAULT_FLAG_TRIED;
604cd7f176aSSuren Baghdasaryan
605cd7f176aSSuren Baghdasaryan /* Quick path to respond to signals */
606cd7f176aSSuren Baghdasaryan if (fault_signal_pending(fault, regs)) {
607cd7f176aSSuren Baghdasaryan if (!user_mode(regs))
608cd7f176aSSuren Baghdasaryan goto no_context;
609cd7f176aSSuren Baghdasaryan return 0;
61046e714c7SSuren Baghdasaryan }
61146e714c7SSuren Baghdasaryan lock_mmap:
612cd7f176aSSuren Baghdasaryan
613cd7f176aSSuren Baghdasaryan retry:
614cd7f176aSSuren Baghdasaryan vma = lock_mm_and_find_vma(mm, addr, regs);
615cd7f176aSSuren Baghdasaryan if (unlikely(!vma)) {
616cd7f176aSSuren Baghdasaryan fault = VM_FAULT_BADMAP;
617cd7f176aSSuren Baghdasaryan goto done;
618cd7f176aSSuren Baghdasaryan }
619cd7f176aSSuren Baghdasaryan
620ae870a68SLinus Torvalds fault = __do_page_fault(mm, vma, addr, mm_flags, vm_flags, regs);
6211d18c47cSCatalin Marinas
622ae870a68SLinus Torvalds /* Quick path to respond to signals */
623ae870a68SLinus Torvalds if (fault_signal_pending(fault, regs)) {
624ae870a68SLinus Torvalds if (!user_mode(regs))
625ae870a68SLinus Torvalds goto no_context;
6261d18c47cSCatalin Marinas return 0;
6271d18c47cSCatalin Marinas }
628ae870a68SLinus Torvalds
6291d18c47cSCatalin Marinas /* The fault is fully completed (including releasing mmap lock) */
630b502f038SPeter Xu if (fault & VM_FAULT_COMPLETED)
631b502f038SPeter Xu return 0;
632289d07a2SMark Rutland
633289d07a2SMark Rutland if (fault & VM_FAULT_RETRY) {
6341d18c47cSCatalin Marinas mm_flags |= FAULT_FLAG_TRIED;
635289d07a2SMark Rutland goto retry;
6361d18c47cSCatalin Marinas }
637d9272525SPeter Xu mmap_read_unlock(mm);
638d9272525SPeter Xu
639d9272525SPeter Xu done:
640d9272525SPeter Xu /*
641b502f038SPeter Xu * Handle the "normal" (no error) case first.
6420e3a9026SPunit Agrawal */
6430e3a9026SPunit Agrawal if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
6440e3a9026SPunit Agrawal VM_FAULT_BADACCESS))))
645d8ed45c5SMichel Lespinasse return 0;
6460e3a9026SPunit Agrawal
647cd7f176aSSuren Baghdasaryan /*
6480e3a9026SPunit Agrawal * If we are in kernel mode at this point, we have no context to
6490e3a9026SPunit Agrawal * handle this fault with.
6500e3a9026SPunit Agrawal */
6510e3a9026SPunit Agrawal if (!user_mode(regs))
6526a1bb025SPeter Xu goto no_context;
6531d18c47cSCatalin Marinas
6541d18c47cSCatalin Marinas if (fault & VM_FAULT_OOM) {
65587134102SJohannes Weiner /*
65687134102SJohannes Weiner * We ran out of memory, call the OOM killer, and return to
65787134102SJohannes Weiner * userspace (which will retry the fault, or kill us if we got
65887134102SJohannes Weiner * oom-killed).
65987134102SJohannes Weiner */
66087134102SJohannes Weiner pagefault_out_of_memory();
66187134102SJohannes Weiner return 0;
6621d18c47cSCatalin Marinas }
6631d18c47cSCatalin Marinas
6641d18c47cSCatalin Marinas inf = esr_to_fault_info(esr);
6651d18c47cSCatalin Marinas set_thread_esr(addr, esr);
6661d18c47cSCatalin Marinas if (fault & VM_FAULT_SIGBUS) {
6671d18c47cSCatalin Marinas /*
6681d18c47cSCatalin Marinas * We had some memory, but were unable to successfully fix up
6691d18c47cSCatalin Marinas * this page fault.
6701d18c47cSCatalin Marinas */
6711d18c47cSCatalin Marinas arm64_force_sig_fault(SIGBUS, BUS_ADRERR, far, inf->name);
6722d2837faSEric W. Biederman } else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
673559d8d91SEric W. Biederman unsigned int lsb;
6741d18c47cSCatalin Marinas
6751d18c47cSCatalin Marinas lsb = PAGE_SHIFT;
6761d18c47cSCatalin Marinas if (fault & VM_FAULT_HWPOISON_LARGE)
6771d18c47cSCatalin Marinas lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
6781d18c47cSCatalin Marinas
679dceec3ffSPeter Collingbourne arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name);
6809ea3a974SEric W. Biederman } else {
6819ea3a974SEric W. Biederman /*
68292ff0674SWill Deacon * Something tried to access memory that isn't in our memory
6839ea3a974SEric W. Biederman * map.
6849ea3a974SEric W. Biederman */
6859ea3a974SEric W. Biederman arm64_force_sig_fault(SIGSEGV,
68692ff0674SWill Deacon fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
687dceec3ffSPeter Collingbourne far, inf->name);
6881d18c47cSCatalin Marinas }
6891d18c47cSCatalin Marinas
6901d18c47cSCatalin Marinas return 0;
6911d18c47cSCatalin Marinas
6921d18c47cSCatalin Marinas no_context:
693feca355bSEric W. Biederman __do_kernel_fault(addr, esr, regs);
694feca355bSEric W. Biederman return 0;
695dceec3ffSPeter Collingbourne }
6961d18c47cSCatalin Marinas
do_translation_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)6971d18c47cSCatalin Marinas static int __kprobes do_translation_fault(unsigned long far,
6981d18c47cSCatalin Marinas unsigned long esr,
6991d18c47cSCatalin Marinas struct pt_regs *regs)
7001d18c47cSCatalin Marinas {
70167ce16ecSKristina Martsenko unsigned long addr = untagged_addr(far);
7021d18c47cSCatalin Marinas
7031d18c47cSCatalin Marinas if (is_ttbr0_addr(addr))
7041d18c47cSCatalin Marinas return do_page_fault(far, esr, regs);
705dceec3ffSPeter Collingbourne
7068d56e5c5SAlexandru Elisei do_bad_area(far, esr, regs);
7071d18c47cSCatalin Marinas return 0;
7081d18c47cSCatalin Marinas }
709dceec3ffSPeter Collingbourne
do_alignment_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)7101d18c47cSCatalin Marinas static int do_alignment_fault(unsigned long far, unsigned long esr,
711dceec3ffSPeter Collingbourne struct pt_regs *regs)
712dceec3ffSPeter Collingbourne {
713dceec3ffSPeter Collingbourne if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) &&
714dceec3ffSPeter Collingbourne compat_user_mode(regs))
7151d18c47cSCatalin Marinas return do_compat_alignment_fixup(far, regs);
7161d18c47cSCatalin Marinas do_bad_area(far, esr, regs);
7171d18c47cSCatalin Marinas return 0;
7188d56e5c5SAlexandru Elisei }
71952d7523dSEunTaik Lee
do_bad(unsigned long far,unsigned long esr,struct pt_regs * regs)72052d7523dSEunTaik Lee static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
7213fc24ef3SArd Biesheuvel {
7223fc24ef3SArd Biesheuvel return 1; /* "fault" */
7233fc24ef3SArd Biesheuvel }
724dceec3ffSPeter Collingbourne
do_sea(unsigned long far,unsigned long esr,struct pt_regs * regs)72552d7523dSEunTaik Lee static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
72652d7523dSEunTaik Lee {
72752d7523dSEunTaik Lee const struct fault_info *inf;
7288d56e5c5SAlexandru Elisei unsigned long siaddr;
7291d18c47cSCatalin Marinas
730f67d5c4fSWill Deacon inf = esr_to_fault_info(esr);
7311d18c47cSCatalin Marinas
7321d18c47cSCatalin Marinas if (user_mode(regs) && apei_claim_sea(regs) == 0) {
7338d56e5c5SAlexandru Elisei /*
73432015c23STyler Baicar * APEI claimed this as a firmware-first notification.
73532015c23STyler Baicar * Some processing deferred to task_work before ret_to_user().
736dceec3ffSPeter Collingbourne */
73732015c23STyler Baicar return 0;
73832015c23STyler Baicar }
73932015c23STyler Baicar
7408fcc4ae6SJames Morse if (esr & ESR_ELx_FnV) {
7417edda088STyler Baicar siaddr = 0;
7428fcc4ae6SJames Morse } else {
7438fcc4ae6SJames Morse /*
7447edda088STyler Baicar * The architecture specifies that the tag bits of FAR_EL1 are
7458fcc4ae6SJames Morse * UNKNOWN for synchronous external aborts. Mask them out now
7468fcc4ae6SJames Morse * so that userspace doesn't see them.
7477edda088STyler Baicar */
748dceec3ffSPeter Collingbourne siaddr = untagged_addr(far);
749dceec3ffSPeter Collingbourne }
750dceec3ffSPeter Collingbourne arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
751dceec3ffSPeter Collingbourne
752dceec3ffSPeter Collingbourne return 0;
753dceec3ffSPeter Collingbourne }
754dceec3ffSPeter Collingbourne
do_tag_check_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)755dceec3ffSPeter Collingbourne static int do_tag_check_fault(unsigned long far, unsigned long esr,
756dceec3ffSPeter Collingbourne struct pt_regs *regs)
757dceec3ffSPeter Collingbourne {
7586fa998e8SEric W. Biederman /*
75932015c23STyler Baicar * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
760faa75e14SDongjiu Geng * for tag check faults. Set them to corresponding bits in the untagged
76132015c23STyler Baicar * address.
76232015c23STyler Baicar */
7638d56e5c5SAlexandru Elisei far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);
764637ec831SVincenzo Frascino do_bad_area(far, esr, regs);
765637ec831SVincenzo Frascino return 0;
766dceec3ffSPeter Collingbourne }
7673ed86b9aSAndrey Konovalov
7683ed86b9aSAndrey Konovalov static const struct fault_info fault_info[] = {
7693ed86b9aSAndrey Konovalov { do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" },
770dceec3ffSPeter Collingbourne { do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" },
7713ed86b9aSAndrey Konovalov { do_bad, SIGKILL, SI_KERNEL, "level 2 address size fault" },
772dceec3ffSPeter Collingbourne { do_bad, SIGKILL, SI_KERNEL, "level 3 address size fault" },
773637ec831SVincenzo Frascino { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
774637ec831SVincenzo Frascino { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
775637ec831SVincenzo Frascino { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
77609a6adf5SVictor Kamensky { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
777af40ff68SDave Martin { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 0 access flag fault" },
778af40ff68SDave Martin { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
779af40ff68SDave Martin { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
780af40ff68SDave Martin { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
7817f73f7aeSWill Deacon { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 0 permission fault" },
7821d18c47cSCatalin Marinas { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
7831d18c47cSCatalin Marinas { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
784760bfb47SWill Deacon { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
785af40ff68SDave Martin { do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" },
786084bd298SSteve Capper { do_tag_check_fault, SIGSEGV, SEGV_MTESERR, "synchronous tag check fault" },
787084bd298SSteve Capper { do_bad, SIGKILL, SI_KERNEL, "unknown 18" },
7881d18c47cSCatalin Marinas { do_sea, SIGKILL, SI_KERNEL, "level -1 (translation table walk)" },
789af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" },
790084bd298SSteve Capper { do_sea, SIGKILL, SI_KERNEL, "level 1 (translation table walk)" },
791084bd298SSteve Capper { do_sea, SIGKILL, SI_KERNEL, "level 2 (translation table walk)" },
7921d18c47cSCatalin Marinas { do_sea, SIGKILL, SI_KERNEL, "level 3 (translation table walk)" },
793af40ff68SDave Martin { do_sea, SIGBUS, BUS_OBJERR, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
794637ec831SVincenzo Frascino { do_bad, SIGKILL, SI_KERNEL, "unknown 25" },
795af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 26" },
796af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level -1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
797af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
798af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
799af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
800af40ff68SDave Martin { do_sea, SIGKILL, SI_KERNEL, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
801af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 32" },
802af40ff68SDave Martin { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
803af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 34" },
804af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 35" },
805af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 36" },
806af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 37" },
807af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 38" },
808af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 39" },
809af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 40" },
81052d7523dSEunTaik Lee { do_bad, SIGKILL, SI_KERNEL, "level -1 address size fault" },
811af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 42" },
812af40ff68SDave Martin { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level -1 translation fault" },
813af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 44" },
814af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 45" },
815af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 46" },
816af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 47" },
817af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "TLB conflict abort" },
818af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "Unsupported atomic hardware update fault" },
819af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 50" },
820af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 51" },
821af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "implementation fault (lockdown abort)" },
822af40ff68SDave Martin { do_bad, SIGBUS, BUS_OBJERR, "implementation fault (unsupported exclusive)" },
823af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 54" },
824af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 55" },
825af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 56" },
826af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 57" },
827af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
828af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
829af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
830af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
831af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
832af40ff68SDave Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
833af40ff68SDave Martin };
834af40ff68SDave Martin
do_mem_abort(unsigned long far,unsigned long esr,struct pt_regs * regs)835af40ff68SDave Martin void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs)
836af40ff68SDave Martin {
837af40ff68SDave Martin const struct fault_info *inf = esr_to_fault_info(esr);
838af40ff68SDave Martin unsigned long addr = untagged_addr(far);
839af40ff68SDave Martin
840af40ff68SDave Martin if (!inf->fn(far, esr, regs))
8411d18c47cSCatalin Marinas return;
8421d18c47cSCatalin Marinas
8438d56e5c5SAlexandru Elisei if (!user_mode(regs))
8441d18c47cSCatalin Marinas die_kernel_fault(inf->name, addr, esr, regs);
84509a6adf5SVictor Kamensky
846dceec3ffSPeter Collingbourne /*
8471d18c47cSCatalin Marinas * At this point we have an unrecognized fault type whose tag bits may
848dceec3ffSPeter Collingbourne * have been defined as UNKNOWN. Therefore we only expose the untagged
8491d18c47cSCatalin Marinas * address to the signal handler.
8501d18c47cSCatalin Marinas */
8516f6cfa58SMark Rutland arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
8526f6cfa58SMark Rutland }
85342dbf54eSMark Rutland NOKPROBE_SYMBOL(do_mem_abort);
854dceec3ffSPeter Collingbourne
do_sp_pc_abort(unsigned long addr,unsigned long esr,struct pt_regs * regs)855dceec3ffSPeter Collingbourne void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
856dceec3ffSPeter Collingbourne {
857dceec3ffSPeter Collingbourne arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
858dceec3ffSPeter Collingbourne addr, esr);
859dceec3ffSPeter Collingbourne }
8601d18c47cSCatalin Marinas NOKPROBE_SYMBOL(do_sp_pc_abort);
861b6e43c0eSJames Morse
8621d18c47cSCatalin Marinas /*
8638d56e5c5SAlexandru Elisei * __refdata because early_brk64 is __init, but the reference to it is
8640f15adbbSWill Deacon * clobbered at arch_initcall time.
865dceec3ffSPeter Collingbourne * See traps.c and debug-monitors.c:debug_traps_init().
866dceec3ffSPeter Collingbourne */
8671d18c47cSCatalin Marinas static struct fault_info __refdata debug_fault_info[] = {
868b6e43c0eSJames Morse { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
8691d18c47cSCatalin Marinas { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
8709fb7410fSDave P Martin { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
8719fb7410fSDave P Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 3" },
8729fb7410fSDave P Martin { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
8739fb7410fSDave P Martin { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" },
8749fb7410fSDave P Martin { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
8759fb7410fSDave P Martin { do_bad, SIGKILL, SI_KERNEL, "unknown 7" },
8761d18c47cSCatalin Marinas };
8771d18c47cSCatalin Marinas
hook_debug_fault_code(int nr,int (* fn)(unsigned long,unsigned long,struct pt_regs *),int sig,int code,const char * name)8781d18c47cSCatalin Marinas void __init hook_debug_fault_code(int nr,
879af40ff68SDave Martin int (*fn)(unsigned long, unsigned long, struct pt_regs *),
8801d18c47cSCatalin Marinas int sig, int code, const char *name)
881af40ff68SDave Martin {
8829fb7410fSDave P Martin BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
883af40ff68SDave Martin
8841d18c47cSCatalin Marinas debug_fault_info[nr].fn = fn;
8851d18c47cSCatalin Marinas debug_fault_info[nr].sig = sig;
8861d18c47cSCatalin Marinas debug_fault_info[nr].code = code;
8878d56e5c5SAlexandru Elisei debug_fault_info[nr].name = name;
8881d18c47cSCatalin Marinas }
8891d18c47cSCatalin Marinas
8901d18c47cSCatalin Marinas /*
8911d18c47cSCatalin Marinas * In debug exception context, we explicitly disable preemption despite
8921d18c47cSCatalin Marinas * having interrupts disabled.
8931d18c47cSCatalin Marinas * This serves two purposes: it makes it much less likely that we would
8941d18c47cSCatalin Marinas * accidentally schedule in exception context and it will force a warning
8951d18c47cSCatalin Marinas * if we somehow manage to schedule by accident.
8961d18c47cSCatalin Marinas */
debug_exception_enter(struct pt_regs * regs)8971d18c47cSCatalin Marinas static void debug_exception_enter(struct pt_regs *regs)
898d8bb6718SMasami Hiramatsu {
899d8bb6718SMasami Hiramatsu preempt_disable();
900d8bb6718SMasami Hiramatsu
901d8bb6718SMasami Hiramatsu /* This code is a bit fragile. Test it. */
902d8bb6718SMasami Hiramatsu RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
903d8bb6718SMasami Hiramatsu }
904d8bb6718SMasami Hiramatsu NOKPROBE_SYMBOL(debug_exception_enter);
905d8bb6718SMasami Hiramatsu
debug_exception_exit(struct pt_regs * regs)906d8bb6718SMasami Hiramatsu static void debug_exception_exit(struct pt_regs *regs)
907d8bb6718SMasami Hiramatsu {
908d8bb6718SMasami Hiramatsu preempt_enable_no_resched();
909d8bb6718SMasami Hiramatsu }
910d8bb6718SMasami Hiramatsu NOKPROBE_SYMBOL(debug_exception_exit);
911d8bb6718SMasami Hiramatsu
do_debug_exception(unsigned long addr_if_watchpoint,unsigned long esr,struct pt_regs * regs)912d8bb6718SMasami Hiramatsu void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
913d8bb6718SMasami Hiramatsu struct pt_regs *regs)
914d8bb6718SMasami Hiramatsu {
915d8bb6718SMasami Hiramatsu const struct fault_info *inf = esr_to_debug_fault_info(esr);
916d8bb6718SMasami Hiramatsu unsigned long pc = instruction_pointer(regs);
917d8bb6718SMasami Hiramatsu
918d8bb6718SMasami Hiramatsu debug_exception_enter(regs);
919d8bb6718SMasami Hiramatsu
9208d56e5c5SAlexandru Elisei if (user_mode(regs) && !is_ttbr0_addr(pc))
9211d18c47cSCatalin Marinas arm64_apply_bp_hardening();
9221d18c47cSCatalin Marinas
923359048f9SAnshuman Khandual if (inf->fn(addr_if_watchpoint, esr, regs)) {
924b9a4b9d0SWill Deacon arm64_notify_die(inf->name, regs, inf->sig, inf->code, pc, esr);
9251d18c47cSCatalin Marinas }
926d8bb6718SMasami Hiramatsu
9271d18c47cSCatalin Marinas debug_exception_exit(regs);
928b9a4b9d0SWill Deacon }
9295dfc6ed2SWill Deacon NOKPROBE_SYMBOL(do_debug_exception);
9305dfc6ed2SWill Deacon
93152c6d145SWill Deacon /*
932dceec3ffSPeter Collingbourne * Used during anonymous page fault handling.
9336afedcd2SJames Morse */
vma_alloc_zeroed_movable_folio(struct vm_area_struct * vma,unsigned long vaddr)9341d18c47cSCatalin Marinas struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
935d8bb6718SMasami Hiramatsu unsigned long vaddr)
9361d18c47cSCatalin Marinas {
9372dd0e8d2SSandeepa Prabhu gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO;
938013bb59dSPeter Collingbourne
939013bb59dSPeter Collingbourne /*
940013bb59dSPeter Collingbourne * If the page is mapped with PROT_MTE, initialise the tags at the
941013bb59dSPeter Collingbourne * point of allocation and page zeroing as this is usually faster than
9426bc56a4dSMatthew Wilcox (Oracle) * separate DC ZVA and STGM.
943013bb59dSPeter Collingbourne */
944013bb59dSPeter Collingbourne if (vma->vm_flags & VM_MTE)
945013bb59dSPeter Collingbourne flags |= __GFP_ZEROTAGS;
946013bb59dSPeter Collingbourne
947013bb59dSPeter Collingbourne return vma_alloc_folio(flags, 0, vma, vaddr, false);
948013bb59dSPeter Collingbourne }
949013bb59dSPeter Collingbourne
tag_clear_highpage(struct page * page)950013bb59dSPeter Collingbourne void tag_clear_highpage(struct page *page)
951013bb59dSPeter Collingbourne {
952013bb59dSPeter Collingbourne /* Newly allocated page, shouldn't have been tagged yet */
953013bb59dSPeter Collingbourne WARN_ON_ONCE(!try_page_mte_tagging(page));
954013bb59dSPeter Collingbourne mte_zero_clear_page_tags(page_address(page));
9556bc56a4dSMatthew Wilcox (Oracle) set_page_mte_tagged(page);
956013bb59dSPeter Collingbourne }
957013bb59dSPeter Collingbourne