^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Based on arch/arm/mm/fault.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1995 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 1995-2004 Russell King
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2012 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/acpi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bitfield.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kfence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/page-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/sched/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/acpi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/cmpxchg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/exception.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/daifflags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/debug-monitors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/esr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/mte.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/sysreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/system_misc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <trace/hooks/fault.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct fault_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int (*fn)(unsigned long far, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) int sig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) int code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static const struct fault_info fault_info[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static struct fault_info debug_fault_info[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return fault_info + (esr & ESR_ELx_FSC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return debug_fault_info + DBG_ESR_EVT(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static void data_abort_decode(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) pr_alert("Data abort info:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (esr & ESR_ELx_ISV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) pr_alert(" Access size = %u byte(s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) pr_alert(" SSE = %lu, SRT = %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) pr_alert(" SF = %lu, AR = %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) pr_alert(" CM = %lu, WnR = %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static void mem_abort_decode(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pr_alert("Mem abort info:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) pr_alert(" ESR = 0x%08x\n", esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ESR_ELx_EC(esr), esr_get_class_string(esr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) (esr & ESR_ELx_IL) ? 32 : 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) pr_alert(" SET = %lu, FnV = %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) pr_alert(" EA = %lu, S1PTW = %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (esr_is_data_abort(esr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) data_abort_decode(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* Either init_pg_dir or swapper_pg_dir */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (mm == &init_mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return __pa_symbol(mm->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return (unsigned long)virt_to_phys(mm->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * Dump out the page tables associated with 'addr' in the currently active mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void show_pte(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) pgd_t *pgdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) pgd_t pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (is_ttbr0_addr(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* TTBR0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) mm = current->active_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (mm == &init_mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) pr_alert("[%016lx] user address but active_mm is swapper\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) } else if (is_ttbr1_addr(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* TTBR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) mm = &init_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) pr_alert("[%016lx] address between user and kernel address ranges\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) vabits_actual, mm_to_pgd_phys(mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) pgdp = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) pgd = READ_ONCE(*pgdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) p4d_t *p4dp, p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) pud_t *pudp, pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) pmd_t *pmdp, pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) pte_t *ptep, pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (pgd_none(pgd) || pgd_bad(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) p4dp = p4d_offset(pgdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) p4d = READ_ONCE(*p4dp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) pr_cont(", p4d=%016llx", p4d_val(p4d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (p4d_none(p4d) || p4d_bad(p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pudp = pud_offset(p4dp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) pud = READ_ONCE(*pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) pr_cont(", pud=%016llx", pud_val(pud));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (pud_none(pud) || pud_bad(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) pmdp = pmd_offset(pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) pmd = READ_ONCE(*pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) pr_cont(", pmd=%016llx", pmd_val(pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (pmd_none(pmd) || pmd_bad(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) ptep = pte_offset_map(pmdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) pr_cont(", pte=%016llx", pte_val(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) pte_unmap(ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) } while(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * This function sets the access flags (dirty, accessed), as well as write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * permission, and only to a more permissive setting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * It needs to cope with hardware update of the accessed/dirty state by other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * agents in the system and can safely skip the __sync_icache_dcache() call as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * like set_pte_at(), the PTE is never changed from no-exec to exec here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * Returns whether or not the PTE actually changed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int ptep_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) unsigned long address, pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) pte_t entry, int dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) pteval_t old_pteval, pteval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pte_t pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (pte_same(pte, entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /* only preserve the access flags and write permission */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * Setting the flags must be done atomically to avoid racing with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * hardware update of the access/dirty state. The PTE_RDONLY bit must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * be set to the most permissive (lowest value) of *ptep and entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * (calculated as: a & b == ~(~a | ~b)).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) pte_val(entry) ^= PTE_RDONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) pteval = pte_val(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) old_pteval = pteval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) pteval ^= PTE_RDONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) pteval |= pte_val(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) pteval ^= PTE_RDONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) } while (pteval != old_pteval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* Invalidate a stale read-only entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) flush_tlb_page(vma, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static bool is_el1_instruction_abort(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) unsigned int ec = ESR_ELx_EC(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (fsc_type == ESR_ELx_FSC_PERM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return fsc_type == ESR_ELx_FSC_FAULT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) (regs->pstate & PSR_PAN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) u64 par, dfsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (ESR_ELx_EC(esr) != ESR_ELx_EC_DABT_CUR ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) asm volatile("at s1e1r, %0" :: "r" (addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) par = read_sysreg_par();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * If we now have a valid translation, treat the translation fault as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * spurious.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!(par & SYS_PAR_EL1_F))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * If we got a different type of fault from the AT instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * treat the translation fault as spurious.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return (dfsc & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static void die_kernel_fault(const char *msg, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) bust_spinlocks(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) trace_android_rvh_die_kernel_fault(regs, esr, addr, msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) mem_abort_decode(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) show_pte(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) die("Oops", regs, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) bust_spinlocks(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) do_exit(SIGKILL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #ifdef CONFIG_KASAN_HW_TAGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static void report_tag_fault(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) static bool reported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) bool is_write;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (READ_ONCE(reported))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * This is used for KASAN tests and assumes that no MTE faults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * happened before running the tests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (mte_report_once())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) WRITE_ONCE(reported, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * SAS bits aren't set for all faults reported in EL1, so we can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * find out access size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) is_write = !!(esr & ESR_ELx_WNR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) kasan_report(addr, 0, is_write, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static inline void report_tag_fault(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct pt_regs *regs) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static void do_tag_recovery(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) report_tag_fault(addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * Disable MTE Tag Checking on the local CPU for the current EL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * It will be done lazily on the other CPUs when they will hit a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * tag fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static bool is_el1_mte_sync_tag_check_fault(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned int ec = ESR_ELx_EC(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) unsigned int fsc = esr & ESR_ELx_FSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (ec != ESR_ELx_EC_DABT_CUR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (fsc == ESR_ELx_FSC_MTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static void __do_kernel_fault(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) const char *msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * Are we prepared to handle this kernel fault?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * We are almost certainly not prepared to handle instruction faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) "Ignoring spurious kernel translation fault at virtual address %016lx\n", addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (is_el1_mte_sync_tag_check_fault(esr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) do_tag_recovery(addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (is_el1_permission_fault(addr, esr, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (esr & ESR_ELx_WNR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) msg = "write to read-only memory";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) else if (is_el1_instruction_abort(esr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) msg = "execute from non-executable memory";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) msg = "read from unreadable memory";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) } else if (addr < PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) msg = "NULL pointer dereference";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) msg = "paging request";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) die_kernel_fault(msg, addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static void set_thread_esr(unsigned long address, unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) current->thread.fault_address = address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * If the faulting address is in the kernel, we must sanitize the ESR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * From userspace's point of view, kernel-only mappings don't exist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * at all, so we report them as level 0 translation faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * (This is not quite the way that "no mapping there at all" behaves:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * an alignment fault not caused by the memory type would take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * precedence over translation fault for a real access to empty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * space. Unfortunately we can't easily distinguish "alignment fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * not caused by memory type" from "alignment fault caused by memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * type", so we ignore this wrinkle and just return the translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * fault.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (!is_ttbr0_addr(current->thread.fault_address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) switch (ESR_ELx_EC(esr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) case ESR_ELx_EC_DABT_LOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * These bits provide only information about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * faulting instruction, which userspace knows already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * We explicitly clear bits which are architecturally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * RES0 in case they are given meanings in future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * We always report the ESR as if the fault was taken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * to EL1 and so ISV and the bits in ISS[23:14] are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * clear. (In fact it always will be a fault to EL1.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ESR_ELx_CM | ESR_ELx_WNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) esr |= ESR_ELx_FSC_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) case ESR_ELx_EC_IABT_LOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * Claim a level 0 translation fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * All other bits are architecturally RES0 for faults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * reported with that DFSC value, so we clear them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) esr |= ESR_ELx_FSC_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * This should never happen (entry.S only brings us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * into this code for insn and data aborts from a lower
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * exception level). Fail safe by not providing an ESR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * context record at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) WARN(1, "ESR 0x%x is not DABT or IABT from EL0\n", esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) esr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) current->thread.fault_code = esr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) static void do_bad_area(unsigned long far, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) unsigned long addr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * If we are in kernel mode at this point, we have no context to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * handle this fault with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) const struct fault_info *inf = esr_to_fault_info(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) set_thread_esr(addr, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) __do_kernel_fault(addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) #define VM_FAULT_BADMAP 0x010000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) #define VM_FAULT_BADACCESS 0x020000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static int __do_page_fault(struct vm_area_struct *vma, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) unsigned int mm_flags, unsigned long vm_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (unlikely(!vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return VM_FAULT_BADMAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * Ok, we have a good vm_area for this memory access, so we can handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (unlikely(vma->vm_start > addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (!(vma->vm_flags & VM_GROWSDOWN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return VM_FAULT_BADMAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (expand_stack(vma, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return VM_FAULT_BADMAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * Check that the permissions on the VMA allow for the fault which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * occurred.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) if (!(vma->vm_flags & vm_flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return VM_FAULT_BADACCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static bool is_el0_instruction_abort(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * Note: not valid for EL1 DC IVAC, but we never use that such that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * should fault. EL0 cannot issue DC IVAC (undef).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) static bool is_write_abort(unsigned int esr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) const struct fault_info *inf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) vm_fault_t fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) unsigned long vm_flags = VM_ACCESS_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) unsigned int mm_flags = FAULT_FLAG_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) struct vm_area_struct *vma = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) unsigned long addr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (kprobe_page_fault(regs, esr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * If we're in an interrupt or have no user context, we must not take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * the fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (faulthandler_disabled() || !mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) mm_flags |= FAULT_FLAG_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (is_el0_instruction_abort(esr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) vm_flags = VM_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) mm_flags |= FAULT_FLAG_INSTRUCTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) } else if (is_write_abort(esr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) vm_flags = VM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) mm_flags |= FAULT_FLAG_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* regs->orig_addr_limit may be 0 if we entered from EL0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (regs->orig_addr_limit == KERNEL_DS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) die_kernel_fault("access to user memory with fs=KERNEL_DS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (is_el1_instruction_abort(esr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) die_kernel_fault("execution of user memory",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (!search_exception_tables(regs->pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) die_kernel_fault("access to user memory outside uaccess routines",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * let's try a speculative page fault without grabbing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * mmap_sem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) fault = handle_speculative_fault(mm, addr, mm_flags, &vma, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (fault != VM_FAULT_RETRY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * As per x86, we may deadlock here. However, since the kernel only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * validly references user space from well defined areas of the code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * we can bug out early if this is from code which shouldn't.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (!mmap_read_trylock(mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (!user_mode(regs) && !search_exception_tables(regs->pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * The above down_read_trylock() might have succeeded in which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * case, we'll have missed the might_sleep() from down_read().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) #ifdef CONFIG_DEBUG_VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (!user_mode(regs) && !search_exception_tables(regs->pc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (!vma || !can_reuse_spf_vma(vma, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) vma = find_vma(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) fault = __do_page_fault(vma, addr, mm_flags, vm_flags, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* Quick path to respond to signals */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (fault_signal_pending(fault, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (fault & VM_FAULT_RETRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) mm_flags |= FAULT_FLAG_TRIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * Do not try to reuse this vma and fetch it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * again since we will release the mmap_sem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) vma = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * Handle the "normal" (no error) case first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) VM_FAULT_BADACCESS))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * If we are in kernel mode at this point, we have no context to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * handle this fault with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (fault & VM_FAULT_OOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * We ran out of memory, call the OOM killer, and return to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * userspace (which will retry the fault, or kill us if we got
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * oom-killed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) pagefault_out_of_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) inf = esr_to_fault_info(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) set_thread_esr(addr, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (fault & VM_FAULT_SIGBUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * We had some memory, but were unable to successfully fix up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * this page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) arm64_force_sig_fault(SIGBUS, BUS_ADRERR, far, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) } else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) unsigned int lsb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) lsb = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (fault & VM_FAULT_HWPOISON_LARGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * Something tried to access memory that isn't in our memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * map.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) arm64_force_sig_fault(SIGSEGV,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) far, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) no_context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) __do_kernel_fault(addr, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) static int __kprobes do_translation_fault(unsigned long far,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) unsigned long addr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (is_ttbr0_addr(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return do_page_fault(far, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) do_bad_area(far, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) static int do_alignment_fault(unsigned long far, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) do_bad_area(far, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) static int do_bad(unsigned long far, unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) unsigned long addr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) int ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) trace_android_vh_handle_tlb_conf(addr, esr, &ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) const struct fault_info *inf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) unsigned long siaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) inf = esr_to_fault_info(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (user_mode(regs) && apei_claim_sea(regs) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * APEI claimed this as a firmware-first notification.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * Some processing deferred to task_work before ret_to_user().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (esr & ESR_ELx_FnV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) siaddr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * The architecture specifies that the tag bits of FAR_EL1 are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * UNKNOWN for synchronous external aborts. Mask them out now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * so that userspace doesn't see them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) siaddr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) trace_android_rvh_do_sea(regs, esr, siaddr, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) static int do_tag_check_fault(unsigned long far, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * for tag check faults. Set them to corresponding bits in the untagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) do_bad_area(far, esr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static const struct fault_info fault_info[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) { do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) { do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) { do_bad, SIGKILL, SI_KERNEL, "level 2 address size fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) { do_bad, SIGKILL, SI_KERNEL, "level 3 address size fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) { do_bad, SIGKILL, SI_KERNEL, "unknown 8" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) { do_bad, SIGKILL, SI_KERNEL, "unknown 12" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) { do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) { do_tag_check_fault, SIGSEGV, SEGV_MTESERR, "synchronous tag check fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) { do_bad, SIGKILL, SI_KERNEL, "unknown 18" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) { do_bad, SIGKILL, SI_KERNEL, "unknown 19" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) { do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) { do_sea, SIGKILL, SI_KERNEL, "level 1 (translation table walk)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) { do_sea, SIGKILL, SI_KERNEL, "level 2 (translation table walk)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) { do_sea, SIGKILL, SI_KERNEL, "level 3 (translation table walk)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) { do_sea, SIGBUS, BUS_OBJERR, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) { do_bad, SIGKILL, SI_KERNEL, "unknown 25" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) { do_bad, SIGKILL, SI_KERNEL, "unknown 26" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) { do_bad, SIGKILL, SI_KERNEL, "unknown 27" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) { do_sea, SIGKILL, SI_KERNEL, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) { do_sea, SIGKILL, SI_KERNEL, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) { do_sea, SIGKILL, SI_KERNEL, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) { do_sea, SIGKILL, SI_KERNEL, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) { do_bad, SIGKILL, SI_KERNEL, "unknown 32" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) { do_bad, SIGKILL, SI_KERNEL, "unknown 34" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) { do_bad, SIGKILL, SI_KERNEL, "unknown 35" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) { do_bad, SIGKILL, SI_KERNEL, "unknown 36" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) { do_bad, SIGKILL, SI_KERNEL, "unknown 37" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) { do_bad, SIGKILL, SI_KERNEL, "unknown 38" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) { do_bad, SIGKILL, SI_KERNEL, "unknown 39" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) { do_bad, SIGKILL, SI_KERNEL, "unknown 40" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) { do_bad, SIGKILL, SI_KERNEL, "unknown 41" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) { do_bad, SIGKILL, SI_KERNEL, "unknown 42" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) { do_bad, SIGKILL, SI_KERNEL, "unknown 43" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) { do_bad, SIGKILL, SI_KERNEL, "unknown 44" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) { do_bad, SIGKILL, SI_KERNEL, "unknown 45" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) { do_bad, SIGKILL, SI_KERNEL, "unknown 46" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) { do_bad, SIGKILL, SI_KERNEL, "unknown 47" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) { do_bad, SIGKILL, SI_KERNEL, "TLB conflict abort" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) { do_bad, SIGKILL, SI_KERNEL, "Unsupported atomic hardware update fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) { do_bad, SIGKILL, SI_KERNEL, "unknown 50" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) { do_bad, SIGKILL, SI_KERNEL, "unknown 51" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) { do_bad, SIGKILL, SI_KERNEL, "implementation fault (lockdown abort)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) { do_bad, SIGBUS, BUS_OBJERR, "implementation fault (unsupported exclusive)" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) { do_bad, SIGKILL, SI_KERNEL, "unknown 54" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) { do_bad, SIGKILL, SI_KERNEL, "unknown 55" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) { do_bad, SIGKILL, SI_KERNEL, "unknown 56" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) { do_bad, SIGKILL, SI_KERNEL, "unknown 57" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) { do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) { do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) { do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) { do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) { do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) { do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) const struct fault_info *inf = esr_to_fault_info(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) unsigned long addr = untagged_addr(far);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (!inf->fn(far, esr, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) if (!user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) pr_alert("Unhandled fault at 0x%016lx\n", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) trace_android_rvh_do_mem_abort(regs, esr, addr, inf->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) mem_abort_decode(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) show_pte(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * At this point we have an unrecognized fault type whose tag bits may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * have been defined as UNKNOWN. Therefore we only expose the untagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * address to the signal handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) NOKPROBE_SYMBOL(do_mem_abort);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) void do_el0_irq_bp_hardening(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) /* PC has already been checked in entry.S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) arm64_apply_bp_hardening();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) NOKPROBE_SYMBOL(do_el0_irq_bp_hardening);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) trace_android_rvh_do_sp_pc_abort(regs, esr, addr, user_mode(regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) addr, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) NOKPROBE_SYMBOL(do_sp_pc_abort);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) int __init early_brk64(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * __refdata because early_brk64 is __init, but the reference to it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * clobbered at arch_initcall time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * See traps.c and debug-monitors.c:debug_traps_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) static struct fault_info __refdata debug_fault_info[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) { do_bad, SIGKILL, SI_KERNEL, "unknown 3" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) { do_bad, SIGKILL, SI_KERNEL, "unknown 7" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) void __init hook_debug_fault_code(int nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) int (*fn)(unsigned long, unsigned int, struct pt_regs *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) int sig, int code, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) debug_fault_info[nr].fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) debug_fault_info[nr].sig = sig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) debug_fault_info[nr].code = code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) debug_fault_info[nr].name = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * In debug exception context, we explicitly disable preemption despite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * having interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * This serves two purposes: it makes it much less likely that we would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * accidentally schedule in exception context and it will force a warning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * if we somehow manage to schedule by accident.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) static void debug_exception_enter(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /* This code is a bit fragile. Test it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) NOKPROBE_SYMBOL(debug_exception_enter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) static void debug_exception_exit(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) NOKPROBE_SYMBOL(debug_exception_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) #ifdef CONFIG_ARM64_ERRATUM_1463225
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * We've taken a dummy step exception from the kernel to ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * that interrupts are re-enabled on the syscall path. Return back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * masked so that we can safely restore the mdscr and get on with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * handling the syscall.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) regs->pstate |= PSR_D_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) #endif /* CONFIG_ARM64_ERRATUM_1463225 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) const struct fault_info *inf = esr_to_debug_fault_info(esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) unsigned long pc = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) if (cortex_a76_erratum_1463225_debug_handler(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) debug_exception_enter(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (user_mode(regs) && !is_ttbr0_addr(pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) arm64_apply_bp_hardening();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if (inf->fn(addr_if_watchpoint, esr, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) arm64_notify_die(inf->name, regs, inf->sig, inf->code, pc, esr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) debug_exception_exit(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) NOKPROBE_SYMBOL(do_debug_exception);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * Used during anonymous page fault handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) unsigned long vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO | __GFP_CMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * If the page is mapped with PROT_MTE, initialise the tags at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) * point of allocation and page zeroing as this is usually faster than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) * separate DC ZVA and STGM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) if (vma->vm_flags & VM_MTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) flags |= __GFP_ZEROTAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return alloc_page_vma(flags, vma, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) void tag_clear_highpage(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) mte_zero_clear_page_tags(page_address(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) page_kasan_tag_reset(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) set_bit(PG_mte_tagged, &page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags