^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/arch/arm/mm/fault.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1995 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Modifications for ARM processor (c) 1995-2004 Russell King
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/page-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/system_misc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/system_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "fault.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * This is useful to dump out the page tables associated with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * 'addr' in mm 'mm'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (!mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) mm = &init_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) printk("%spgd = %p\n", lvl, mm->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (p4d_none(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (p4d_bad(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) pr_cont("(bad)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (PTRS_PER_PUD != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if (pud_none(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (pud_bad(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) pr_cont("(bad)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (PTRS_PER_PMD != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (pmd_bad(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) pr_cont("(bad)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* We must not map this if we have highmem enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) pte = pte_offset_map(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) pr_cont(", *pte=%08llx", (long long)pte_val(*pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #ifndef CONFIG_ARM_LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) pr_cont(", *ppte=%08llx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) pte_unmap(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) } while(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #else /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #endif /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * Oops. The kernel tried to access some page that wasn't present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * Are we prepared to handle this kernel fault?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (fixup_exception(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * No handler, we'll have to terminate things with extreme prejudice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) bust_spinlocks(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) pr_alert("8<--- cut here ---\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) (addr < PAGE_SIZE) ? "NULL pointer dereference" :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) "paging request", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) show_pte(KERN_ALERT, mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) die("Oops", regs, fsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) bust_spinlocks(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) do_exit(SIGKILL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * Something tried to access memory that isn't in our memory map..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * User mode accesses just cause a SIGSEGV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) int code, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct task_struct *tsk = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (addr > TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) harden_branch_predictor();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #ifdef CONFIG_DEBUG_USER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ((user_debug & UDBG_BUS) && (sig == SIGBUS))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) pr_err("8<--- cut here ---\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) tsk->comm, sig, addr, fsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) show_pte(KERN_ERR, tsk->mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) show_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #ifndef CONFIG_KUSER_HELPERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) printk_ratelimited(KERN_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) tsk->comm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) tsk->thread.address = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) tsk->thread.error_code = fsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) tsk->thread.trap_no = 14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) force_sig_fault(sig, code, (void __user *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct task_struct *tsk = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct mm_struct *mm = tsk->active_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * If we are in kernel mode at this point, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * have no context to handle this fault with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) __do_kernel_fault(mm, addr, fsr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define VM_FAULT_BADMAP 0x010000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #define VM_FAULT_BADACCESS 0x020000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * Check that the permissions on the VMA allow for the fault which occurred.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * If we encountered a write fault, we must have write permission, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * we allow any permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned int mask = VM_ACCESS_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) mask = VM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (fsr & FSR_LNX_PF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mask = VM_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return vma->vm_flags & mask ? false : true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static vm_fault_t __kprobes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) unsigned int flags, struct task_struct *tsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) vm_fault_t fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) vma = find_vma(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) fault = VM_FAULT_BADMAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (unlikely(!vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (unlikely(vma->vm_start > addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) goto check_stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * Ok, we have a good vm_area for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * memory access, so we can handle it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) good_area:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (access_error(fsr, vma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) fault = VM_FAULT_BADACCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return handle_mm_fault(vma, addr & PAGE_MASK, flags, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) check_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* Don't allow expansion below FIRST_USER_ADDRESS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (vma->vm_flags & VM_GROWSDOWN &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) addr >= FIRST_USER_ADDRESS && !expand_stack(vma, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) goto good_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static int __kprobes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct task_struct *tsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) int sig, code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) vm_fault_t fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) unsigned int flags = FAULT_FLAG_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (kprobe_page_fault(regs, fsr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) tsk = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) mm = tsk->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /* Enable interrupts if they were enabled in the parent context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (interrupts_enabled(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * If we're in an interrupt or have no user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * context, we must not take the fault..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (faulthandler_disabled() || !mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) flags |= FAULT_FLAG_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) flags |= FAULT_FLAG_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * As per x86, we may deadlock here. However, since the kernel only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * validly references user space from well defined areas of the code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * we can bug out early if this is from code which shouldn't.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!mmap_read_trylock(mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * The above down_read_trylock() might have succeeded in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * which case, we'll have missed the might_sleep() from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * down_read()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #ifdef CONFIG_DEBUG_VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (!user_mode(regs) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) !search_exception_tables(regs->ARM_pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) fault = __do_page_fault(mm, addr, fsr, flags, tsk, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /* If we need to retry but a fatal signal is pending, handle the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * signal first. We do not need to release the mmap_lock because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * it would already be released in __lock_page_or_retry in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * mm/filemap.c. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (fault_signal_pending(fault, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (fault & VM_FAULT_RETRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) flags |= FAULT_FLAG_TRIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * Handle the "normal" case first - VM_FAULT_MAJOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * If we are in kernel mode at this point, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * have no context to handle this fault with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (fault & VM_FAULT_OOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * We ran out of memory, call the OOM killer, and return to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * userspace (which will retry the fault, or kill us if we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * got oom-killed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) pagefault_out_of_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (fault & VM_FAULT_SIGBUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * We had some memory, but were unable to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * successfully fix up this page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) sig = SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) code = BUS_ADRERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * Something tried to access memory that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * isn't in our memory map..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) sig = SIGSEGV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) code = fault == VM_FAULT_BADACCESS ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) SEGV_ACCERR : SEGV_MAPERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) __do_user_fault(addr, fsr, sig, code, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) no_context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) __do_kernel_fault(mm, addr, fsr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #else /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) #endif /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * First Level Translation Fault Handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * We enter here because the first level page table doesn't contain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * a valid entry for the address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * If the address is in kernel space (>= TASK_SIZE), then we are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * probably faulting in the vmalloc() area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * If the init_task's first level page tables contains the relevant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * entry, we copy the it to this task. If not, we send the process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * a signal, fixup the exception, or oops the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * NOTE! We MUST NOT take any locks for this case. We may be in an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * interrupt or a critical region, and should only copy the information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * from the master page table, nothing more.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static int __kprobes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) do_translation_fault(unsigned long addr, unsigned int fsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) pgd_t *pgd, *pgd_k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) p4d_t *p4d, *p4d_k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) pud_t *pud, *pud_k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) pmd_t *pmd, *pmd_k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (addr < TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return do_page_fault(addr, fsr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) goto bad_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) index = pgd_index(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) pgd = cpu_get_pgd() + index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) pgd_k = init_mm.pgd + index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) p4d_k = p4d_offset(pgd_k, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (p4d_none(*p4d_k))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) goto bad_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!p4d_present(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) set_p4d(p4d, *p4d_k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) pud_k = pud_offset(p4d_k, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (pud_none(*pud_k))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) goto bad_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (!pud_present(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) set_pud(pud, *pud_k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) pmd_k = pmd_offset(pud_k, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #ifdef CONFIG_ARM_LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * Only one hardware entry per PMD with LPAE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * On ARM one Linux PGD entry contains two hardware entries (see page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * tables layout in pgtable.h). We normally guarantee that we always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * fill both L1 entries. But create_mapping() doesn't follow the rule.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * It can create inidividual L1 entries, so here we have to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * pmd_none() check for the entry really corresponded to address, not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * for the first of pair.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) index = (addr >> SECTION_SHIFT) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (pmd_none(pmd_k[index]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) goto bad_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) copy_pmd(pmd, pmd_k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) bad_area:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) do_bad_area(addr, fsr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) #else /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) do_translation_fault(unsigned long addr, unsigned int fsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) #endif /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * Some section permission faults need to be handled gracefully.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * They can happen due to a __{get,put}_user during an oops.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) #ifndef CONFIG_ARM_LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) do_bad_area(addr, fsr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) #endif /* CONFIG_ARM_LPAE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * This abort handler always returns "fault".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) struct fsr_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) int sig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) int code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /* FSR definition */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #ifdef CONFIG_ARM_LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) #include "fsr-3level.c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) #include "fsr-2level.c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) int sig, int code, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) fsr_info[nr].fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) fsr_info[nr].sig = sig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) fsr_info[nr].code = code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) fsr_info[nr].name = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * Dispatch a data abort to the relevant handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) asmlinkage void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) pr_alert("8<--- cut here ---\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) inf->name, fsr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) show_pte(KERN_ALERT, current->mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) fsr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int sig, int code, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ifsr_info[nr].fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) ifsr_info[nr].sig = sig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) ifsr_info[nr].code = code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) ifsr_info[nr].name = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) asmlinkage void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) inf->name, ifsr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) ifsr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * Abort handler to be used only during first unmasking of asynchronous aborts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * on the boot CPU. This makes sure that the machine will not die if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * firmware/bootloader left an imprecise abort pending for us to trip over.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) static int __init early_abort_handler(unsigned long addr, unsigned int fsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) "first unmask, this is most likely caused by a "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) "firmware/bootloader bug.\n", fsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) void __init early_abt_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) fsr_info[FSR_FS_AEA].fn = early_abort_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) local_abt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) fsr_info[FSR_FS_AEA].fn = do_bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) #ifndef CONFIG_ARM_LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static int __init exceptions_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (cpu_architecture() >= CPU_ARCH_ARMv6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) "I-cache maintenance fault");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (cpu_architecture() >= CPU_ARCH_ARMv7) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * TODO: Access flag faults introduced in ARMv6K.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * Runtime check for 'K' extension is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) "section access flag fault");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) "section access flag fault");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) arch_initcall(exceptions_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags