^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * PowerPC version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Derived from "arch/i386/mm/fault.c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Modified by Cort Dougan and Paul Mackerras.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Modified for PPC64 by Dave Engebretsen (engebret@ibm.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sched/task_stack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/context_tracking.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/firmware.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/mmu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/siginfo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/kup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/inst.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * do_page_fault error handling helpers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * If we are in kernel mode, bail out with a SEGV, this will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * be caught by the assembly which will restore the non-volatile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * registers before calling bad_page_fault()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) return SIGSEGV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) _exception(SIGSEGV, regs, si_code, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return __bad_area_nosemaphore(regs, address, SEGV_MAPERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Something tried to access memory that isn't in our memory map..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Fix it, but check if it's kernel or user first..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return __bad_area_nosemaphore(regs, address, si_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static noinline int bad_area(struct pt_regs *regs, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return __bad_area(regs, address, SEGV_MAPERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #ifdef CONFIG_PPC_MEM_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) int pkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * We don't try to fetch the pkey from page table because reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * page table without locking doesn't guarantee stable pte value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * Hence the pkey value that we return to userspace can be different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * from the pkey that actually caused access error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * It does *not* guarantee that the VMA we find here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * was the one that we faulted on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * 1. T1 : mprotect_key(foo, PAGE_SIZE, pkey=4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * 2. T1 : set AMR to deny access to pkey=4, touches, page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * 3. T1 : faults...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * 5. T1 : enters fault handler, takes mmap_lock, etc...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * faulted on a pte with its pkey=4.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) pkey = vma_pkey(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * If we are in kernel mode, bail out with a SEGV, this will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * be caught by the assembly which will restore the non-volatile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * registers before calling bad_page_fault()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return SIGSEGV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) _exception_pkey(regs, address, pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static noinline int bad_access(struct pt_regs *regs, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return __bad_area(regs, address, SEGV_ACCERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static int do_sigbus(struct pt_regs *regs, unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) vm_fault_t fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) current->thread.trap_nr = BUS_ADRERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #ifdef CONFIG_MEMORY_FAILURE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned int lsb = 0; /* shutup gcc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) pr_err("MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) current->comm, current->pid, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (fault & VM_FAULT_HWPOISON_LARGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (fault & VM_FAULT_HWPOISON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) lsb = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static int mm_fault_error(struct pt_regs *regs, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) vm_fault_t fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * Kernel page fault interrupted by SIGKILL. We have no reason to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * continue processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (fatal_signal_pending(current) && !user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) return SIGKILL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Out of memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (fault & VM_FAULT_OOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * We ran out of memory, or some other thing happened to us that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * made us unable to handle the page fault gracefully.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return SIGSEGV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) pagefault_out_of_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) VM_FAULT_HWPOISON_LARGE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return do_sigbus(regs, addr, fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) else if (fault & VM_FAULT_SIGSEGV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return bad_area_nosemaphore(regs, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* Is this a bad kernel fault ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) unsigned long address, bool is_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int is_exec = TRAP(regs) == 0x400;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (is_exec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pr_crit_ratelimited("kernel tried to execute %s page (%lx) - exploit attempt? (uid: %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) address >= TASK_SIZE ? "exec-protected" : "user",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) from_kuid(&init_user_ns, current_uid()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) // Kernel exec fault is always bad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (!is_exec && address < TASK_SIZE && (error_code & DSISR_PROTFAULT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) !search_exception_tables(regs->nip)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) pr_crit_ratelimited("Kernel attempted to access user page (%lx) - exploit attempt? (uid: %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) from_kuid(&init_user_ns, current_uid()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) // Kernel fault on kernel address is bad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (address >= TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) // Fault on user outside of certain regions (eg. copy_tofrom_user()) is bad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (!search_exception_tables(regs->nip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) // Read/write fault in a valid region (the exception table search passed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) // above), but blocked by KUAP is bad, it can never succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (bad_kuap_fault(regs, address, is_write))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) // What's left? Kernel fault on user in well defined regions (extable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) // matched), and allowed by KUAP in the faulting context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #ifdef CONFIG_PPC_MEM_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * Make sure to check the VMA so that we do not perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * faults just to hit a pkey fault as soon as we fill in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * page. Only called for current mm, hence foreign == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!arch_vma_access_permitted(vma, is_write, is_exec, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * Allow execution from readable areas if the MMU does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * provide separate controls over reading and executing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * Note: That code used to not be enabled for 4xx/BookE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * It is now as I/D cache coherency for these is done at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * set_pte_at() time and I see no reason why the test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * below wouldn't be valid on those processors. This -may-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * break programs compiled with a really old ABI though.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (is_exec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return !(vma->vm_flags & VM_EXEC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) (cpu_has_feature(CPU_FTR_NOEXECUTE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) !(vma->vm_flags & (VM_READ | VM_WRITE)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (is_write) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (unlikely(!(vma->vm_flags & VM_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (unlikely(!vma_is_accessible(vma)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * We should ideally do the vma pkey access check here. But in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * fault path, handle_mm_fault() also does the same check. To avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * these multiple checks, we skip it here and handle access error due
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * to pkeys later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) #ifdef CONFIG_PPC_SMLPAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static inline void cmo_account_page_fault(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (firmware_has_feature(FW_FEATURE_CMO)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) u32 page_ins;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) page_ins = be32_to_cpu(get_lppaca()->page_ins);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) page_ins += 1 << PAGE_FACTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) get_lppaca()->page_ins = cpu_to_be32(page_ins);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static inline void cmo_account_page_fault(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #endif /* CONFIG_PPC_SMLPAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void sanity_check_fault(bool is_write, bool is_user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) unsigned long error_code, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * Userspace trying to access kernel address, we get PROTFAULT for that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (is_user && address >= TASK_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if ((long)address == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) pr_crit_ratelimited("%s[%d]: User access of kernel address (%lx) - exploit attempt? (uid: %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) current->comm, current->pid, address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) from_kuid(&init_user_ns, current_uid()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (!IS_ENABLED(CONFIG_PPC_BOOK3S))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * For hash translation mode, we should never get a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * PROTFAULT. Any update to pte to reduce access will result in us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * removing the hash page table entry, thus resulting in a DSISR_NOHPTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * fault instead of DSISR_PROTFAULT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * A pte update to relax the access will not result in a hash page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * entry invalidate and hence can result in DSISR_PROTFAULT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * ptep_set_access_flags() doesn't do a hpte flush. This is why we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * the special !is_write in the below conditional.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * For platforms that doesn't supports coherent icache and do support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * per page noexec bit, we do setup things such that we do the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * sync between D/I cache via fault. But that is handled via low level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * hash fault code (hash_page_do_lazy_icache()) and we should not reach
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * here in such case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * For wrong access that can result in PROTFAULT, the above vma->vm_flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * check should handle those and hence we should fall to the bad_area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * handling correctly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * For embedded with per page exec support that doesn't support coherent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * icache we do get PROTFAULT and we handle that D/I cache sync in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * set_pte_at while taking the noexec/prot fault. Hence this is WARN_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * is conditional for server MMU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * For radix, we can get prot fault for autonuma case, because radix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * page table will have them marked noaccess for user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (radix_enabled() || is_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) WARN_ON_ONCE(error_code & DSISR_PROTFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * Define the correct "is_write" bit in error_code based
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * on the processor family
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #define page_fault_is_write(__err) ((__err) & ESR_DST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #define page_fault_is_bad(__err) (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #if defined(CONFIG_PPC_8xx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #elif defined(CONFIG_PPC64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * For 600- and 800-family processors, the error_code parameter is DSISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * for a data fault, SRR1 for an instruction fault. For 400-family processors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * the error_code parameter is ESR for a data fault, 0 for an instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * For 64-bit processors, the error_code parameter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * - DSISR for a non-SLB data access fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * - SRR1 & 0x08000000 for a non-SLB instruction access fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * - 0 any SLB fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * The return value is 0 if the fault was handled, or the signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * number if this is a kernel fault that can't be handled here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static int __do_page_fault(struct pt_regs *regs, unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) unsigned long error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct vm_area_struct * vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned int flags = FAULT_FLAG_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int is_exec = TRAP(regs) == 0x400;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) int is_user = user_mode(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int is_write = page_fault_is_write(error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) vm_fault_t fault, major = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) bool kprobe_fault = kprobe_page_fault(regs, 11);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (unlikely(debugger_fault_handler(regs) || kprobe_fault))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (unlikely(page_fault_is_bad(error_code))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (is_user) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) _exception(SIGBUS, regs, BUS_OBJERR, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* Additional sanity check(s) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) sanity_check_fault(is_write, is_user, error_code, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * The kernel should never take an execute fault nor should it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * take a page fault to a kernel address or a page fault to a user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * address outside of dedicated places
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return SIGSEGV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * If we're in an interrupt, have no user context or are running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * in a region with pagefaults disabled then we must not take the fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (unlikely(faulthandler_disabled() || !mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (is_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) printk_ratelimited(KERN_ERR "Page fault in user mode"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) " with faulthandler_disabled()=%d"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) " mm=%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) faulthandler_disabled(), mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return bad_area_nosemaphore(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) /* We restore the interrupt state now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (!arch_irq_disabled_regs(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * We want to do this outside mmap_lock, because reading code around nip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * can result in fault, which will cause a deadlock when called with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * mmap_lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (is_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) flags |= FAULT_FLAG_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (is_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) flags |= FAULT_FLAG_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) if (is_exec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) flags |= FAULT_FLAG_INSTRUCTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /* When running in the kernel we expect faults to occur only to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * addresses in user space. All other faults represent errors in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * kernel and should generate an OOPS. Unfortunately, in the case of an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * erroneous fault occurring in a code path which already holds mmap_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * we will deadlock attempting to validate the fault against the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * address space. Luckily the kernel only validly references user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * space from well defined areas of code, which are listed in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * exceptions table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * As the vast majority of faults will be valid we will only perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * the source reference check when there is a possibility of a deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * Attempt to lock the address space, if we cannot we then validate the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * source. If this is invalid we can skip the address space check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * thus avoiding the deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (unlikely(!mmap_read_trylock(mm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (!is_user && !search_exception_tables(regs->nip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return bad_area_nosemaphore(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * The above down_read_trylock() might have succeeded in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * which case we'll have missed the might_sleep() from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * down_read():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) vma = find_vma(mm, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (unlikely(!vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return bad_area(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (unlikely(vma->vm_start > address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return bad_area(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (unlikely(expand_stack(vma, address)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return bad_area(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) #ifdef CONFIG_PPC_MEM_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (unlikely(access_pkey_error(is_write, is_exec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) (error_code & DSISR_KEYFAULT), vma)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return bad_access_pkey(regs, address, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) #endif /* CONFIG_PPC_MEM_KEYS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (unlikely(access_error(is_write, is_exec, vma)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return bad_access(regs, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * If for any reason at all we couldn't handle the fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * make sure we exit gracefully rather than endlessly redo
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * the fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) fault = handle_mm_fault(vma, address, flags, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) major |= fault & VM_FAULT_MAJOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (fault_signal_pending(fault, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return user_mode(regs) ? 0 : SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * Handle the retry right now, the mmap_lock has been released in that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (unlikely(fault & VM_FAULT_RETRY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (flags & FAULT_FLAG_ALLOW_RETRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) flags |= FAULT_FLAG_TRIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) mmap_read_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (unlikely(fault & VM_FAULT_ERROR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return mm_fault_error(regs, address, fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * Major/minor page fault accounting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (major)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) cmo_account_page_fault();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) NOKPROBE_SYMBOL(__do_page_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) int do_page_fault(struct pt_regs *regs, unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) unsigned long error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) enum ctx_state prev_state = exception_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) int rc = __do_page_fault(regs, address, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) exception_exit(prev_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) NOKPROBE_SYMBOL(do_page_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * bad_page_fault is called when we have a bad access from the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * It is called from the DSI and ISI handlers in head.S and from some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * of the procedures in traps.c.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) const struct exception_table_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) int is_write = page_fault_is_write(regs->dsisr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* Are we prepared to handle this fault? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if ((entry = search_exception_tables(regs->nip)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) regs->nip = extable_fixup(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) /* kernel has accessed a bad area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) switch (TRAP(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) case 0x300:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) case 0x380:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) case 0xe00:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) pr_alert("BUG: %s on %s at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) regs->dar < PAGE_SIZE ? "Kernel NULL pointer dereference" :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) "Unable to handle kernel data access",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) is_write ? "write" : "read", regs->dar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) case 0x400:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) case 0x480:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) pr_alert("BUG: Unable to handle kernel instruction fetch%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) regs->nip < PAGE_SIZE ? " (NULL pointer?)\n" : "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) case 0x600:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) pr_alert("BUG: Unable to handle kernel unaligned access at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) regs->dar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) pr_alert("BUG: Unable to handle unknown paging fault at 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) regs->dar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) printk(KERN_ALERT "Faulting instruction address: 0x%08lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) regs->nip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (task_stack_end_corrupted(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) die("Kernel access of bad area", regs, sig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags