^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * CoProcessor (SPU/AFU) mm fault handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * (C) Copyright IBM Deutschland Entwicklung GmbH 2007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Arnd Bergmann <arndb@de.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Jeremy Kerr <jk@ozlabs.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/reg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/copro.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/spu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <misc/cxl-base.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * This ought to be kept in sync with the powerpc specific do_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * function. Currently, there are a few corner cases that we haven't had
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * to handle fortunately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) unsigned long dsisr, vm_fault_t *flt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) unsigned long is_write;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) if (mm == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (mm->pgd == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) vma = find_vma(mm, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (!vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) if (ea < vma->vm_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (!(vma->vm_flags & VM_GROWSDOWN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) if (expand_stack(vma, ea))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) is_write = dsisr & DSISR_ISSTORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (is_write) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (!(vma->vm_flags & VM_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * PROT_NONE is covered by the VMA check above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * and hash should get a NOHPTE fault instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * a PROTFAULT in case fixup is needed for things
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * like autonuma.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) if (!radix_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) WARN_ON_ONCE(dsisr & DSISR_PROTFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (unlikely(*flt & VM_FAULT_ERROR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (*flt & VM_FAULT_OOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) } else if (*flt & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) EXPORT_SYMBOL_GPL(copro_handle_mm_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) u64 vsid, vsidkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int psize, ssize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) switch (get_region_id(ea)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) case USER_REGION_ID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pr_devel("%s: 0x%llx -- USER_REGION_ID\n", __func__, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (mm == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) psize = get_slice_psize(mm, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ssize = user_segment_size(ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) vsid = get_user_vsid(&mm->context, ea, ssize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) vsidkey = SLB_VSID_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) case VMALLOC_REGION_ID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) pr_devel("%s: 0x%llx -- VMALLOC_REGION_ID\n", __func__, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) psize = mmu_vmalloc_psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) ssize = mmu_kernel_ssize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) vsidkey = SLB_VSID_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) case IO_REGION_ID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) pr_devel("%s: 0x%llx -- IO_REGION_ID\n", __func__, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) psize = mmu_io_psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ssize = mmu_kernel_ssize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) vsidkey = SLB_VSID_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) case LINEAR_MAP_REGION_ID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pr_devel("%s: 0x%llx -- LINEAR_MAP_REGION_ID\n", __func__, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) psize = mmu_linear_psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ssize = mmu_kernel_ssize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) vsidkey = SLB_VSID_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) pr_debug("%s: invalid region access at %016llx\n", __func__, ea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Bad address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (!vsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) vsid = (vsid << slb_vsid_shift(ssize)) | vsidkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) vsid |= mmu_psize_defs[psize].sllp |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) ((ssize == MMU_SEGSIZE_1T) ? SLB_VSID_B_1T : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) slb->esid = (ea & (ssize == MMU_SEGSIZE_1T ? ESID_MASK_1T : ESID_MASK)) | SLB_ESID_V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) slb->vsid = vsid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) EXPORT_SYMBOL_GPL(copro_calculate_slb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) void copro_flush_all_slbs(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #ifdef CONFIG_SPU_BASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) spu_flush_all_slbs(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) cxl_slbia(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) EXPORT_SYMBOL_GPL(copro_flush_all_slbs);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags