^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * SPARC64 Huge TLB page support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) /* Slightly simplified from the non-hugepage variant because by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * definition we don't have to worry about any page coloring stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) unsigned long len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) unsigned long pgoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) struct hstate *h = hstate_file(filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) unsigned long task_size = TASK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct vm_unmapped_area_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (test_thread_flag(TIF_32BIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) task_size = STACK_TOP32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) info.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) info.length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) info.low_limit = TASK_UNMAPPED_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) info.high_limit = min(task_size, VA_EXCLUDE_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) info.align_mask = PAGE_MASK & ~huge_page_mask(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) info.align_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) addr = vm_unmapped_area(&info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) VM_BUG_ON(addr != -ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) info.low_limit = VA_EXCLUDE_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) info.high_limit = task_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) addr = vm_unmapped_area(&info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) const unsigned long len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) const unsigned long pgoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) const unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct hstate *h = hstate_file(filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) unsigned long addr = addr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct vm_unmapped_area_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* This should only ever run for 32-bit processes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) BUG_ON(!test_thread_flag(TIF_32BIT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) info.flags = VM_UNMAPPED_AREA_TOPDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) info.length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) info.low_limit = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) info.high_limit = mm->mmap_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) info.align_mask = PAGE_MASK & ~huge_page_mask(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) info.align_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) addr = vm_unmapped_area(&info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * A failed mmap() very likely causes application failure,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * so fall back to the bottom-up function here. This scenario
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * can happen with large stack limits and large mmap()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (addr & ~PAGE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) VM_BUG_ON(addr != -ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) info.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) info.low_limit = TASK_UNMAPPED_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) info.high_limit = STACK_TOP32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) addr = vm_unmapped_area(&info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) unsigned long len, unsigned long pgoff, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct hstate *h = hstate_file(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned long task_size = TASK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (test_thread_flag(TIF_32BIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) task_size = STACK_TOP32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (len & ~huge_page_mask(h))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (len > task_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (flags & MAP_FIXED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (prepare_hugepage_range(file, addr, len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) addr = ALIGN(addr, huge_page_size(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) vma = find_vma(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (task_size - len >= addr &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) (!vma || addr + len <= vm_start_gap(vma)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (mm->get_unmapped_area == arch_get_unmapped_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return hugetlb_get_unmapped_area_bottomup(file, addr, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) pgoff, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return hugetlb_get_unmapped_area_topdown(file, addr, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) pgoff, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) unsigned long hugepage_size = _PAGE_SZ4MB_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) switch (shift) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) case HPAGE_16GB_SHIFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) hugepage_size = _PAGE_SZ16GB_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) pte_val(entry) |= _PAGE_PUD_HUGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) case HPAGE_2GB_SHIFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) hugepage_size = _PAGE_SZ2GB_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) pte_val(entry) |= _PAGE_PMD_HUGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) case HPAGE_256MB_SHIFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) hugepage_size = _PAGE_SZ256MB_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) pte_val(entry) |= _PAGE_PMD_HUGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) case HPAGE_SHIFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) pte_val(entry) |= _PAGE_PMD_HUGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) case HPAGE_64K_SHIFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) hugepage_size = _PAGE_SZ64K_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) pte_val(entry) = pte_val(entry) | hugepage_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return sun4v_hugepage_shift_to_tte(entry, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return sun4u_hugepage_shift_to_tte(entry, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) struct page *page, int writeable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) unsigned int shift = huge_page_shift(hstate_vma(vma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) pte_t pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) pte = hugepage_shift_to_tte(entry, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #ifdef CONFIG_SPARC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /* If this vma has ADI enabled on it, turn on TTE.mcd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (vma->vm_flags & VM_SPARC_ADI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return pte_mkmcd(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return pte_mknotmcd(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) unsigned int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) switch (tte_szbits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) case _PAGE_SZ16GB_4V:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) shift = HPAGE_16GB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) case _PAGE_SZ2GB_4V:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) shift = HPAGE_2GB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) case _PAGE_SZ256MB_4V:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) shift = HPAGE_256MB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) case _PAGE_SZ4MB_4V:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) shift = REAL_HPAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) case _PAGE_SZ64K_4V:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) shift = HPAGE_64K_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) unsigned int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) switch (tte_szbits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) case _PAGE_SZ256MB_4U:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) shift = HPAGE_256MB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) case _PAGE_SZ4MB_4U:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) shift = REAL_HPAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) case _PAGE_SZ64K_4U:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) shift = HPAGE_64K_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static unsigned int huge_tte_to_shift(pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) unsigned long shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) shift = sun4v_huge_tte_to_shift(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) shift = sun4u_huge_tte_to_shift(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (shift == PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) pte_val(entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static unsigned long huge_tte_to_size(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) unsigned long size = 1UL << huge_tte_to_shift(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (size == REAL_HPAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) size = HPAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) pud = pud_alloc(mm, p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (!pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (sz >= PUD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return (pte_t *)pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) pmd = pmd_alloc(mm, pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (!pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (sz >= PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) return (pte_t *)pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return pte_alloc_map(mm, pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) pte_t *huge_pte_offset(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (pgd_none(*pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (p4d_none(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (pud_none(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (is_hugetlb_pud(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return (pte_t *)pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (is_hugetlb_pmd(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return (pte_t *)pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return pte_offset_map(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) pte_t *ptep, pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) unsigned int nptes, orig_shift, shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) unsigned long i, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) pte_t orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) size = huge_tte_to_size(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (size >= PUD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) shift = PUD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) else if (size >= PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) shift = PMD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) nptes = size >> shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (!pte_present(*ptep) && pte_present(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) mm->context.hugetlb_pte_count += nptes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) addr &= ~(size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) orig = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) for (i = 0; i < nptes; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) ptep[i] = __pte(pte_val(entry) + (i << shift));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (size == HPAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) orig_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) unsigned int i, nptes, orig_shift, shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) pte_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) entry = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) size = huge_tte_to_size(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (size >= PUD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) shift = PUD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) else if (size >= PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) shift = PMD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) nptes = size >> shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (pte_present(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) mm->context.hugetlb_pte_count -= nptes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) addr &= ~(size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) for (i = 0; i < nptes; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ptep[i] = __pte(0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (size == HPAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) orig_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) int pmd_huge(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return !pmd_none(pmd) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) int pud_huge(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) return !pud_none(pud) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) (pud_val(pud) & (_PAGE_VALID|_PAGE_PUD_HUGE)) != _PAGE_VALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) pgtable_t token = pmd_pgtable(*pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) pmd_clear(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) pte_free_tlb(tlb, token, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) mm_dec_nr_ptes(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) unsigned long floor, unsigned long ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) unsigned long start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) next = pmd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (is_hugetlb_pmd(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) pmd_clear(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) hugetlb_free_pte_range(tlb, pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) } while (pmd++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) start &= PUD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (start < floor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (ceiling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) ceiling &= PUD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (!ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (end - 1 > ceiling - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) pmd = pmd_offset(pud, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) pud_clear(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) pmd_free_tlb(tlb, pmd, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) mm_dec_nr_pmds(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) unsigned long floor, unsigned long ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) next = pud_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (pud_none_or_clear_bad(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (is_hugetlb_pud(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) pud_clear(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) ceiling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) } while (pud++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) start &= PGDIR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (start < floor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (ceiling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) ceiling &= PGDIR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (!ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (end - 1 > ceiling - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) pud = pud_offset(p4d, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) p4d_clear(p4d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) pud_free_tlb(tlb, pud, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) mm_dec_nr_puds(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) void hugetlb_free_pgd_range(struct mmu_gather *tlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) unsigned long floor, unsigned long ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) addr &= PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (addr < floor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) addr += PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (ceiling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) ceiling &= PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (!ceiling)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (end - 1 > ceiling - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) end -= PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (addr > end - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) pgd = pgd_offset(tlb->mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) next = p4d_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if (p4d_none_or_clear_bad(p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) } while (p4d++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags