^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * arch/arm64/mm/hugetlbpage.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013 Linaro Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Based on arch/x86/mm/hugetlbpage.c.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * HugeTLB Support Matrix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * ---------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * | Page Size | CONT PTE | PMD | CONT PMD | PUD |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * ---------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * | 4K | 64K | 2M | 32M | 1G |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * | 16K | 2M | 32M | 1G | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * | 64K | 2M | 512M | 16G | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * ---------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * Reserve CMA areas for the largest supported gigantic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * huge page when requested. Any other smaller gigantic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * huge pages could still be served from those areas.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #ifdef CONFIG_CMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) void __init arm64_hugetlb_cma_reserve(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #ifdef CONFIG_ARM64_4K_PAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) order = PUD_SHIFT - PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) order = CONT_PMD_SHIFT - PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * HugeTLB CMA reservation is required for gigantic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * huge pages which could not be allocated via the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * page allocator. Just warn if there is any change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * breaking this assumption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) WARN_ON(order <= MAX_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) hugetlb_cma_reserve(order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #endif /* CONFIG_CMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) bool arch_hugetlb_migration_supported(struct hstate *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) size_t pagesize = huge_page_size(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) switch (pagesize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #ifdef CONFIG_ARM64_4K_PAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) case PUD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) case PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) case CONT_PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) case CONT_PTE_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) pr_warn("%s: unrecognized huge page size 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) __func__, pagesize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) int pmd_huge(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int pud_huge(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #ifndef __PAGETABLE_PMD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Select all bits except the pfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static inline pgprot_t pte_pgprot(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) unsigned long pfn = pte_pfn(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static int find_num_contig(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pte_t *ptep, size_t *pgsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) pgd_t *pgdp = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) p4d_t *p4dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) pud_t *pudp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) *pgsize = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) p4dp = p4d_offset(pgdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) pudp = pud_offset(p4dp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) pmdp = pmd_offset(pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if ((pte_t *)pmdp == ptep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) *pgsize = PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) return CONT_PMDS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return CONT_PTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) int contig_ptes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *pgsize = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) switch (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #ifdef CONFIG_ARM64_4K_PAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) case PUD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) case PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) contig_ptes = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) case CONT_PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) *pgsize = PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) contig_ptes = CONT_PMDS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) case CONT_PTE_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) *pgsize = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) contig_ptes = CONT_PTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return contig_ptes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * Changing some bits of contiguous entries requires us to follow a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Break-Before-Make approach, breaking the whole contiguous set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * before we can change any entries. See ARM DDI 0487A.k_iss10775,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * "Misprogramming of the Contiguous bit", page D4-1762.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * This helper performs the break step.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static pte_t get_clear_flush(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned long pgsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) unsigned long ncontig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) pte_t orig_pte = huge_ptep_get(ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) bool valid = pte_valid(orig_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) unsigned long i, saddr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) pte_t pte = ptep_get_and_clear(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * If HW_AFDBM is enabled, then the HW could turn on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * the dirty or accessed bit for any page in the set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * so check them all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (pte_dirty(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) orig_pte = pte_mkdirty(orig_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (pte_young(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) orig_pte = pte_mkyoung(orig_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (valid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) flush_tlb_range(&vma, saddr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return orig_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Changing some bits of contiguous entries requires us to follow a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * Break-Before-Make approach, breaking the whole contiguous set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * before we can change any entries. See ARM DDI 0487A.k_iss10775,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * "Misprogramming of the Contiguous bit", page D4-1762.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * This helper performs the break step for use cases where the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * original pte is not needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void clear_flush(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) unsigned long pgsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) unsigned long ncontig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) unsigned long i, saddr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) pte_clear(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) flush_tlb_range(&vma, saddr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) pte_t *ptep, pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) int ncontig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) unsigned long pfn, dpfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) pgprot_t hugeprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * Code needs to be expanded to handle huge swap and migration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * entries. Needed for HUGETLB and MEMORY_FAILURE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) WARN_ON(!pte_present(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (!pte_cont(pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) set_pte_at(mm, addr, ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) ncontig = find_num_contig(mm, addr, ptep, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) pfn = pte_pfn(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) dpfn = pgsize >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) hugeprot = pte_pgprot(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) clear_flush(mm, addr, ptep, pgsize, ncontig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pte_t *ptep, pte_t pte, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) int i, ncontig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ncontig = num_contig_ptes(sz, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) for (i = 0; i < ncontig; i++, ptep++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) set_pte(ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pgd_t *pgdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) p4d_t *p4dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) pud_t *pudp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) pte_t *ptep = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) pgdp = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) p4dp = p4d_offset(pgdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) pudp = pud_alloc(mm, p4dp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (!pudp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (sz == PUD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) ptep = (pte_t *)pudp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) } else if (sz == (CONT_PTE_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pmdp = pmd_alloc(mm, pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (!pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) WARN_ON(addr & (sz - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * Note that if this code were ever ported to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * 32-bit arm platform then it will cause trouble in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * the case where CONFIG_HIGHPTE is set, since there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * will be no pte_unmap() to correspond with this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * pte_alloc_map().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) ptep = pte_alloc_map(mm, pmdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) } else if (sz == PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) ptep = huge_pmd_share(mm, vma, addr, pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) } else if (sz == (CONT_PMD_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pmdp = pmd_alloc(mm, pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) WARN_ON(addr & (sz - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) return (pte_t *)pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) pte_t *huge_pte_offset(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) pgd_t *pgdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) p4d_t *p4dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) pud_t *pudp, pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) pmd_t *pmdp, pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) pgdp = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (!pgd_present(READ_ONCE(*pgdp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) p4dp = p4d_offset(pgdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!p4d_present(READ_ONCE(*p4dp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pudp = pud_offset(p4dp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) pud = READ_ONCE(*pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (sz != PUD_SIZE && pud_none(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* hugepage or swap? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (pud_huge(pud) || !pud_present(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return (pte_t *)pudp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* table; check the next level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (sz == CONT_PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) addr &= CONT_PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) pmdp = pmd_offset(pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) pmd = READ_ONCE(*pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) pmd_none(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (pmd_huge(pmd) || !pmd_present(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return (pte_t *)pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (sz == CONT_PTE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct page *page, int writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) size_t pagesize = huge_page_size(hstate_vma(vma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (pagesize == CONT_PTE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) entry = pte_mkcont(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) } else if (pagesize == CONT_PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) pr_warn("%s: unrecognized huge page size 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) __func__, pagesize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) pte_t *ptep, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) int i, ncontig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) ncontig = num_contig_ptes(sz, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pte_clear(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int ncontig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) pte_t orig_pte = huge_ptep_get(ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (!pte_cont(orig_pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return ptep_get_and_clear(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) ncontig = find_num_contig(mm, addr, ptep, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * huge_ptep_set_access_flags will update access flags (dirty, accesssed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * and write permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * For a contiguous huge pte range we need to check whether or not write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * permission has to change only on the first pte in the set. Then for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * all the contiguous ptes we need to check whether or not there is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * discrepancy between dirty or young.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (pte_write(pte) != pte_write(huge_ptep_get(ptep)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) for (i = 0; i < ncontig; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pte_t orig_pte = huge_ptep_get(ptep + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (pte_dirty(pte) != pte_dirty(orig_pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (pte_young(pte) != pte_young(orig_pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) int huge_ptep_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) unsigned long addr, pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) pte_t pte, int dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) int ncontig, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) size_t pgsize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) unsigned long pfn = pte_pfn(pte), dpfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) pgprot_t hugeprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) pte_t orig_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (!pte_cont(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) dpfn = pgsize >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (!__cont_access_flags_changed(ptep, pte, ncontig))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /* Make sure we don't lose the dirty or young state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (pte_dirty(orig_pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) pte = pte_mkdirty(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (pte_young(orig_pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) pte = pte_mkyoung(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) hugeprot = pte_pgprot(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) void huge_ptep_set_wrprotect(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) unsigned long pfn, dpfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) pgprot_t hugeprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) int ncontig, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) pte_t pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (!pte_cont(READ_ONCE(*ptep))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) ptep_set_wrprotect(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) ncontig = find_num_contig(mm, addr, ptep, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) dpfn = pgsize >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) pte = pte_wrprotect(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) hugeprot = pte_pgprot(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) pfn = pte_pfn(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) void huge_ptep_clear_flush(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) size_t pgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) int ncontig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (!pte_cont(READ_ONCE(*ptep))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) ptep_clear_flush(vma, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) static int __init hugetlbpage_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) #ifdef CONFIG_ARM64_4K_PAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) arch_initcall(hugetlbpage_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) bool __init arch_hugetlb_valid_size(unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) switch (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) #ifdef CONFIG_ARM64_4K_PAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) case PUD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) case CONT_PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) case PMD_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) case CONT_PTE_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags