^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * PARISC64 Huge TLB page support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This parisc implementation is heavily based on the SPARC and x86 code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2015 Helge Deller <deller@gmx.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) unsigned long len, unsigned long pgoff, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct hstate *h = hstate_file(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) if (len & ~huge_page_mask(h))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) if (len > TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) if (flags & MAP_FIXED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (prepare_hugepage_range(file, addr, len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) addr = ALIGN(addr, huge_page_size(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* we need to make sure the colouring is OK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return arch_get_unmapped_area(file, addr, len, pgoff, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) pte_t *pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* We must align the address, because our caller will run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * set_huge_pte_at() on whatever we return, which writes out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * all of the sub-ptes for the hugepage range. So we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * to give it the first such sub-pte.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) addr &= HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) pud = pud_alloc(mm, p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (pud) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) pmd = pmd_alloc(mm, pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) pte = pte_alloc_map(mm, pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) pte_t *huge_pte_offset(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) unsigned long addr, unsigned long sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) pte_t *pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) addr &= HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (!pgd_none(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (!p4d_none(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!pud_none(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (!pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) pte = pte_offset_map(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* Purge data and instruction TLB entries. Must be called holding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * the pa_tlb_lock. The TLB purge instructions are slow on SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * machines since the purge must be broadcast to all CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static inline void purge_tlb_entries_huge(struct mm_struct *mm, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* We may use multiple physical huge pages (e.g. 2x1 MB) to emulate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Linux standard huge pages (e.g. 2 MB) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) BUILD_BUG_ON(REAL_HPAGE_SHIFT > HPAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) addr &= HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) addr |= _HUGE_PAGE_SIZE_ENCODING_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) for (i = 0; i < (1 << (HPAGE_SHIFT-REAL_HPAGE_SHIFT)); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) purge_tlb_entries(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) addr += (1UL << REAL_HPAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /* __set_huge_pte_at() must be called holding the pa_tlb_lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) pte_t *ptep, pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long addr_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) addr &= HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) addr_start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) set_pte(ptep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) ptep++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) addr += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) pte_val(entry) += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) purge_tlb_entries_huge(mm, addr_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) pte_t *ptep, pte_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __set_huge_pte_at(mm, addr, ptep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) pte_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) entry = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) __set_huge_pte_at(mm, addr, ptep, __pte(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) void huge_ptep_set_wrprotect(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) pte_t old_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) old_pte = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) int huge_ptep_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned long addr, pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) pte_t pte, int dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) int changed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct mm_struct *mm = vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) changed = !pte_same(*ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) if (changed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) __set_huge_pte_at(mm, addr, ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return changed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int pmd_huge(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int pud_huge(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate