^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _ASM_X86_PGTABLE_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _ASM_X86_PGTABLE_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/mem_encrypt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/pgtable_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Macro to mark a page protection value as UC-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define pgprot_noncached(prot) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) ((boot_cpu_data.x86 > 3) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) ? (__pgprot(pgprot_val(prot) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS))) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) : (prot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Macros to add or remove encryption attribute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define pgprot_encrypted(prot) __pgprot(__sme_set(pgprot_val(prot)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define pgprot_decrypted(prot) __pgprot(__sme_clr(pgprot_val(prot)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/x86_init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/fpu/xstate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/fpu/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm-generic/pgtable_uffd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) extern pgd_t early_top_pgt[PTRS_PER_PGD];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) bool user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) void ptdump_walk_pgd_level_checkwx(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) void ptdump_walk_user_pgd_level_checkwx(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #ifdef CONFIG_DEBUG_WX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define debug_checkwx() ptdump_walk_pgd_level_checkwx()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define debug_checkwx() do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define debug_checkwx_user() do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * ZERO_PAGE is a global shared page that is always zero: used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * for zero-mapped memory areas etc..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) __visible;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) extern spinlock_t pgd_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) extern struct list_head pgd_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) extern struct mm_struct *pgd_page_get_mm(struct page *page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) extern pmdval_t early_pmd_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #ifdef CONFIG_PARAVIRT_XXL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <asm/paravirt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #else /* !CONFIG_PARAVIRT_XXL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define set_pte(ptep, pte) native_set_pte(ptep, pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define set_pte_atomic(ptep, pte) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) native_set_pte_atomic(ptep, pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #ifndef __PAGETABLE_P4D_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define pgd_clear(pgd) (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #ifndef set_p4d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) # define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #ifndef __PAGETABLE_PUD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define p4d_clear(p4d) native_p4d_clear(p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #ifndef set_pud
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) # define set_pud(pudp, pud) native_set_pud(pudp, pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #ifndef __PAGETABLE_PUD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define pud_clear(pud) native_pud_clear(pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define pmd_clear(pmd) native_pmd_clear(pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define pgd_val(x) native_pgd_val(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define __pgd(x) native_make_pgd(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #ifndef __PAGETABLE_P4D_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define p4d_val(x) native_p4d_val(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define __p4d(x) native_make_p4d(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #ifndef __PAGETABLE_PUD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define pud_val(x) native_pud_val(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define __pud(x) native_make_pud(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #ifndef __PAGETABLE_PMD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define pmd_val(x) native_pmd_val(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define __pmd(x) native_make_pmd(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define pte_val(x) native_pte_val(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define __pte(x) native_make_pte(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define arch_end_context_switch(prev) do {} while(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #endif /* CONFIG_PARAVIRT_XXL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * The following only work if pte_present() is true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * Undefined behaviour if not..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static inline int pte_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return pte_flags(pte) & _PAGE_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static inline u32 read_pkru(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (boot_cpu_has(X86_FEATURE_OSPKE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return rdpkru();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static inline void write_pkru(u32 pkru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct pkru_state *pk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (!boot_cpu_has(X86_FEATURE_OSPKE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pk = get_xsave_addr(¤t->thread.fpu.state.xsave, XFEATURE_PKRU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * The PKRU value in xstate needs to be in sync with the value that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * written to the CPU. The FPU restore on return to userland would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * otherwise load the previous value again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) fpregs_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (pk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) pk->pkru = pkru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) __write_pkru(pkru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) fpregs_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static inline int pte_young(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return pte_flags(pte) & _PAGE_ACCESSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static inline int pmd_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return pmd_flags(pmd) & _PAGE_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static inline int pmd_young(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return pmd_flags(pmd) & _PAGE_ACCESSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static inline int pud_dirty(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return pud_flags(pud) & _PAGE_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) static inline int pud_young(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return pud_flags(pud) & _PAGE_ACCESSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static inline int pte_write(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return pte_flags(pte) & _PAGE_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static inline int pte_huge(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return pte_flags(pte) & _PAGE_PSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static inline int pte_global(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return pte_flags(pte) & _PAGE_GLOBAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) static inline int pte_exec(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return !(pte_flags(pte) & _PAGE_NX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static inline int pte_special(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return pte_flags(pte) & _PAGE_SPECIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /* Entries that were set to PROT_NONE are inverted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static inline u64 protnone_mask(u64 val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline unsigned long pte_pfn(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) phys_addr_t pfn = pte_val(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) pfn ^= protnone_mask(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static inline unsigned long pmd_pfn(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) phys_addr_t pfn = pmd_val(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) pfn ^= protnone_mask(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static inline unsigned long pud_pfn(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) phys_addr_t pfn = pud_val(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) pfn ^= protnone_mask(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static inline unsigned long p4d_pfn(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static inline unsigned long pgd_pfn(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define p4d_leaf p4d_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static inline int p4d_large(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /* No 512 GiB pages yet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #define pte_page(pte) pfn_to_page(pte_pfn(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #define pmd_leaf pmd_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static inline int pmd_large(pmd_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return pmd_flags(pte) & _PAGE_PSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) /* NOTE: when predicate huge page, consider also pmd_devmap, or use pmd_large */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static inline int pmd_trans_huge(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static inline int pud_trans_huge(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #define has_transparent_hugepage has_transparent_hugepage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static inline int has_transparent_hugepage(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return boot_cpu_has(X86_FEATURE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static inline int pmd_devmap(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return !!(pmd_val(pmd) & _PAGE_DEVMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static inline int pud_devmap(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return !!(pud_val(pud) & _PAGE_DEVMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static inline int pud_devmap(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static inline int pgd_devmap(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) pteval_t v = native_pte_val(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return native_make_pte(v | set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) pteval_t v = native_pte_val(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return native_make_pte(v & ~clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) static inline int pte_uffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return pte_flags(pte) & _PAGE_UFFD_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) static inline pte_t pte_mkuffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return pte_set_flags(pte, _PAGE_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static inline pte_t pte_clear_uffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return pte_clear_flags(pte, _PAGE_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static inline pte_t pte_mkclean(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return pte_clear_flags(pte, _PAGE_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static inline pte_t pte_mkold(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return pte_clear_flags(pte, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static inline pte_t pte_wrprotect(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return pte_clear_flags(pte, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static inline pte_t pte_mkexec(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return pte_clear_flags(pte, _PAGE_NX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static inline pte_t pte_mkdirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) static inline pte_t pte_mkyoung(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return pte_set_flags(pte, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static inline pte_t pte_mkwrite(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) return pte_set_flags(pte, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) static inline pte_t pte_mkhuge(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return pte_set_flags(pte, _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) static inline pte_t pte_clrhuge(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return pte_clear_flags(pte, _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static inline pte_t pte_mkglobal(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return pte_set_flags(pte, _PAGE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) static inline pte_t pte_clrglobal(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return pte_clear_flags(pte, _PAGE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static inline pte_t pte_mkspecial(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return pte_set_flags(pte, _PAGE_SPECIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static inline pte_t pte_mkdevmap(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) pmdval_t v = native_pmd_val(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return native_make_pmd(v | set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) pmdval_t v = native_pmd_val(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return native_make_pmd(v & ~clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static inline int pmd_uffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return pmd_flags(pmd) & _PAGE_UFFD_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) static inline pmd_t pmd_mkuffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return pmd_set_flags(pmd, _PAGE_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) return pmd_clear_flags(pmd, _PAGE_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static inline pmd_t pmd_mkold(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return pmd_clear_flags(pmd, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static inline pmd_t pmd_mkclean(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return pmd_clear_flags(pmd, _PAGE_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static inline pmd_t pmd_wrprotect(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return pmd_clear_flags(pmd, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static inline pmd_t pmd_mkdirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static inline pmd_t pmd_mkdevmap(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return pmd_set_flags(pmd, _PAGE_DEVMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static inline pmd_t pmd_mkhuge(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return pmd_set_flags(pmd, _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static inline pmd_t pmd_mkyoung(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return pmd_set_flags(pmd, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static inline pmd_t pmd_mkwrite(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return pmd_set_flags(pmd, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) pudval_t v = native_pud_val(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return native_make_pud(v | set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) pudval_t v = native_pud_val(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return native_make_pud(v & ~clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static inline pud_t pud_mkold(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return pud_clear_flags(pud, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static inline pud_t pud_mkclean(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return pud_clear_flags(pud, _PAGE_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static inline pud_t pud_wrprotect(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return pud_clear_flags(pud, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static inline pud_t pud_mkdirty(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static inline pud_t pud_mkdevmap(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return pud_set_flags(pud, _PAGE_DEVMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) static inline pud_t pud_mkhuge(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return pud_set_flags(pud, _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static inline pud_t pud_mkyoung(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return pud_set_flags(pud, _PAGE_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) static inline pud_t pud_mkwrite(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return pud_set_flags(pud, _PAGE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static inline int pte_soft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return pte_flags(pte) & _PAGE_SOFT_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) static inline int pmd_soft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static inline int pud_soft_dirty(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return pud_flags(pud) & _PAGE_SOFT_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) static inline pte_t pte_mksoft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static inline pud_t pud_mksoft_dirty(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) static inline pte_t pte_clear_soft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static inline pud_t pud_clear_soft_dirty(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * Mask out unsupported bits in a present pgprot. Non-present pgprots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * can use those bits for other purposes, so leave them be.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) pgprotval_t protval = pgprot_val(pgprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (protval & _PAGE_PRESENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) protval &= __supported_pte_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return protval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static inline pgprotval_t check_pgprot(pgprot_t pgprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) pgprotval_t massaged_val = massage_pgprot(pgprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /* mmdebug.h can not be included here because of dependencies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) #ifdef CONFIG_DEBUG_VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) WARN_ONCE(pgprot_val(pgprot) != massaged_val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) "attempted to set unsupported pgprot: %016llx "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) "bits: %016llx supported: %016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) (u64)pgprot_val(pgprot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) (u64)pgprot_val(pgprot) ^ massaged_val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) (u64)__supported_pte_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return massaged_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) pfn ^= protnone_mask(pgprot_val(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) pfn &= PTE_PFN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) return __pte(pfn | check_pgprot(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) pfn ^= protnone_mask(pgprot_val(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) pfn &= PHYSICAL_PMD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return __pmd(pfn | check_pgprot(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) pfn ^= protnone_mask(pgprot_val(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) pfn &= PHYSICAL_PUD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return __pud(pfn | check_pgprot(pgprot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) static inline pmd_t pmd_mkinvalid(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return pfn_pmd(pmd_pfn(pmd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) pteval_t val = pte_val(pte), oldval = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * Chop off the NX bit (if present), and add the NX portion of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * the newprot (if present):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) val &= _PAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return __pte(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) pmdval_t val = pmd_val(pmd), oldval = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) val &= _HPAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) return __pmd(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * mprotect needs to preserve PAT and encryption bits when updating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * vm_page_prot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) #define pgprot_modify pgprot_modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) pgprotval_t addbits = pgprot_val(newprot) & ~_PAGE_CHG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) return __pgprot(preservebits | addbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) #define pte_pgprot(x) __pgprot(pte_flags(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) #define pmd_pgprot(x) __pgprot(pmd_flags(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) #define pud_pgprot(x) __pgprot(pud_flags(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) #define p4d_pgprot(x) __pgprot(p4d_flags(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) #define canon_pgprot(p) __pgprot(massage_pgprot(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) return canon_pgprot(prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) enum page_cache_mode pcm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) enum page_cache_mode new_pcm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) * PAT type is always WB for untracked ranges, so no need to check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * Certain new memtypes are not allowed with certain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * requested memtype:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * - request is uncached, return cannot be write-back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * - request is write-combine, return cannot be write-back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * - request is write-through, return cannot be write-back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * - request is write-through, return cannot be write-combine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) new_pcm == _PAGE_CACHE_MODE_WB) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) (pcm == _PAGE_CACHE_MODE_WC &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) new_pcm == _PAGE_CACHE_MODE_WB) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) (pcm == _PAGE_CACHE_MODE_WT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) new_pcm == _PAGE_CACHE_MODE_WB) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) (pcm == _PAGE_CACHE_MODE_WT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) new_pcm == _PAGE_CACHE_MODE_WC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) pmd_t *populate_extra_pmd(unsigned long vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) pte_t *populate_extra_pte(unsigned long vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) #ifdef CONFIG_PAGE_TABLE_ISOLATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * Take a PGD location (pgdp) and a pgd value that needs to be set there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * Populates the user and returns the resulting PGD that must be set in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * the kernel copy of the page tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (!static_cpu_has(X86_FEATURE_PTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) return pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return __pti_set_user_pgtbl(pgdp, pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) #else /* CONFIG_PAGE_TABLE_ISOLATION */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) #endif /* CONFIG_PAGE_TABLE_ISOLATION */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) #endif /* __ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) # include <asm/pgtable_32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) # include <asm/pgtable_64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) #include <linux/mm_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) #include <linux/mmdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) static inline int pte_none(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) #define __HAVE_ARCH_PTE_SAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) static inline int pte_same(pte_t a, pte_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return a.pte == b.pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static inline int pte_present(pte_t a)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) static inline int pte_devmap(pte_t a)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) #define pte_accessible pte_accessible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (pte_flags(a) & _PAGE_PRESENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if ((pte_flags(a) & _PAGE_PROTNONE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) mm_tlb_flush_pending(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) static inline int pmd_present(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * Checking for _PAGE_PSE is needed too because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * split_huge_page will temporarily clear the present bit (but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * the _PAGE_PSE flag will remain set at all times while the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * _PAGE_PRESENT bit is clear).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) #ifdef CONFIG_NUMA_BALANCING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * These work without NUMA balancing but the kernel does not care. See the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * comment in include/linux/pgtable.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static inline int pte_protnone(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) == _PAGE_PROTNONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) static inline int pmd_protnone(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) == _PAGE_PROTNONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) #endif /* CONFIG_NUMA_BALANCING */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) static inline int pmd_none(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /* Only check low word on 32-bit platforms, since it might be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) out of sync with upper half. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) unsigned long val = native_pmd_val(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) static inline unsigned long pmd_page_vaddr(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * Currently stuck as a macro due to indirect forward reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * linux/mmzone.h's __section_mem_map_addr() definition:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * Conversion functions: convert a page and protection to a page entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * and a page entry and page directory to the page they refer to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * (Currently stuck as a macro because of indirect forward reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * to linux/mm.h:page_to_nid())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) static inline int pmd_bad(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) static inline unsigned long pages_to_mb(unsigned long npg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) return npg >> (20 - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) #if CONFIG_PGTABLE_LEVELS > 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) static inline int pud_none(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) static inline int pud_present(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) return pud_flags(pud) & _PAGE_PRESENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) static inline unsigned long pud_page_vaddr(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * Currently stuck as a macro due to indirect forward reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * linux/mmzone.h's __section_mem_map_addr() definition:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) #define pud_page(pud) pfn_to_page(pud_pfn(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) #define pud_leaf pud_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) static inline int pud_large(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) (_PAGE_PSE | _PAGE_PRESENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) static inline int pud_bad(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) #define pud_leaf pud_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) static inline int pud_large(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) #endif /* CONFIG_PGTABLE_LEVELS > 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) #if CONFIG_PGTABLE_LEVELS > 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) static inline int p4d_none(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) static inline int p4d_present(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return p4d_flags(p4d) & _PAGE_PRESENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) static inline unsigned long p4d_page_vaddr(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * Currently stuck as a macro due to indirect forward reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * linux/mmzone.h's __section_mem_map_addr() definition:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) static inline int p4d_bad(p4d_t p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) ignore_flags |= _PAGE_NX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) return (p4d_flags(p4d) & ~ignore_flags) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) #endif /* CONFIG_PGTABLE_LEVELS > 3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) static inline unsigned long p4d_index(unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) #if CONFIG_PGTABLE_LEVELS > 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) static inline int pgd_present(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (!pgtable_l5_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return pgd_flags(pgd) & _PAGE_PRESENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) static inline unsigned long pgd_page_vaddr(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * Currently stuck as a macro due to indirect forward reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * linux/mmzone.h's __section_mem_map_addr() definition:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) #define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) /* to find an entry in a page-table-directory. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) if (!pgtable_l5_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return (p4d_t *)pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) static inline int pgd_bad(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) unsigned long ignore_flags = _PAGE_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (!pgtable_l5_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) ignore_flags |= _PAGE_NX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) static inline int pgd_none(pgd_t pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) if (!pgtable_l5_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * There is no need to do a workaround for the KNL stray
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * A/D bit erratum here. PGDs only point to page tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * except on 32-bit non-PAE which is not supported on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * KNL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return !native_pgd_val(pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) #endif /* CONFIG_PGTABLE_LEVELS > 4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) #endif /* __ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) #define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) #define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) extern int direct_gbpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) void init_mem_mapping(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) void early_alloc_pgt_buf(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) extern void memblock_find_dma_reserve(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) void __init poking_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) unsigned long init_memory_mapping(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) unsigned long end, pgprot_t prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) extern pgd_t trampoline_pgd_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /* local pte updates need not use xchg for locking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) pte_t res = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) /* Pure native function needs no input for mm, addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) native_pte_clear(NULL, 0, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) pmd_t res = *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) native_pmd_clear(pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) pud_t res = *pudp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) native_pud_clear(pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) pte_t *ptep, pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) set_pte(ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) pmd_t *pmdp, pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) set_pmd(pmdp, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) pud_t *pudp, pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) native_set_pud(pudp, pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * We only update the dirty/accessed state if we set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * the dirty bit by hand in the kernel, since the hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) * will do the accessed bit for us, and we don't want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) * race with other CPU's that might be updating the dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * bit at the same time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) struct vm_area_struct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) extern int ptep_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) unsigned long address, pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) pte_t entry, int dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) unsigned long addr, pte_t *ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) extern int ptep_clear_flush_young(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) unsigned long address, pte_t *ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) pte_t pte = native_ptep_get_and_clear(ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) unsigned long addr, pte_t *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) int full)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) pte_t pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) if (full) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * Full address destruction in progress; paravirt does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * care about updates and native needs no locking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) pte = native_local_ptep_get_and_clear(ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) pte = ptep_get_and_clear(mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) #define __HAVE_ARCH_PTEP_SET_WRPROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) static inline void ptep_set_wrprotect(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) #define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) extern int pmdp_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) unsigned long address, pmd_t *pmdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) pmd_t entry, int dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) extern int pudp_set_access_flags(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) unsigned long address, pud_t *pudp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) pud_t entry, int dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) unsigned long addr, pmd_t *pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) unsigned long addr, pud_t *pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) unsigned long address, pmd_t *pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) #define pmd_write pmd_write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) static inline int pmd_write(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) return pmd_flags(pmd) & _PAGE_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) pmd_t *pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) return native_pmdp_get_and_clear(pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) unsigned long addr, pud_t *pudp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) return native_pudp_get_and_clear(pudp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) #define __HAVE_ARCH_PMDP_SET_WRPROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) static inline void pmdp_set_wrprotect(struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) unsigned long addr, pmd_t *pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) #define pud_write pud_write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) static inline int pud_write(pud_t pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) return pud_flags(pud) & _PAGE_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) #ifndef pmdp_establish
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) #define pmdp_establish pmdp_establish
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) unsigned long address, pmd_t *pmdp, pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) if (IS_ENABLED(CONFIG_SMP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) return xchg(pmdp, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) pmd_t old = *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) WRITE_ONCE(*pmdp, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) return old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * Page table pages are page-aligned. The lower half of the top
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * level is used for userspace and the top half for the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * Returns true for parts of the PGD that map userspace and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * false for the parts that map the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) static inline bool pgdp_maps_userspace(void *__ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) unsigned long ptr = (unsigned long)__ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) #define pgd_leaf pgd_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) static inline int pgd_large(pgd_t pgd) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) #ifdef CONFIG_PAGE_TABLE_ISOLATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * the user one is in the last 4k. To switch between them, you
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * just need to flip the 12th bit in their addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) #define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * This generates better code than the inline assembly in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * __set_bit().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) static inline void *ptr_set_bit(void *ptr, int bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) unsigned long __ptr = (unsigned long)ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) __ptr |= BIT(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) return (void *)__ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) static inline void *ptr_clear_bit(void *ptr, int bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) unsigned long __ptr = (unsigned long)ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) __ptr &= ~BIT(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) return (void *)__ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) #endif /* CONFIG_PAGE_TABLE_ISOLATION */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * dst - pointer to pgd range anwhere on a pgd page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) * src - ""
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) * count - the number of pgds to copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * dst and src can be on the same page, but the range must not overlap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * and must not cross a page boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) memcpy(dst, src, count * sizeof(pgd_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) #ifdef CONFIG_PAGE_TABLE_ISOLATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (!static_cpu_has(X86_FEATURE_PTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) /* Clone the user space pgd as well */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) count * sizeof(pgd_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) #define PTE_SHIFT ilog2(PTRS_PER_PTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) static inline int page_level_shift(enum pg_level level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) static inline unsigned long page_level_size(enum pg_level level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) return 1UL << page_level_shift(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) static inline unsigned long page_level_mask(enum pg_level level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) return ~(page_level_size(level) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) * The x86 doesn't have any external MMU info: the kernel page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * tables contain all the necessary information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) static inline void update_mmu_cache(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) unsigned long addr, pte_t *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) unsigned long addr, pmd_t *pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) unsigned long addr, pud_t *pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) static inline int pte_swp_soft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) static inline int pmd_swp_soft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) return pte_set_flags(pte, _PAGE_SWP_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) static inline int pte_swp_uffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) return pte_flags(pte) & _PAGE_SWP_UFFD_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) static inline int pmd_swp_uffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) #define PKRU_AD_BIT 0x1u
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) #define PKRU_WD_BIT 0x2u
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) #define PKRU_BITS_PER_PKEY 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) extern u32 init_pkru_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) #define init_pkru_value 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * Access-disable disables writes too so we need to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * both bits here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) static inline u16 pte_flags_pkey(unsigned long pte_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) /* ifdef to avoid doing 59-bit shift on 32-bit values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) static inline bool __pkru_allows_pkey(u16 pkey, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) u32 pkru = read_pkru();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) if (!__pkru_allows_read(pkru, pkey))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) if (write && !__pkru_allows_write(pkru, pkey))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) * 'pteval' can come from a PTE, PMD or PUD. We only check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) * same value on all 3 types.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) static inline bool __pte_access_permitted(unsigned long pteval, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) need_pte_bits |= _PAGE_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if ((pteval & need_pte_bits) != need_pte_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) #define pte_access_permitted pte_access_permitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static inline bool pte_access_permitted(pte_t pte, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) return __pte_access_permitted(pte_val(pte), write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) #define pmd_access_permitted pmd_access_permitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) static inline bool pmd_access_permitted(pmd_t pmd, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return __pte_access_permitted(pmd_val(pmd), write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) #define pud_access_permitted pud_access_permitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) static inline bool pud_access_permitted(pud_t pud, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) return __pte_access_permitted(pud_val(pud), write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) #define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) static inline bool arch_has_pfn_modify_check(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) return boot_cpu_has_bug(X86_BUG_L1TF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) #define arch_faults_on_old_pte arch_faults_on_old_pte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) static inline bool arch_faults_on_old_pte(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) #endif /* __ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) #endif /* _ASM_X86_PGTABLE_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags