^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _S390_TLB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _S390_TLB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * TLB flushing on s390 is complicated. The following requirement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * from the principles of operation is the most arduous:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * "A valid table entry must not be changed while it is attached
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * to any CPU and may be used for translation by that CPU except to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * table entry, or (3) make a change by means of a COMPARE AND SWAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * AND PURGE instruction that purges the TLB."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * The modification of a pte of an active mm struct therefore is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * a two step process: i) invalidate the pte, ii) store the new pte.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * This is true for the page protection bit as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * The only possible optimization is to flush at the beginning of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * a tlb_gather_mmu cycle if the mm_struct is currently not in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * Pages used for the page tables is a different story. FIXME: more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) void __tlb_remove_table(void *_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static inline void tlb_flush(struct mmu_gather *tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct page *page, int page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define tlb_start_vma(tlb, vma) do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define tlb_end_vma(tlb, vma) do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define tlb_flush tlb_flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define pte_free_tlb pte_free_tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define pmd_free_tlb pmd_free_tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define p4d_free_tlb p4d_free_tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define pud_free_tlb pud_free_tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm-generic/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * Release the page cache reference for a pte removed by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * has already been freed, so just do free_page_and_swap_cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct page *page, int page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) free_page_and_swap_cache(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static inline void tlb_flush(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) __tlb_flush_mm_lazy(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * pte_free_tlb frees a pte table and clears the CRSTE for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * page table from the tlb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) __tlb_adjust_range(tlb, address, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) tlb->mm->context.flush_mm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) tlb->freed_tables = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) tlb->cleared_ptes = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * page_table_free_rcu takes care of the allocation bit masks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * of the 2K table fragments in the 4K page table page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * then calls tlb_remove_table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) page_table_free_rcu(tlb, (unsigned long *) pte, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * pmd_free_tlb frees a pmd table and clears the CRSTE for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * segment table entry from the tlb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * If the mm uses a two level page table the single pmd is freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * as the pgd. pmd_free_tlb checks the asce_limit against 2GB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * to avoid the double free of the pmd in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (mm_pmd_folded(tlb->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) pgtable_pmd_page_dtor(virt_to_page(pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __tlb_adjust_range(tlb, address, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) tlb->mm->context.flush_mm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) tlb->freed_tables = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) tlb->cleared_puds = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) tlb_remove_table(tlb, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * p4d_free_tlb frees a pud table and clears the CRSTE for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * region second table entry from the tlb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * If the mm uses a four level page table the single p4d is freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * as the pgd. p4d_free_tlb checks the asce_limit against 8PB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * to avoid the double free of the p4d in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (mm_p4d_folded(tlb->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) __tlb_adjust_range(tlb, address, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) tlb->mm->context.flush_mm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) tlb->freed_tables = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) tlb->cleared_p4ds = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) tlb_remove_table(tlb, p4d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * pud_free_tlb frees a pud table and clears the CRSTE for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * region third table entry from the tlb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * If the mm uses a three level page table the single pud is freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * as the pgd. pud_free_tlb checks the asce_limit against 4TB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * to avoid the double free of the pud in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (mm_pud_folded(tlb->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) tlb->mm->context.flush_mm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) tlb->freed_tables = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) tlb->cleared_puds = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) tlb_remove_table(tlb, pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #endif /* _S390_TLB_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags