^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Handling Page Tables through page fragments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) void pte_frag_destroy(void *pte_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) page = virt_to_page(pte_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /* drop all the pending references */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* We allow PTE_FRAG_NR fragments from a PTE page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) if (atomic_sub_and_test(PTE_FRAG_NR - count, &page->pt_frag_refcount)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) pgtable_pte_page_dtor(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static pte_t *get_pte_from_cache(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) void *pte_frag, *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (PTE_FRAG_NR == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) spin_lock(&mm->page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) ret = pte_frag_get(&mm->context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) pte_frag = ret + PTE_FRAG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * If we have taken up all the fragments mark PTE page NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) pte_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) pte_frag_set(&mm->context, pte_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) spin_unlock(&mm->page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return (pte_t *)ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) void *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (!kernel) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (!pgtable_pte_page_ctor(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) page = alloc_page(PGALLOC_GFP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) atomic_set(&page->pt_frag_refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) ret = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * if we support only one fragment just return the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * allocated page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (PTE_FRAG_NR == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) spin_lock(&mm->page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * If we find pgtable_page set, we return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * the allocated page with single fragement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (likely(!pte_frag_get(&mm->context))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) atomic_set(&page->pt_frag_refcount, PTE_FRAG_NR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) spin_unlock(&mm->page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return (pte_t *)ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) pte = get_pte_from_cache(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return __alloc_for_ptecache(mm, kernel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) void pte_fragment_free(unsigned long *table, int kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct page *page = virt_to_page(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (PageReserved(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return free_reserved_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (atomic_dec_and_test(&page->pt_frag_refcount)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (!kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) pgtable_pte_page_dtor(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags