^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright IBM Corp. 2006
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/pfn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/setup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static DEFINE_MUTEX(vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static void __ref *vmem_alloc_pages(unsigned int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) unsigned long size = PAGE_SIZE << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) if (slab_is_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) return (void *)__get_free_pages(GFP_KERNEL, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) return (void *) memblock_phys_alloc(size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static void vmem_free_pages(unsigned long addr, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /* We don't expect boot memory to be removed ever. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) if (!slab_is_available() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) WARN_ON_ONCE(PageReserved(phys_to_page(addr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) free_pages(addr, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) void *vmem_crst_alloc(unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) table = vmem_alloc_pages(CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) crst_table_init(table, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) pte_t __ref *vmem_pte_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) unsigned long size = PTRS_PER_PTE * sizeof(pte_t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (slab_is_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) pte = (pte_t *) page_table_alloc(&init_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) pte = (pte_t *) memblock_phys_alloc(size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static void vmem_pte_free(unsigned long *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* We don't expect boot memory to be removed ever. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (!slab_is_available() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) WARN_ON_ONCE(PageReserved(virt_to_page(table))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) page_table_free(&init_mm, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define PAGE_UNUSED 0xFD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * from unused_pmd_start to next PMD_SIZE boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static unsigned long unused_pmd_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static void vmemmap_flush_unused_pmd(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (!unused_pmd_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) memset(__va(unused_pmd_start), PAGE_UNUSED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) ALIGN(unused_pmd_start, PMD_SIZE) - unused_pmd_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) unused_pmd_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static void __vmemmap_use_sub_pmd(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * As we expect to add in the same granularity as we remove, it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * sufficient to mark only some piece used to block the memmap page from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * getting removed (just in case the memmap never gets initialized,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * e.g., because the memory block never gets onlined).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) memset(__va(start), 0, sizeof(struct page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * We only optimize if the new used range directly follows the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * previously unused range (esp., when populating consecutive sections).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (unused_pmd_start == start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unused_pmd_start = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (likely(IS_ALIGNED(unused_pmd_start, PMD_SIZE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) unused_pmd_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) vmemmap_flush_unused_pmd();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) __vmemmap_use_sub_pmd(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void *page = __va(ALIGN_DOWN(start, PMD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) vmemmap_flush_unused_pmd();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Could be our memmap page is filled with PAGE_UNUSED already ... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) __vmemmap_use_sub_pmd(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* Mark the unused parts of the new memmap page PAGE_UNUSED. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (!IS_ALIGNED(start, PMD_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) memset(page, PAGE_UNUSED, start - __pa(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * consecutive sections. Remember for the last added PMD the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * unused range in the populated PMD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (!IS_ALIGNED(end, PMD_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unused_pmd_start = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* Returns true if the PMD is completely unused and can be freed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) void *page = __va(ALIGN_DOWN(start, PMD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) vmemmap_flush_unused_pmd();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) memset(__va(start), PAGE_UNUSED, end - start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return !memchr_inv(page, PAGE_UNUSED, PMD_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) unsigned long end, bool add, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) unsigned long prot, pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) prot = pgprot_val(PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!MACHINE_HAS_NX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) prot &= ~_PAGE_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) pte = pte_offset_kernel(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) for (; addr < end; addr += PAGE_SIZE, pte++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (!add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (pte_none(*pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (!direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) vmem_free_pages(pfn_to_phys(pte_pfn(*pte)), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) pte_clear(&init_mm, addr, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) } else if (pte_none(*pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (!direct) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) void *new_page = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (!new_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) pte_val(*pte) = __pa(new_page) | prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) pte_val(*pte) = addr | prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) update_page_count(PG_DIRECT_MAP_4K, add ? pages : -pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static void try_free_pte_table(pmd_t *pmd, unsigned long start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* We can safely assume this is fully in 1:1 mapping & vmemmap area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) pte = pte_offset_kernel(pmd, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (!pte_none(*pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) vmem_pte_free(__va(pmd_deref(*pmd)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) pmd_clear(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) unsigned long end, bool add, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) unsigned long next, prot, pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) prot = pgprot_val(SEGMENT_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (!MACHINE_HAS_NX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) prot &= ~_SEGMENT_ENTRY_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) for (; addr < end; addr = next, pmd++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) next = pmd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (!add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (pmd_large(*pmd) && !add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (IS_ALIGNED(addr, PMD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) IS_ALIGNED(next, PMD_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) pmd_clear(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) } else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) pmd_clear(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) } else if (pmd_none(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (IS_ALIGNED(addr, PMD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) IS_ALIGNED(next, PMD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) MACHINE_HAS_EDAT1 && addr && direct &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) !debug_pagealloc_enabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pmd_val(*pmd) = addr | prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) } else if (!direct && MACHINE_HAS_EDAT1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) void *new_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * Use 1MB frames for vmemmap if available. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * always use large frames even if they are only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * partially used. Otherwise we would have also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * page tables since vmemmap_populate gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * called for each section separately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) new_page = vmemmap_alloc_block(PMD_SIZE, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (new_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pmd_val(*pmd) = __pa(new_page) | prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (!IS_ALIGNED(addr, PMD_SIZE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) !IS_ALIGNED(next, PMD_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) vmemmap_use_new_sub_pmd(addr, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) pte = vmem_pte_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) pmd_populate(&init_mm, pmd, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) } else if (pmd_large(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (!direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) vmemmap_use_sub_pmd(addr, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) ret = modify_pte_table(pmd, addr, next, add, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (!add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) try_free_pte_table(pmd, addr & PMD_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) update_page_count(PG_DIRECT_MAP_1M, add ? pages : -pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static void try_free_pmd_table(pud_t *pud, unsigned long start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) const unsigned long end = start + PUD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (end > VMALLOC_START)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) #ifdef CONFIG_KASAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) pmd = pmd_offset(pud, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) for (i = 0; i < PTRS_PER_PMD; i++, pmd++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (!pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) vmem_free_pages(pud_deref(*pud), CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) pud_clear(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) bool add, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) unsigned long next, prot, pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) prot = pgprot_val(REGION3_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (!MACHINE_HAS_NX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) prot &= ~_REGION_ENTRY_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) for (; addr < end; addr = next, pud++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) next = pud_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (pud_none(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (pud_large(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (IS_ALIGNED(addr, PUD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) IS_ALIGNED(next, PUD_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) pud_clear(pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) } else if (pud_none(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (IS_ALIGNED(addr, PUD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) IS_ALIGNED(next, PUD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) MACHINE_HAS_EDAT2 && addr && direct &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) !debug_pagealloc_enabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) pud_val(*pud) = addr | prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (!pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) pud_populate(&init_mm, pud, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) } else if (pud_large(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ret = modify_pmd_table(pud, addr, next, add, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (!add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) try_free_pmd_table(pud, addr & PUD_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) update_page_count(PG_DIRECT_MAP_2G, add ? pages : -pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static void try_free_pud_table(p4d_t *p4d, unsigned long start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) const unsigned long end = start + P4D_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (end > VMALLOC_START)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) #ifdef CONFIG_KASAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) pud = pud_offset(p4d, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (!pud_none(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) p4d_clear(p4d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) bool add, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) for (; addr < end; addr = next, p4d++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) next = p4d_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (!add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (p4d_none(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) } else if (p4d_none(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (!pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) p4d_populate(&init_mm, p4d, pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) ret = modify_pud_table(p4d, addr, next, add, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (!add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) try_free_pud_table(p4d, addr & P4D_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static void try_free_p4d_table(pgd_t *pgd, unsigned long start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) const unsigned long end = start + PGDIR_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (end > VMALLOC_START)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #ifdef CONFIG_KASAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) p4d = p4d_offset(pgd, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (!p4d_none(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) pgd_clear(pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static int modify_pagetable(unsigned long start, unsigned long end, bool add,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned long addr, next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) for (addr = start; addr < end; addr = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) next = pgd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) pgd = pgd_offset_k(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (!add) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (pgd_none(*pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) } else if (pgd_none(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (!p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) pgd_populate(&init_mm, pgd, p4d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) ret = modify_p4d_table(pgd, addr, next, add, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (!add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) try_free_p4d_table(pgd, addr & PGDIR_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (!add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) flush_tlb_kernel_range(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static int add_pagetable(unsigned long start, unsigned long end, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return modify_pagetable(start, end, true, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static int remove_pagetable(unsigned long start, unsigned long end, bool direct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return modify_pagetable(start, end, false, direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * Add a physical memory range to the 1:1 mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) static int vmem_add_range(unsigned long start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return add_pagetable(start, start + size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * Remove a physical memory range from the 1:1 mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static void vmem_remove_range(unsigned long start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) remove_pagetable(start, start + size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * Add a backed mem_map array to the virtual mem_map array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) mutex_lock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /* We don't care about the node, just use NUMA_NO_NODE on allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) ret = add_pagetable(start, end, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) remove_pagetable(start, end, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) mutex_unlock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) void vmemmap_free(unsigned long start, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) mutex_lock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) remove_pagetable(start, end, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) mutex_unlock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) void vmem_remove_mapping(unsigned long start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) mutex_lock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) vmem_remove_range(start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) mutex_unlock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) int vmem_add_mapping(unsigned long start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (start + size > VMEM_MAX_PHYS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) start + size < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) mutex_lock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) ret = vmem_add_range(start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) vmem_remove_range(start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) mutex_unlock(&vmem_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * map whole physical memory to virtual memory (identity mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * we reserve enough space in the vmalloc area for vmemmap to hotplug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * additional memory segments.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) void __init vmem_map_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) phys_addr_t base, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) u64 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) for_each_mem_range(i, &base, &end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) vmem_add_range(base, end - base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) __set_memory((unsigned long)_stext,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) (unsigned long)(_etext - _stext) >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) SET_MEMORY_RO | SET_MEMORY_X);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) __set_memory((unsigned long)_etext,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) (unsigned long)(__end_rodata - _etext) >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) SET_MEMORY_RO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) __set_memory((unsigned long)_sinittext,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) (unsigned long)(_einittext - _sinittext) >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) SET_MEMORY_RO | SET_MEMORY_X);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) __set_memory(__stext_dma, (__etext_dma - __stext_dma) >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) SET_MEMORY_RO | SET_MEMORY_X);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /* we need lowcore executable for our LPSWE instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) set_memory_x(0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) pr_info("Write protected kernel read-only data: %luk\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) (unsigned long)(__end_rodata - _stext) >> 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags