^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * PowerPC version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * and Cort Dougan (PReP) (cort@cs.nmt.edu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 1996 Paul Mackerras
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Derived from "arch/i386/mm/init.c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Dave Engebretsen <engebret@us.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Rework for PPC64 port.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #undef DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/kup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) phys_addr_t memstart_addr __ro_after_init = (phys_addr_t)~0ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) EXPORT_SYMBOL_GPL(memstart_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) phys_addr_t kernstart_addr __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) EXPORT_SYMBOL_GPL(kernstart_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) unsigned long kernstart_virt_addr __ro_after_init = KERNELBASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) EXPORT_SYMBOL_GPL(kernstart_virt_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static bool disable_kuep = !IS_ENABLED(CONFIG_PPC_KUEP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static int __init parse_nosmep(char *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) disable_kuep = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) pr_warn("Disabling Kernel Userspace Execution Prevention\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) early_param("nosmep", parse_nosmep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static int __init parse_nosmap(char *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) disable_kuap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) pr_warn("Disabling Kernel Userspace Access Protection\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) early_param("nosmap", parse_nosmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) void __ref setup_kup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) setup_kuep(disable_kuep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) setup_kuap(disable_kuap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define CTOR(shift) static void ctor_##shift(void *addr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) memset(addr, 0, sizeof(void *) << (shift)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) CTOR(0); CTOR(1); CTOR(2); CTOR(3); CTOR(4); CTOR(5); CTOR(6); CTOR(7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) CTOR(8); CTOR(9); CTOR(10); CTOR(11); CTOR(12); CTOR(13); CTOR(14); CTOR(15);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static inline void (*ctor(int shift))(void *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) BUILD_BUG_ON(MAX_PGTABLE_INDEX_SIZE != 15);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) switch (shift) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) case 0: return ctor_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) case 1: return ctor_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) case 2: return ctor_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) case 3: return ctor_3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) case 4: return ctor_4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) case 5: return ctor_5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) case 6: return ctor_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) case 7: return ctor_7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) case 8: return ctor_8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) case 9: return ctor_9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) case 10: return ctor_10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) case 11: return ctor_11;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) case 12: return ctor_12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) case 13: return ctor_13;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) case 14: return ctor_14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) case 15: return ctor_15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) EXPORT_SYMBOL_GPL(pgtable_cache); /* used by kvm_hv module */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * Create a kmem_cache() for pagetables. This is not used for PTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * pages - they're linked to struct page, come from the normal free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * pages pool and have a different entry size (see real_pte_t) to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * everything else. Caches created by this function are used for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * the higher level pagetables, and for hugepage pagetables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) void pgtable_cache_add(unsigned int shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned long table_size = sizeof(void *) << shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned long align = table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* When batching pgtable pointers for RCU freeing, we store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * the index size in the low bits. Table alignment must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * big enough to fit it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Likewise, hugeapge pagetable pointers contain a (different)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * shift value in the low bits. All tables must be aligned so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * as to leave enough 0 bits in the address to contain it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) HUGEPD_SHIFT_MASK + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) struct kmem_cache *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* It would be nice if this was a BUILD_BUG_ON(), but at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * moment, gcc doesn't seem to recognize is_power_of_2 as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * constant expression, so so much for that. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) BUG_ON(!is_power_of_2(minalign));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (PGT_CACHE(shift))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return; /* Already have a cache of this size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) align = max_t(unsigned long, align, minalign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) new = kmem_cache_create(name, table_size, align, 0, ctor(shift));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) panic("Could not allocate pgtable cache for order %d", shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) kfree(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) pgtable_cache[shift] = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) pr_debug("Allocated pgtable cache for order %d\n", shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) EXPORT_SYMBOL_GPL(pgtable_cache_add); /* used by kvm_hv module */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) void pgtable_cache_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) pgtable_cache_add(PGD_INDEX_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (PMD_CACHE_INDEX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) pgtable_cache_add(PMD_CACHE_INDEX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * In all current configs, when the PUD index exists it's the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * same size as either the pgd or pmd index except with THP enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * on book3s 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (PUD_CACHE_INDEX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) pgtable_cache_add(PUD_CACHE_INDEX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags