^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/sched/task.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/mem_detect.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/sclp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/facility.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/setup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/uv.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) unsigned long kasan_vmax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) static unsigned long segment_pos __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) static unsigned long segment_low __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static unsigned long pgalloc_pos __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static unsigned long pgalloc_low __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static unsigned long pgalloc_freeable __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static bool has_edat __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static bool has_nx __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static void __init kasan_early_panic(const char *reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) sclp_early_printk(reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) disabled_wait();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static void * __init kasan_early_alloc_segment(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) segment_pos -= _SEGMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) if (segment_pos < segment_low)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) kasan_early_panic("out of memory during initialisation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return (void *)segment_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void * __init kasan_early_alloc_pages(unsigned int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) pgalloc_pos -= (PAGE_SIZE << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (pgalloc_pos < pgalloc_low)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) kasan_early_panic("out of memory during initialisation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return (void *)pgalloc_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static void * __init kasan_early_crst_alloc(unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if (table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) crst_table_init(table, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static pte_t * __init kasan_early_pte_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static void *pte_leftover;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (!pte_leftover) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) pte_leftover = kasan_early_alloc_pages(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) pte = pte_leftover + _PAGE_TABLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) pte = pte_leftover;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) pte_leftover = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) enum populate_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) POPULATE_ONE2ONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) POPULATE_MAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) POPULATE_ZERO_SHADOW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) POPULATE_SHALLOW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static void __init kasan_early_vmemmap_populate(unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) enum populate_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) pgd_t *pg_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) p4d_t *p4_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) pud_t *pu_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) pmd_t *pm_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) pte_t *pt_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (!has_nx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) pgt_prot_zero &= ~_PAGE_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pgt_prot = pgprot_val(PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) sgt_prot = pgprot_val(SEGMENT_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (!has_nx || mode == POPULATE_ONE2ONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) pgt_prot &= ~_PAGE_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * The first 1MB of 1:1 mapping is mapped with 4KB pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) while (address < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pg_dir = pgd_offset_k(address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (pgd_none(*pg_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (mode == POPULATE_ZERO_SHADOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) IS_ALIGNED(address, PGDIR_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) end - address >= PGDIR_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) pgd_populate(&init_mm, pg_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) kasan_early_shadow_p4d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) address = (address + PGDIR_SIZE) & PGDIR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) pgd_populate(&init_mm, pg_dir, p4_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) mode == POPULATE_SHALLOW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) address = (address + P4D_SIZE) & P4D_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) p4_dir = p4d_offset(pg_dir, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (p4d_none(*p4_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (mode == POPULATE_ZERO_SHADOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) IS_ALIGNED(address, P4D_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) end - address >= P4D_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) p4d_populate(&init_mm, p4_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) kasan_early_shadow_pud);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) address = (address + P4D_SIZE) & P4D_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) p4d_populate(&init_mm, p4_dir, pu_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) mode == POPULATE_SHALLOW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) address = (address + PUD_SIZE) & PUD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) pu_dir = pud_offset(p4_dir, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (pud_none(*pu_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (mode == POPULATE_ZERO_SHADOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) IS_ALIGNED(address, PUD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) end - address >= PUD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) pud_populate(&init_mm, pu_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) kasan_early_shadow_pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) address = (address + PUD_SIZE) & PUD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pud_populate(&init_mm, pu_dir, pm_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) pm_dir = pmd_offset(pu_dir, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (pmd_none(*pm_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (IS_ALIGNED(address, PMD_SIZE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) end - address >= PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (mode == POPULATE_ZERO_SHADOW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) address = (address + PMD_SIZE) & PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) } else if (has_edat && address) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) void *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (mode == POPULATE_ONE2ONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) page = (void *)address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) page = kasan_early_alloc_segment();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) memset(page, 0, _SEGMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) pmd_val(*pm_dir) = __pa(page) | sgt_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) address = (address + PMD_SIZE) & PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) pt_dir = kasan_early_pte_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) pmd_populate(&init_mm, pm_dir, pt_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) } else if (pmd_large(*pm_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) address = (address + PMD_SIZE) & PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) pt_dir = pte_offset_kernel(pm_dir, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (pte_none(*pt_dir)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) case POPULATE_ONE2ONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) page = (void *)address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) pte_val(*pt_dir) = __pa(page) | pgt_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) case POPULATE_MAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) page = kasan_early_alloc_pages(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) memset(page, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) pte_val(*pt_dir) = __pa(page) | pgt_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) case POPULATE_ZERO_SHADOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) page = kasan_early_shadow_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) case POPULATE_SHALLOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /* should never happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) address += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) unsigned long asce_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) asce_bits = asce_type | _ASCE_TABLE_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) S390_lowcore.user_asce = S390_lowcore.kernel_asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) __ctl_load(S390_lowcore.kernel_asce, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) __ctl_load(S390_lowcore.kernel_asce, 7, 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) __ctl_load(S390_lowcore.kernel_asce, 13, 13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static void __init kasan_enable_dat(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) psw_t psw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) psw.mask = __extract_psw();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) psw_bits(psw).dat = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) psw_bits(psw).as = PSW_BITS_AS_HOME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) __load_psw_mask(psw.mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static void __init kasan_early_detect_facilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (test_facility(8)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) has_edat = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) __ctl_set_bit(0, 23);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (!noexec_disabled && test_facility(130)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) has_nx = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) __ctl_set_bit(0, 20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static bool __init has_uv_sec_stor_limit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * keep these conditions in line with setup_uv()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (!is_prot_virt_host())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (is_prot_virt_guest())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!test_facility(158))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return !!uv_info.max_sec_stor_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) void __init kasan_early_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) unsigned long untracked_mem_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) unsigned long shadow_alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) unsigned long vmax_unlimited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) unsigned long initrd_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned long asce_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned long memsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) pte_t pte_z;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) kasan_early_detect_facilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (!has_nx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) pgt_prot &= ~_PAGE_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) memsize = get_mem_detect_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (!memsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) kasan_early_panic("cannot detect physical memory size\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /* respect mem= cmdline parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (memory_end_set && memsize > memory_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) memsize = memory_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) memsize = min(memsize, OLDMEM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) memsize = min(memsize, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /* 4 level paging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) crst_table_init((unsigned long *)early_pg_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) _REGION2_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (has_uv_sec_stor_limit())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) asce_type = _ASCE_TYPE_REGION2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* 3 level paging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) crst_table_init((unsigned long *)early_pg_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) _REGION3_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) asce_type = _ASCE_TYPE_REGION3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* init kasan zero shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) crst_table_init((unsigned long *)kasan_early_shadow_p4d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) p4d_val(p4d_z));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) crst_table_init((unsigned long *)kasan_early_shadow_pud,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) pud_val(pud_z));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) crst_table_init((unsigned long *)kasan_early_shadow_pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) pmd_val(pmd_z));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) initrd_end =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) pgalloc_low = max(pgalloc_low, initrd_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (pgalloc_low + shadow_alloc_size > memsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) kasan_early_panic("out of memory during initialisation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (has_edat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) segment_pos = round_down(memsize, _SEGMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) segment_low = segment_pos - shadow_alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) pgalloc_pos = segment_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) pgalloc_pos = memsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) init_mm.pgd = early_pg_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * Current memory layout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * +- 0 -------------+ +- shadow start -+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * | 1:1 ram mapping | /| 1/8 ram |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * | | / | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * +- end of ram ----+ / +----------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * | ... gap ... | / | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * | |/ | kasan |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * +- shadow start --+ | zero |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * | 1/8 addr space | | page |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * +- shadow end -+ | mapping |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * | ... gap ... |\ | (untracked) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * +- vmalloc area -+ \ | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * | vmalloc_size | \ | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * +- modules vaddr -+ \ +----------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * | 2Gb | \| unmapped | allocated per module
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * +-----------------+ +- shadow end ---+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * Current memory layout (KASAN_VMALLOC):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * +- 0 -------------+ +- shadow start -+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * | 1:1 ram mapping | /| 1/8 ram |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * | | / | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * +- end of ram ----+ / +----------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * | ... gap ... | / | kasan |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * | |/ | zero |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * +- shadow start --+ | page |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * | 1/8 addr space | | mapping |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * +- shadow end -+ | (untracked) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * | ... gap ... |\ | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * +- vmalloc area -+ \ +- vmalloc area -+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * | vmalloc_size | \ |shallow populate|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * +- modules vaddr -+ \ +- modules area -+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * | 2Gb | \|shallow populate|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * +-----------------+ +- shadow end ---+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* populate kasan shadow (for identity mapping and zero page mapping) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (IS_ENABLED(CONFIG_MODULES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) untracked_mem_end = kasan_vmax - MODULES_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /* shallowly populate kasan shadow for vmalloc and modules */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) kasan_early_vmemmap_populate(__sha(untracked_mem_end),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) __sha(kasan_vmax), POPULATE_SHALLOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /* populate kasan shadow for untracked memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) kasan_early_vmemmap_populate(__sha(max_physmem_end),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) __sha(untracked_mem_end),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) POPULATE_ZERO_SHADOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) kasan_early_vmemmap_populate(__sha(kasan_vmax),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) __sha(vmax_unlimited),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) POPULATE_ZERO_SHADOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) /* memory allocated for identity mapping structs will be freed later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) pgalloc_freeable = pgalloc_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* populate identity mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) kasan_set_pgd(early_pg_dir, asce_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) kasan_enable_dat();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* enable kasan */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) init_task.kasan_depth = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) sclp_early_printk("KernelAddressSanitizer initialized\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) void __init kasan_copy_shadow(pgd_t *pg_dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * At this point we are still running on early pages setup early_pg_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * while swapper_pg_dir has just been initialized with identity mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) pgd_t *pg_dir_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) pgd_t *pg_dir_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) p4d_t *p4_dir_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) p4d_t *p4_dir_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) pud_t *pu_dir_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) pud_t *pu_dir_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (!p4d_folded(*p4_dir_src)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* 4 level paging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) memcpy(p4_dir_dst, p4_dir_src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /* 3 level paging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) memcpy(pu_dir_dst, pu_dir_src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) (KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) void __init kasan_free_early_identity(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags