^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * This file contains kasan initialization code for ARM64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2015 Samsung Electronics Co., Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #define pr_fmt(fmt) "kasan: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sched/task.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/start_kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/kernel-pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * The p*d_populate functions call virt_to_phys implicitly so they can't be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * directly on kernel symbols (bm_p*d). All the early functions are called too
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * early to use lm_alias so __p*d_populate functions must be used to populate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * with the physical address from __pa_symbol.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static phys_addr_t __init kasan_alloc_zeroed_page(int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) __pa(MAX_DMA_ADDRESS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) MEMBLOCK_ALLOC_KASAN, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) __func__, PAGE_SIZE, PAGE_SIZE, node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) __pa(MAX_DMA_ADDRESS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return __pa(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static phys_addr_t __init kasan_alloc_raw_page(int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) void *p = memblock_alloc_try_nid_raw(PAGE_SIZE, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) __pa(MAX_DMA_ADDRESS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) MEMBLOCK_ALLOC_KASAN, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) __func__, PAGE_SIZE, PAGE_SIZE, node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) __pa(MAX_DMA_ADDRESS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return __pa(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (pmd_none(READ_ONCE(*pmdp))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) phys_addr_t pte_phys = early ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) __pa_symbol(kasan_early_shadow_pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) : kasan_alloc_zeroed_page(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return early ? pte_offset_kimg(pmdp, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) : pte_offset_kernel(pmdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (pud_none(READ_ONCE(*pudp))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) phys_addr_t pmd_phys = early ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) __pa_symbol(kasan_early_shadow_pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) : kasan_alloc_zeroed_page(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (p4d_none(READ_ONCE(*p4dp))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) phys_addr_t pud_phys = early ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __pa_symbol(kasan_early_shadow_pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) : kasan_alloc_zeroed_page(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __p4d_populate(p4dp, pud_phys, PMD_TYPE_TABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned long end, int node, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) phys_addr_t page_phys = early ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) __pa_symbol(kasan_early_shadow_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) : kasan_alloc_raw_page(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (!early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) memset(__va(page_phys), KASAN_SHADOW_INIT, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) next = addr + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) } while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned long end, int node, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) next = pmd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) kasan_pte_populate(pmdp, addr, next, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unsigned long end, int node, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) next = pud_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) kasan_pmd_populate(pudp, addr, next, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned long end, int node, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) p4d_t *p4dp = p4d_offset(pgdp, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) next = p4d_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) kasan_pud_populate(p4dp, addr, next, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) } while (p4dp++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int node, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) pgd_t *pgdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) pgdp = pgd_offset_k(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) next = pgd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) kasan_p4d_populate(pgdp, addr, next, node, early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) } while (pgdp++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /* The early shadow maps everything to a single page of zeroes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) asmlinkage void __init kasan_early_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) BUILD_BUG_ON(KASAN_SHADOW_OFFSET !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS), PGDIR_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS_MIN), PGDIR_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /* Set up full kasan mappings, ensuring that the mapped pages are zeroed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static void __init kasan_map_populate(unsigned long start, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * Copy the current shadow region into a new pgdir.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) void __init kasan_copy_shadow(pgd_t *pgdir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) pgd_t *pgdp, *pgdp_new, *pgdp_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) pgdp = pgd_offset_k(KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) pgdp_end = pgd_offset_k(KASAN_SHADOW_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) pgdp_new = pgd_offset_pgd(pgdir, KASAN_SHADOW_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) set_pgd(pgdp_new, READ_ONCE(*pgdp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) } while (pgdp++, pgdp_new++, pgdp != pgdp_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static void __init clear_pgds(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * Remove references to kasan page tables from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * swapper_pg_dir. pgd_clear() can't be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * here because it's nop on 2,3-level pagetable setups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) for (; start < end; start += PGDIR_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) set_pgd(pgd_offset_k(start), __pgd(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static void __init kasan_init_shadow(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) u64 kimg_shadow_start, kimg_shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) u64 mod_shadow_start, mod_shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) u64 vmalloc_shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) phys_addr_t pa_start, pa_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) u64 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) kimg_shadow_start = (u64)kasan_mem_to_shadow(KERNEL_START) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(KERNEL_END));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) vmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)VMALLOC_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * We are going to perform proper setup of shadow memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * At first we should unmap early shadow (clear_pgds() call below).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * However, instrumented code couldn't execute without shadow memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * tmp_pg_dir used to keep early shadow mapped until full shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * setup will be finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) dsb(ishst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) cpu_replace_ttbr1(lm_alias(tmp_pg_dir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) early_pfn_to_nid(virt_to_pfn(lm_alias(KERNEL_START))));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) kasan_populate_early_shadow(kasan_mem_to_shadow((void *)PAGE_END),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) (void *)mod_shadow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) BUILD_BUG_ON(VMALLOC_START != MODULES_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) kasan_populate_early_shadow((void *)vmalloc_shadow_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) (void *)KASAN_SHADOW_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) kasan_populate_early_shadow((void *)kimg_shadow_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) (void *)KASAN_SHADOW_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (kimg_shadow_start > mod_shadow_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) kasan_populate_early_shadow((void *)mod_shadow_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) (void *)kimg_shadow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) for_each_mem_range(i, &pa_start, &pa_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) void *start = (void *)__phys_to_virt(pa_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) void *end = (void *)__phys_to_virt(pa_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (start >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) kasan_map_populate((unsigned long)kasan_mem_to_shadow(start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) (unsigned long)kasan_mem_to_shadow(end),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) early_pfn_to_nid(virt_to_pfn(start)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * KAsan may reuse the contents of kasan_early_shadow_pte directly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * so we should make sure that it maps the zero page read-only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) for (i = 0; i < PTRS_PER_PTE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) set_pte(&kasan_early_shadow_pte[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) pfn_pte(sym_to_pfn(kasan_early_shadow_page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) PAGE_KERNEL_RO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static void __init kasan_init_depth(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) init_task.kasan_depth = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) void __init kasan_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) kasan_init_shadow();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) kasan_init_depth();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #if defined(CONFIG_KASAN_GENERIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /* CONFIG_KASAN_SW_TAGS also requires kasan_init_sw_tags(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) pr_info("KernelAddressSanitizer initialized\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags