^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * This file contains KASAN runtime code that manages shadow memory for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * generic and software tag-based KASAN modes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2014 Samsung Electronics Co., Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Some code borrowed from https://github.com/xairy/kasan-prototype by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Andrey Konovalov <andreyknvl@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kfence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kmemleak.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include "kasan.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) bool __kasan_check_read(const volatile void *p, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) return kasan_check_range((unsigned long)p, size, false, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) EXPORT_SYMBOL(__kasan_check_read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) bool __kasan_check_write(const volatile void *p, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return kasan_check_range((unsigned long)p, size, true, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) EXPORT_SYMBOL(__kasan_check_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #undef memset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) void *memset(void *addr, int c, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) if (!kasan_check_range((unsigned long)addr, len, true, _RET_IP_))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return __memset(addr, c, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #ifdef __HAVE_ARCH_MEMMOVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #undef memmove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) void *memmove(void *dest, const void *src, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) !kasan_check_range((unsigned long)dest, len, true, _RET_IP_))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return __memmove(dest, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #undef memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) void *memcpy(void *dest, const void *src, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) !kasan_check_range((unsigned long)dest, len, true, _RET_IP_))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return __memcpy(dest, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) void kasan_poison(const void *addr, size_t size, u8 value, bool init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) void *shadow_start, *shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Perform shadow offset calculation based on untagged address, as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * some of the callers (e.g. kasan_poison_object_data) pass tagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * addresses to this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) addr = kasan_reset_tag(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Skip KFENCE memory if called explicitly outside of sl*b. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (is_kfence_address(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (WARN_ON(size & KASAN_GRANULE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) shadow_start = kasan_mem_to_shadow(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) shadow_end = kasan_mem_to_shadow(addr + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __memset(shadow_start, value, shadow_end - shadow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) EXPORT_SYMBOL(kasan_poison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #ifdef CONFIG_KASAN_GENERIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) void kasan_poison_last_granule(const void *addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (size & KASAN_GRANULE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) u8 *shadow = (u8 *)kasan_mem_to_shadow(addr + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *shadow = size & KASAN_GRANULE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) void kasan_unpoison(const void *addr, size_t size, bool init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) u8 tag = get_tag(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * Perform shadow offset calculation based on untagged address, as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * some of the callers (e.g. kasan_unpoison_object_data) pass tagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * addresses to this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) addr = kasan_reset_tag(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Skip KFENCE memory if called explicitly outside of sl*b. Also note
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * that calls to ksize(), where size is not a multiple of machine-word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * size, would otherwise poison the invalid portion of the word.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (is_kfence_address(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /* Unpoison all granules that cover the object. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) kasan_poison(addr, round_up(size, KASAN_GRANULE_SIZE), tag, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* Partially poison the last granule for the generic mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (IS_ENABLED(CONFIG_KASAN_GENERIC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) kasan_poison_last_granule(addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #ifdef CONFIG_MEMORY_HOTPLUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static bool shadow_mapped(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) pgd_t *pgd = pgd_offset_k(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (pgd_none(*pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (p4d_none(*p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (pud_none(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * We can't use pud_large() or pud_huge(), the first one is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * arch-specific, the last one depends on HUGETLB_PAGE. So let's abuse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * pud_bad(), if pud is bad then it's bad because it's huge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (pud_bad(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (pmd_none(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (pmd_bad(*pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) pte = pte_offset_kernel(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return !pte_none(*pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static int __meminit kasan_mem_notifier(struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) unsigned long action, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) struct memory_notify *mem_data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) unsigned long nr_shadow_pages, start_kaddr, shadow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned long shadow_end, shadow_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) nr_shadow_pages = mem_data->nr_pages >> KASAN_SHADOW_SCALE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) start_kaddr = (unsigned long)pfn_to_kaddr(mem_data->start_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) shadow_start = (unsigned long)kasan_mem_to_shadow((void *)start_kaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) shadow_size = nr_shadow_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) shadow_end = shadow_start + shadow_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (WARN_ON(mem_data->nr_pages % KASAN_GRANULE_SIZE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) WARN_ON(start_kaddr % KASAN_MEMORY_PER_SHADOW_PAGE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return NOTIFY_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) switch (action) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) case MEM_GOING_ONLINE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * If shadow is mapped already than it must have been mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * during the boot. This could happen if we onlining previously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * offlined memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (shadow_mapped(shadow_start))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) shadow_end, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) PAGE_KERNEL, VM_NO_GUARD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) pfn_to_nid(mem_data->start_pfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) __builtin_return_address(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return NOTIFY_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) kmemleak_ignore(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) case MEM_CANCEL_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) case MEM_OFFLINE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct vm_struct *vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * shadow_start was either mapped during boot by kasan_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * or during memory online by __vmalloc_node_range().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * In the latter case we can use vfree() to free shadow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * Non-NULL result of the find_vm_area() will tell us if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * that was the second case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * Currently it's not possible to free shadow mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * during boot by kasan_init(). It's because the code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * to do that hasn't been written yet. So we'll just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * leak the memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) vm = find_vm_area((void *)shadow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (vm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) vfree((void *)shadow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static int __init kasan_memhotplug_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) hotplug_memory_notifier(kasan_mem_notifier, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) core_initcall(kasan_memhotplug_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #ifdef CONFIG_KASAN_VMALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned long page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) pte_t pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (likely(!pte_none(*ptep)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) page = __get_free_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) spin_lock(&init_mm.page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (likely(pte_none(*ptep))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) set_pte_at(&init_mm, addr, ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) page = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) spin_unlock(&init_mm.page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) int kasan_populate_vmalloc(unsigned long addr, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) unsigned long shadow_start, shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (!is_vmalloc_or_module_addr((void *)addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) shadow_start = (unsigned long)kasan_mem_to_shadow((void *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) shadow_end = (unsigned long)kasan_mem_to_shadow((void *)addr + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) shadow_end = ALIGN(shadow_end, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) ret = apply_to_page_range(&init_mm, shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) shadow_end - shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) kasan_populate_vmalloc_pte, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) flush_cache_vmap(shadow_start, shadow_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * We need to be careful about inter-cpu effects here. Consider:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * CPU#0 CPU#1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * WRITE_ONCE(p, vmalloc(100)); while (x = READ_ONCE(p)) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * p[99] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * With compiler instrumentation, that ends up looking like this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * CPU#0 CPU#1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * // vmalloc() allocates memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * // let a = area->addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * // we reach kasan_populate_vmalloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * // and call kasan_unpoison:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * STORE shadow(a), unpoison_val
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * STORE shadow(a+99), unpoison_val x = LOAD p
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * // rest of vmalloc process <data dependency>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * STORE p, a LOAD shadow(x+99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * If there is no barrier between the end of unpoisioning the shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * and the store of the result to p, the stores could be committed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * in a different order by CPU#0, and CPU#1 could erroneously observe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * poison in the shadow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * We need some sort of barrier between the stores.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * In the vmalloc() case, this is provided by a smp_wmb() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * clear_vm_uninitialized_flag(). In the per-cpu allocator and in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * get_vm_area() and friends, the caller gets shadow allocated but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * doesn't have any pages mapped into the virtual address space that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * has been reserved. Mapping those pages in will involve taking and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * releasing a page-table lock, which will provide the barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * Poison the shadow for a vmalloc region. Called as part of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * freeing process at the time the region is freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) void kasan_poison_vmalloc(const void *start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!is_vmalloc_or_module_addr(start))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) size = round_up(size, KASAN_GRANULE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) kasan_poison(start, size, KASAN_VMALLOC_INVALID, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) void kasan_unpoison_vmalloc(const void *start, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (!is_vmalloc_or_module_addr(start))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) kasan_unpoison(start, size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) unsigned long page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) spin_lock(&init_mm.page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (likely(!pte_none(*ptep))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) pte_clear(&init_mm, addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) spin_unlock(&init_mm.page_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Release the backing for the vmalloc region [start, end), which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * lies within the free region [free_region_start, free_region_end).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * This can be run lazily, long after the region was freed. It runs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * infrastructure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * How does this work?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * -------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * We have a region that is page aligned, labelled as A.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * That might not map onto the shadow in a way that is page-aligned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * start end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * v v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * |????????|????????|AAAAAAAA|AA....AA|AAAAAAAA|????????| < vmalloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * -------- -------- -------- -------- --------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * | | | | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * | | | /-------/ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * \-------\|/------/ |/---------------/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * ||| ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * |??AAAAAA|AAAAAAAA|AA??????| < shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * (1) (2) (3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * First we align the start upwards and the end downwards, so that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * shadow of the region aligns with shadow page boundaries. In the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * example, this gives us the shadow page (2). This is the shadow entirely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * covered by this allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * Then we have the tricky bits. We want to know if we can free the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * partially covered shadow pages - (1) and (3) in the example. For this,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * we are given the start and end of the free region that contains this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * allocation. Extending our previous example, we could have:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * free_region_start free_region_end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * | start end |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * v v v v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * |FFFFFFFF|FFFFFFFF|AAAAAAAA|AA....AA|AAAAAAAA|FFFFFFFF| < vmalloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * -------- -------- -------- -------- --------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * | | | | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * | | | /-------/ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * \-------\|/------/ |/---------------/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * ||| ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * |FFAAAAAA|AAAAAAAA|AAF?????| < shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * (1) (2) (3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * Once again, we align the start of the free region up, and the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * the free region down so that the shadow is page aligned. So we can free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * page (1) - we know no allocation currently uses anything in that page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * because all of it is in the vmalloc free region. But we cannot free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * page (3), because we can't be sure that the rest of it is unused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * We only consider pages that contain part of the original region for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * freeing: we don't try to free other pages from the free region or we'd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * end up trying to free huge chunks of virtual address space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * Concurrency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * -----------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * How do we know that we're not freeing a page that is simultaneously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * being used for a fresh allocation in kasan_populate_vmalloc(_pte)?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * at the same time. While we run under free_vmap_area_lock, the population
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * code does not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * free_vmap_area_lock instead operates to ensure that the larger range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * [free_region_start, free_region_end) is safe: because __alloc_vmap_area and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * the per-cpu region-finding algorithm both run under free_vmap_area_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * no space identified as free will become used while we are running. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * means that so long as we are careful with alignment and only free shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * pages entirely covered by the free region, we will not run in to any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * trouble - any simultaneous allocations will be for disjoint regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) void kasan_release_vmalloc(unsigned long start, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) unsigned long free_region_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) unsigned long free_region_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) void *shadow_start, *shadow_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) unsigned long region_start, region_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) region_start = ALIGN(start, KASAN_MEMORY_PER_SHADOW_PAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) region_end = ALIGN_DOWN(end, KASAN_MEMORY_PER_SHADOW_PAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) free_region_start = ALIGN(free_region_start, KASAN_MEMORY_PER_SHADOW_PAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (start != region_start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) free_region_start < region_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) region_start -= KASAN_MEMORY_PER_SHADOW_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) free_region_end = ALIGN_DOWN(free_region_end, KASAN_MEMORY_PER_SHADOW_PAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (end != region_end &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) free_region_end > region_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) region_end += KASAN_MEMORY_PER_SHADOW_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) shadow_start = kasan_mem_to_shadow((void *)region_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) shadow_end = kasan_mem_to_shadow((void *)region_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (shadow_end > shadow_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) size = shadow_end - shadow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) apply_to_existing_page_range(&init_mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) (unsigned long)shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) size, kasan_depopulate_vmalloc_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) flush_tlb_kernel_range((unsigned long)shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) (unsigned long)shadow_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) #else /* CONFIG_KASAN_VMALLOC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) int kasan_module_alloc(void *addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) size_t scaled_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) size_t shadow_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) unsigned long shadow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) scaled_size = (size + KASAN_GRANULE_SIZE - 1) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) KASAN_SHADOW_SCALE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) shadow_size = round_up(scaled_size, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) shadow_start + shadow_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) __builtin_return_address(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) __memset(ret, KASAN_SHADOW_INIT, shadow_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) find_vm_area(addr)->flags |= VM_KASAN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) kmemleak_ignore(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) void kasan_free_shadow(const struct vm_struct *vm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if (vm->flags & VM_KASAN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) vfree(kasan_mem_to_shadow(vm->addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags