Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Clang Control Flow Integrity (CFI) error and slowpath handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2019 Google LLC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/printk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) /* Compiler-defined handler names */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #ifdef CONFIG_CFI_PERMISSIVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define cfi_failure_handler	__ubsan_handle_cfi_check_fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define cfi_slowpath_handler	__cfi_slowpath_diag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #else /* enforcing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define cfi_failure_handler	__ubsan_handle_cfi_check_fail_abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define cfi_slowpath_handler	__cfi_slowpath
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #endif /* CONFIG_CFI_PERMISSIVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) static inline void handle_cfi_failure(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 		WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 		panic("CFI failure (target: %pS)\n", ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #ifdef CONFIG_MODULES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #ifdef CONFIG_CFI_CLANG_SHADOW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * Index type. A 16-bit index can address at most (2^16)-2 pages (taking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * into account SHADOW_INVALID), i.e. ~256M with 4k pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) typedef u16 shadow_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #define SHADOW_INVALID		((shadow_t)~0UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) struct cfi_shadow {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	/* Page index for the beginning of the shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	unsigned long base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	/* An array of __cfi_check locations (as indices to the shadow) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	shadow_t shadow[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53)  * The shadow covers ~128M from the beginning of the module region. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * the region is larger, we fall back to __module_address for the rest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) #define __SHADOW_RANGE		(_UL(SZ_128M) >> PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) /* The in-memory size of struct cfi_shadow, always at least one page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) #define __SHADOW_PAGES		((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) #define SHADOW_PAGES		max(1UL, __SHADOW_PAGES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) #define SHADOW_SIZE		(SHADOW_PAGES << PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) /* The actual size of the shadow array, minus metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define SHADOW_ARR_SIZE		(SHADOW_SIZE - offsetof(struct cfi_shadow, shadow))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) #define SHADOW_ARR_SLOTS	(SHADOW_ARR_SIZE / sizeof(shadow_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) static DEFINE_MUTEX(shadow_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) static struct cfi_shadow __rcu *cfi_shadow __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) /* Returns the index in the shadow for the given address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	unsigned long index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	unsigned long page = ptr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	if (unlikely(page < s->base))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 		return -1; /* Outside of module area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	index = page - s->base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	if (index >= SHADOW_ARR_SLOTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		return -1; /* Cannot be addressed with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	return (int)index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) /* Returns the page address for an index in the shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	return (s->base + index) << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) /* Returns the __cfi_check function address for the given shadow location */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	if (unlikely(s->shadow[index] == SHADOW_INVALID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	/* __cfi_check is always page aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	return (s->base + s->shadow[index]) << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		struct cfi_shadow *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	int i, index, check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	/* Mark everything invalid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	memset(next->shadow, 0xFF, SHADOW_ARR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		return; /* No previous shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/* If the base address didn't change, an update is not needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	if (prev->base == next->base) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	/* Convert the previous shadow to the new address range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	for (i = 0; i < SHADOW_ARR_SLOTS; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		if (prev->shadow[i] == SHADOW_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		if (index < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		check = ptr_to_shadow(next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 				shadow_to_check_fn(prev, prev->shadow[i]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		if (check < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		next->shadow[index] = (shadow_t)check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 			unsigned long min_addr, unsigned long max_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	int check_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	unsigned long check = (unsigned long)mod->cfi_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	unsigned long ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	if (unlikely(!PAGE_ALIGNED(check))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 		pr_warn("cfi: not using shadow for module %s\n", mod->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	check_index = ptr_to_shadow(s, check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	if (check_index < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		return; /* Module not addressable with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	/* For each page, store the check function index in the shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		int index = ptr_to_shadow(s, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		if (index >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 			/* Each page must only contain one module */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 			WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 			s->shadow[index] = (shadow_t)check_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		unsigned long min_addr, unsigned long max_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	unsigned long ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		int index = ptr_to_shadow(s, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 		if (index >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 			s->shadow[index] = SHADOW_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 			unsigned long min_addr, unsigned long max_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static void update_shadow(struct module *mod, unsigned long base_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		update_shadow_fn fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	struct cfi_shadow *prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	struct cfi_shadow *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	unsigned long min_addr, max_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	next = (struct cfi_shadow *)vmalloc(SHADOW_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	WARN_ON(!next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	mutex_lock(&shadow_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	prev = rcu_dereference_protected(cfi_shadow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 					 mutex_is_locked(&shadow_update_lock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	if (next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 		next->base = base_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 		prepare_next_shadow(prev, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		min_addr = (unsigned long)mod->core_layout.base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 		max_addr = min_addr + mod->core_layout.text_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		set_memory_ro((unsigned long)next, SHADOW_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	rcu_assign_pointer(cfi_shadow, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	mutex_unlock(&shadow_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	synchronize_rcu_expedited();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	if (prev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		set_memory_rw((unsigned long)prev, SHADOW_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		vfree(prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) void cfi_module_add(struct module *mod, unsigned long base_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	update_shadow(mod, base_addr, add_module_to_shadow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) void cfi_module_remove(struct module *mod, unsigned long base_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	update_shadow(mod, base_addr, remove_module_from_shadow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	if (unlikely(!s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		return NULL; /* No shadow available */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	index = ptr_to_shadow(s, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	if (index < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		return NULL; /* Cannot be addressed with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	return (cfi_check_fn)shadow_to_check_fn(s, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static inline cfi_check_fn __find_shadow_check_fn(unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	cfi_check_fn fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	rcu_read_lock_sched_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	rcu_read_unlock_sched_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	return fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) #else /* !CONFIG_CFI_CLANG_SHADOW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static inline cfi_check_fn __find_shadow_check_fn(unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #endif /* CONFIG_CFI_CLANG_SHADOW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static inline cfi_check_fn __find_module_check_fn(unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	cfi_check_fn fn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	struct module *mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	rcu_read_lock_sched_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	mod = __module_address(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	if (mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		fn = mod->cfi_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	rcu_read_unlock_sched_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	return fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static inline cfi_check_fn find_check_fn(unsigned long ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	bool rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	cfi_check_fn fn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	 * Indirect call checks can happen when RCU is not watching. Both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	 * the shadow and __module_address use RCU, so we need to wake it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	 * up before proceeding. Use rcu_nmi_enter/exit() as these calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	 * can happen anywhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	rcu = rcu_is_watching();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	if (!rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 		rcu_nmi_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		fn = __find_shadow_check_fn(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		if (fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 			goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	if (is_kernel_text(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 		fn = __cfi_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	fn = __find_module_check_fn(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	if (!rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 		rcu_nmi_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	return fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	cfi_check_fn fn = find_check_fn((unsigned long)ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	if (!IS_ENABLED(CONFIG_CFI_PERMISSIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		diag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	if (likely(fn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 		fn(id, ptr, diag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	else /* Don't allow unchecked modules */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 		handle_cfi_failure(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #else /* !CONFIG_MODULES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	handle_cfi_failure(ptr); /* No modules */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #endif /* CONFIG_MODULES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) EXPORT_SYMBOL(cfi_slowpath_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) void cfi_failure_handler(void *data, void *ptr, void *vtable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	handle_cfi_failure(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) EXPORT_SYMBOL(cfi_failure_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) void __cfi_check_fail(void *data, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	handle_cfi_failure(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }