^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/static_call.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) extern struct static_call_site __start_static_call_sites[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) __stop_static_call_sites[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) extern struct static_call_tramp_key __start_static_call_tramp_key[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) __stop_static_call_tramp_key[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) static bool static_call_initialized;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) /* mutex to protect key modules/sites */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static DEFINE_MUTEX(static_call_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static void static_call_lock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) mutex_lock(&static_call_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static void static_call_unlock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) mutex_unlock(&static_call_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static inline void *static_call_addr(struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) return (void *)((long)site->addr + (long)&site->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static inline unsigned long __static_call_key(const struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return (long)site->key + (long)&site->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static inline struct static_call_key *static_call_key(const struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /* These assume the key is word-aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static inline bool static_call_is_init(struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return __static_call_key(site) & STATIC_CALL_SITE_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static inline bool static_call_is_tail(struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static inline void static_call_set_init(struct static_call_site *site)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) (long)&site->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static int static_call_site_cmp(const void *_a, const void *_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) const struct static_call_site *a = _a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) const struct static_call_site *b = _b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) const struct static_call_key *key_a = static_call_key(a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) const struct static_call_key *key_b = static_call_key(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (key_a < key_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (key_a > key_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static void static_call_site_swap(void *_a, void *_b, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) long delta = (unsigned long)_a - (unsigned long)_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct static_call_site *a = _a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct static_call_site *b = _b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct static_call_site tmp = *a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) a->addr = b->addr - delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) a->key = b->key - delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) b->addr = tmp.addr + delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) b->key = tmp.key + delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static inline void static_call_sort_entries(struct static_call_site *start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct static_call_site *stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) sort(start, stop - start, sizeof(struct static_call_site),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static_call_site_cmp, static_call_site_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static inline bool static_call_key_has_mods(struct static_call_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return !(key->type & 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!static_call_key_has_mods(key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return key->mods;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (static_call_key_has_mods(key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return (struct static_call_site *)(key->type & ~1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) void __static_call_update(struct static_call_key *key, void *tramp, void *func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct static_call_site *site, *stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct static_call_mod *site_mod, first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static_call_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (key->func == func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) key->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) arch_static_call_transform(NULL, tramp, func, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * If uninitialized, we'll not update the callsites, but they still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * point to the trampoline and we just patched that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (WARN_ON_ONCE(!static_call_initialized))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) first = (struct static_call_mod){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .next = static_call_key_next(key),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .mod = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .sites = static_call_key_sites(key),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) for (site_mod = &first; site_mod; site_mod = site_mod->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) bool init = system_state < SYSTEM_RUNNING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct module *mod = site_mod->mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (!site_mod->sites) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * This can happen if the static call key is defined in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * a module which doesn't use it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * It also happens in the has_mods case, where the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * 'first' entry has no sites associated with it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) stop = __stop_static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (mod) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #ifdef CONFIG_MODULES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) stop = mod->static_call_sites +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) mod->num_static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) init = mod->state == MODULE_STATE_COMING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) for (site = site_mod->sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) site < stop && static_call_key(site) == key; site++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) void *site_addr = static_call_addr(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (!init && static_call_is_init(site))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (!kernel_text_address((unsigned long)site_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * This skips patching built-in __exit, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * is part of init_section_contains() but is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * not part of kernel_text_address().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Skipping built-in __exit is fine since it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * will never be executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) WARN_ONCE(!static_call_is_init(site),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) "can't patch static call site at %pS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) site_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) arch_static_call_transform(site_addr, NULL, func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static_call_is_tail(site));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static_call_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) EXPORT_SYMBOL_GPL(__static_call_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static int __static_call_init(struct module *mod,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct static_call_site *start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct static_call_site *stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct static_call_site *site;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct static_call_key *key, *prev_key = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct static_call_mod *site_mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (start == stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static_call_sort_entries(start, stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) for (site = start; site < stop; site++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) void *site_addr = static_call_addr(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) (!mod && init_section_contains(site_addr, 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static_call_set_init(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) key = static_call_key(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (key != prev_key) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) prev_key = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * For vmlinux (!mod) avoid the allocation by storing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * the sites pointer in the key itself. Also see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * __static_call_update()'s @first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * This allows architectures (eg. x86) to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * static_call_init() before memory allocation works.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (!mod) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) key->sites = site;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) key->type |= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) goto do_transform;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (!site_mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * When the key has a direct sites pointer, extract
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * that into an explicit struct static_call_mod, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * can have a list of modules.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (static_call_key_sites(key)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) site_mod->mod = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) site_mod->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) site_mod->sites = static_call_key_sites(key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) key->mods = site_mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (!site_mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) site_mod->mod = mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) site_mod->sites = site;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) site_mod->next = static_call_key_next(key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) key->mods = site_mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) do_transform:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) arch_static_call_transform(site_addr, NULL, key->func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static_call_is_tail(site));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static int addr_conflict(struct static_call_site *site, void *start, void *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned long addr = (unsigned long)static_call_addr(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (addr <= (unsigned long)end &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) addr + CALL_INSN_SIZE > (unsigned long)start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static int __static_call_text_reserved(struct static_call_site *iter_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct static_call_site *iter_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) void *start, void *end, bool init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct static_call_site *iter = iter_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) while (iter < iter_stop) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (init || !static_call_is_init(iter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (addr_conflict(iter, start, end))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) iter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #ifdef CONFIG_MODULES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static int __static_call_mod_text_reserved(void *start, void *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct module *mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) mod = __module_text_address((unsigned long)start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (!try_module_get(mod))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) mod = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) ret = __static_call_text_reserved(mod->static_call_sites,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) mod->static_call_sites + mod->num_static_call_sites,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) start, end, mod->state == MODULE_STATE_COMING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) module_put(mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) static unsigned long tramp_key_lookup(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct static_call_tramp_key *start = __start_static_call_tramp_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct static_call_tramp_key *tramp_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) for (tramp_key = start; tramp_key != stop; tramp_key++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) unsigned long tramp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (tramp == addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return (long)tramp_key->key + (long)&tramp_key->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static int static_call_add_module(struct module *mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct static_call_site *start = mod->static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct static_call_site *stop = start + mod->num_static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct static_call_site *site;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) for (site = start; site != stop; site++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) unsigned long s_key = __static_call_key(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) unsigned long key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * Is the key is exported, 'addr' points to the key, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * means modules are allowed to call static_call_update() on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Otherwise, the key isn't exported, and 'addr' points to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * trampoline so we need to lookup the key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * We go through this dance to prevent crazy modules from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * abusing sensitive static calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (!kernel_text_address(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) key = tramp_key_lookup(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (!key) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static_call_addr(site));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) key |= s_key & STATIC_CALL_SITE_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) site->key = key - (long)&site->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return __static_call_init(mod, start, stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) static void static_call_del_module(struct module *mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct static_call_site *start = mod->static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct static_call_site *stop = mod->static_call_sites +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) mod->num_static_call_sites;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) struct static_call_key *key, *prev_key = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) struct static_call_mod *site_mod, **prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) struct static_call_site *site;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) for (site = start; site < stop; site++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) key = static_call_key(site);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (key == prev_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) prev_key = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) for (prev = &key->mods, site_mod = key->mods;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) site_mod && site_mod->mod != mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) prev = &site_mod->next, site_mod = site_mod->next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (!site_mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) *prev = site_mod->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) kfree(site_mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) static int static_call_module_notify(struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) unsigned long val, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) struct module *mod = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static_call_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) switch (val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) case MODULE_STATE_COMING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) ret = static_call_add_module(mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) WARN(1, "Failed to allocate memory for static calls");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static_call_del_module(mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) case MODULE_STATE_GOING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) static_call_del_module(mod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static_call_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return notifier_from_errno(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static struct notifier_block static_call_module_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .notifier_call = static_call_module_notify,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static inline int __static_call_mod_text_reserved(void *start, void *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #endif /* CONFIG_MODULES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) int static_call_text_reserved(void *start, void *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) bool init = system_state < SYSTEM_RUNNING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) int ret = __static_call_text_reserved(__start_static_call_sites,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) __stop_static_call_sites, start, end, init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return __static_call_mod_text_reserved(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) int __init static_call_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (static_call_initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static_call_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) ret = __static_call_init(NULL, __start_static_call_sites,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) __stop_static_call_sites);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static_call_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) pr_err("Failed to allocate memory for static_call!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static_call_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #ifdef CONFIG_MODULES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) register_module_notifier(&static_call_module_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) early_initcall(static_call_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) #ifdef CONFIG_STATIC_CALL_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static int func_a(int x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return x+1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) static int func_b(int x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return x+2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) DEFINE_STATIC_CALL(sc_selftest, func_a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static struct static_call_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) int (*func)(int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) int expect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) } static_call_data [] __initdata = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) { NULL, 2, 3 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) { func_b, 2, 4 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) { func_a, 2, 3 }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static int __init test_static_call_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct static_call_data *scd = &static_call_data[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (scd->func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static_call_update(sc_selftest, scd->func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) early_initcall(test_static_call_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) #endif /* CONFIG_STATIC_CALL_SELFTEST */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags