^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2013 Linaro Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2017 Andes Technology Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/patch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #ifdef CONFIG_DYNAMIC_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) int ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) mutex_lock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) int ftrace_arch_code_modify_post_process(void) __releases(&text_mutex)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) mutex_unlock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static int ftrace_check_current_call(unsigned long hook_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) unsigned int *expected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) unsigned int replaced[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) unsigned int nops[2] = {NOP4, NOP4};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* we expect nops at the hook position */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) if (!expected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) expected = nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Read the text we want to modify;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * return must be -EFAULT on read error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (copy_from_kernel_nofault(replaced, (void *)hook_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) MCOUNT_INSN_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * Make sure it is what we expect it to be;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * return must be -EINVAL on failed comparison
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) if (memcmp(expected, replaced, sizeof(replaced))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) (void *)hook_pos, expected[0], expected[1], replaced[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) replaced[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) bool enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) unsigned int call[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned int nops[2] = {NOP4, NOP4};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) make_call(hook_pos, target, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (patch_text_nosync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ((void *)hook_pos, enable ? call : nops, MCOUNT_INSN_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) int ret = ftrace_check_current_call(rec->ip, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return __ftrace_modify_call(rec->ip, addr, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) unsigned int call[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) make_call(rec->ip, addr, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) ret = ftrace_check_current_call(rec->ip, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return __ftrace_modify_call(rec->ip, addr, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * This is called early on, and isn't wrapped by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * text_mutex, which triggers a lockdep failure. SMP isn't running so we could
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * just directly poke the text, but it's simpler to just take the lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * ourselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) int out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) ftrace_arch_code_modify_prepare();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) out = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ftrace_arch_code_modify_post_process();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) int ftrace_update_ftrace_func(ftrace_func_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int ret = __ftrace_modify_call((unsigned long)&ftrace_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) (unsigned long)func, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) (unsigned long)func, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int __init ftrace_dyn_arch_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) unsigned int call[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) make_call(rec->ip, old_addr, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) ret = ftrace_check_current_call(rec->ip, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) return __ftrace_modify_call(rec->ip, addr, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * Most of this function is copied from arm64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned long frame_pointer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) unsigned long return_hooker = (unsigned long)&return_to_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned long old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (unlikely(atomic_read(¤t->tracing_graph_pause)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * We don't suffer access faults, so no extra fault-recovery assembly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * is needed here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) old = *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (!function_graph_enter(old, self_addr, frame_pointer, parent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) *parent = return_hooker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #ifdef CONFIG_DYNAMIC_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) extern void ftrace_graph_call(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int ftrace_enable_ftrace_graph_caller(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int call[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) static int init_graph = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) make_call(&ftrace_graph_call, &ftrace_stub, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * When enabling graph tracer for the first time, ftrace_graph_call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * should contains a call to ftrace_stub. Once it has been disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * the 8-bytes at the position becomes NOPs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (init_graph) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) init_graph = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return __ftrace_modify_call((unsigned long)&ftrace_graph_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) (unsigned long)&prepare_ftrace_return, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) int ftrace_disable_ftrace_graph_caller(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) unsigned int call[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) make_call(&ftrace_graph_call, &prepare_ftrace_return, call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * This is to make sure that ftrace_enable_ftrace_graph_caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * did the right thing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return __ftrace_modify_call((unsigned long)&ftrace_graph_call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) (unsigned long)&prepare_ftrace_return, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #endif /* CONFIG_DYNAMIC_FTRACE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */