^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * uprobes-based tracing events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) IBM Corporation, 2010-2012
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define pr_fmt(fmt) "trace_uprobe: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/security.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/ctype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/uprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/namei.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "trace_dynevent.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "trace_probe.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "trace_probe_tmpl.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define UPROBE_EVENT_SYSTEM "uprobes"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct uprobe_trace_entry_head {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct trace_entry ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) unsigned long vaddr[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define SIZEOF_TRACE_ENTRY(is_return) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) (sizeof(struct uprobe_trace_entry_head) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) sizeof(unsigned long) * (is_return ? 2 : 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define DATAOF_TRACE_ENTRY(entry, is_return) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static int trace_uprobe_create(int argc, const char **argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static int trace_uprobe_release(struct dyn_event *ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static bool trace_uprobe_is_busy(struct dyn_event *ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static bool trace_uprobe_match(const char *system, const char *event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) int argc, const char **argv, struct dyn_event *ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static struct dyn_event_operations trace_uprobe_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) .create = trace_uprobe_create,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) .show = trace_uprobe_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) .is_busy = trace_uprobe_is_busy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) .free = trace_uprobe_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) .match = trace_uprobe_match,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * uprobe event core functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct trace_uprobe {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct dyn_event devent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct uprobe_consumer consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct path path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) char *filename;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) unsigned long ref_ctr_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned long nhit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct trace_probe tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static bool is_trace_uprobe(struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return ev->ops == &trace_uprobe_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) return container_of(ev, struct trace_uprobe, devent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * for_each_trace_uprobe - iterate over the trace_uprobe list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * @pos: the struct trace_uprobe * for each entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * @dpos: the struct dyn_event * to use as a loop cursor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define for_each_trace_uprobe(pos, dpos) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) for_each_dyn_event(dpos) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define SIZEOF_TRACE_UPROBE(n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) (offsetof(struct trace_uprobe, tp.args) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) (sizeof(struct probe_arg) * (n)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static int register_uprobe_event(struct trace_uprobe *tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static int unregister_uprobe_event(struct trace_uprobe *tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct uprobe_dispatch_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) unsigned long bp_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static int uretprobe_dispatcher(struct uprobe_consumer *con,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long func, struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #ifdef CONFIG_STACK_GROWSUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return addr - (n * sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return addr + (n * sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) unsigned long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) unsigned long addr = user_stack_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) addr = adjust_stack_addr(addr, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * Uprobes-specific fetch functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) probe_mem_read(void *dest, void *src, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) void __user *vaddr = (void __force __user *)src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) probe_mem_read_user(void *dest, void *src, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return probe_mem_read(dest, src, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * length and relative data location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) fetch_store_string(unsigned long addr, void *dest, void *base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) u32 loc = *(u32 *)dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) int maxlen = get_loc_len(loc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) u8 *dst = get_loc_data(dest, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) void __user *src = (void __force __user *) addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (unlikely(!maxlen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (addr == FETCH_TOKEN_COMM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ret = strlcpy(dst, current->comm, maxlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) ret = strncpy_from_user(dst, src, maxlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (ret >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (ret == maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) dst[ret - 1] = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * Include the terminating null byte. In this case it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * was copied by strncpy_from_user but not accounted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * for in ret.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) ret++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) fetch_store_string_user(unsigned long addr, void *dest, void *base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return fetch_store_string(addr, dest, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* Return the length of string -- including null terminal byte */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) fetch_store_strlen(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) void __user *vaddr = (void __force __user *) addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (addr == FETCH_TOKEN_COMM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) len = strlen(current->comm) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) len = strnlen_user(vaddr, MAX_STRING_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return (len > MAX_STRING_SIZE) ? 0 : len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static nokprobe_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) fetch_store_strlen_user(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return fetch_store_strlen(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static unsigned long translate_user_vaddr(unsigned long file_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) unsigned long base_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct uprobe_dispatch_data *udd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) udd = (void *) current->utask->vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) base_addr = udd->bp_addr - udd->tu->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return base_addr + file_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) /* Note that we don't verify it, since the code does not come from user space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) void *base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* 1st stage: get value from context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) switch (code->op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) case FETCH_OP_REG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) val = regs_get_register(regs, code->param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) case FETCH_OP_STACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) val = get_user_stack_nth(regs, code->param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) case FETCH_OP_STACKP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) val = user_stack_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) case FETCH_OP_RETVAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) val = regs_return_value(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) case FETCH_OP_IMM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) val = code->immediate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) case FETCH_OP_COMM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) val = FETCH_TOKEN_COMM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) case FETCH_OP_DATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) val = (unsigned long)code->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) case FETCH_OP_FOFFS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) val = translate_user_vaddr(code->immediate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) return -EILSEQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) code++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return process_fetch_insn_bottom(code, val, dest, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) NOKPROBE_SYMBOL(process_fetch_insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) rwlock_init(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) filter->nr_systemwide = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) INIT_LIST_HEAD(&filter->perf_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return !filter->nr_systemwide && list_empty(&filter->perf_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static inline bool is_ret_probe(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return tu->consumer.ret_handler != NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) static bool trace_uprobe_is_busy(struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct trace_uprobe *tu = to_trace_uprobe(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return trace_probe_is_enabled(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) int argc, const char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) char buf[MAX_ARGSTR_LEN + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (!argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) len = strlen(tu->filename);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (tu->ref_ctr_offset == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) snprintf(buf, sizeof(buf), "0x%0*lx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) (int)(sizeof(void *) * 2), tu->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) (int)(sizeof(void *) * 2), tu->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) tu->ref_ctr_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (strcmp(buf, &argv[0][len + 1]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) argc--; argv++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return trace_probe_match_command_args(&tu->tp, argc, argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static bool trace_uprobe_match(const char *system, const char *event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) int argc, const char **argv, struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct trace_uprobe *tu = to_trace_uprobe(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return strcmp(trace_probe_name(&tu->tp), event) == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) trace_uprobe_match_command_head(tu, argc, argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static nokprobe_inline struct trace_uprobe *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) trace_uprobe_primary_from_call(struct trace_event_call *call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct trace_probe *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) tp = trace_probe_primary_from_call(call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (WARN_ON_ONCE(!tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return container_of(tp, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * Allocate new trace_uprobe and initialize it (including uprobes).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static struct trace_uprobe *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (!tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ret = trace_probe_init(&tu->tp, event, group, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) dyn_event_init(&tu->devent, &trace_uprobe_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) tu->consumer.handler = uprobe_dispatcher;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (is_ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) tu->consumer.ret_handler = uretprobe_dispatcher;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) init_trace_uprobe_filter(tu->tp.event->filter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) kfree(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static void free_trace_uprobe(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (!tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) path_put(&tu->path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) trace_probe_cleanup(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) kfree(tu->filename);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) kfree(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static struct trace_uprobe *find_probe_event(const char *event, const char *group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct dyn_event *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) for_each_trace_uprobe(tu, pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) strcmp(trace_probe_group_name(&tu->tp), group) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /* Unregister a trace_uprobe and probe_event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static int unregister_trace_uprobe(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (trace_probe_has_sibling(&tu->tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) goto unreg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) ret = unregister_uprobe_event(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) unreg:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) dyn_event_remove(&tu->devent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) trace_probe_unlink(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) free_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct trace_uprobe *comp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct trace_probe_event *tpe = orig->tp.event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) struct trace_probe *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) struct inode *comp_inode = d_real_inode(comp->path.dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) list_for_each_entry(pos, &tpe->probes, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) orig = container_of(pos, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (comp_inode != d_real_inode(orig->path.dentry) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) comp->offset != orig->offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * trace_probe_compare_arg_type() ensured that nr_args and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * each argument name and type are same. Let's compare comm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) for (i = 0; i < orig->tp.nr_args; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (strcmp(orig->tp.args[i].comm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) comp->tp.args[i].comm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (i == orig->tp.nr_args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* Note that argument starts index = 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) trace_probe_log_set_index(ret + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) trace_probe_log_err(0, DIFF_ARG_TYPE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (trace_uprobe_has_same_uprobe(to, tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) trace_probe_log_set_index(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) trace_probe_log_err(0, SAME_PROBE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /* Append to existing event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) ret = trace_probe_append(&tu->tp, &to->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) dyn_event_add(&tu->devent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * Uprobe with multiple reference counter is not allowed. i.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * If inode and offset matches, reference counter offset *must*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * match as well. Though, there is one exception: If user is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * replacing old trace_uprobe with new one(same group/event),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * then we allow same uprobe with new reference counter as far
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * as the new one does not conflict with any other existing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static int validate_ref_ctr_offset(struct trace_uprobe *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct dyn_event *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct trace_uprobe *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct inode *new_inode = d_real_inode(new->path.dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) for_each_trace_uprobe(tmp, pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (new_inode == d_real_inode(tmp->path.dentry) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) new->offset == tmp->offset &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) new->ref_ctr_offset != tmp->ref_ctr_offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) pr_warn("Reference counter offset mismatch.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /* Register a trace_uprobe and probe_event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) static int register_trace_uprobe(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct trace_uprobe *old_tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) mutex_lock(&event_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) ret = validate_ref_ctr_offset(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /* register as an event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) old_tu = find_probe_event(trace_probe_name(&tu->tp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) trace_probe_group_name(&tu->tp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (old_tu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) trace_probe_log_set_index(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) trace_probe_log_err(0, DIFF_PROBE_TYPE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) ret = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) ret = append_trace_uprobe(tu, old_tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ret = register_uprobe_event(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (ret == -EEXIST) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) trace_probe_log_set_index(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) trace_probe_log_err(0, EVENT_EXIST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) pr_warn("Failed to register probe event(%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) dyn_event_add(&tu->devent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) mutex_unlock(&event_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * Argument syntax:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * - Add uprobe: p|r[:[GRP/]EVENT] PATH:OFFSET[%return][(REF)] [FETCHARGS]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) static int trace_uprobe_create(int argc, const char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) char *arg, *filename, *rctr, *rctr_end, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) char buf[MAX_EVENT_NAME_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct path path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) unsigned long offset, ref_ctr_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) bool is_return = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ref_ctr_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) switch (argv[0][0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) case 'r':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) is_return = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) case 'p':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return -ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (argc < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) return -ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (argv[0][1] == ':')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) event = &argv[0][2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!strchr(argv[1], '/'))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return -ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) filename = kstrdup(argv[1], GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (!filename)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /* Find the last occurrence, in case the path contains ':' too. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) arg = strrchr(filename, ':');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (!arg || !isdigit(arg[1])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) kfree(filename);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return -ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) trace_probe_log_init("trace_uprobe", argc, argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) trace_probe_log_set_index(1); /* filename is the 2nd argument */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) *arg++ = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) ret = kern_path(filename, LOOKUP_FOLLOW, &path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) trace_probe_log_err(0, FILE_NOT_FOUND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) kfree(filename);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) trace_probe_log_clear();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (!d_is_reg(path.dentry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) trace_probe_log_err(0, NO_REGULAR_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) /* Parse reference counter offset if specified. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) rctr = strchr(arg, '(');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (rctr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) rctr_end = strchr(rctr, ')');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (!rctr_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) rctr_end = rctr + strlen(rctr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) trace_probe_log_err(rctr_end - filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) REFCNT_OPEN_BRACE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) } else if (rctr_end[1] != '\0') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) trace_probe_log_err(rctr_end + 1 - filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) BAD_REFCNT_SUFFIX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) *rctr++ = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) *rctr_end = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ret = kstrtoul(rctr, 0, &ref_ctr_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) trace_probe_log_err(rctr - filename, BAD_REFCNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /* Check if there is %return suffix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) tmp = strchr(arg, '%');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (!strcmp(tmp, "%return")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) *tmp = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) is_return = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) /* Parse uprobe offset. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ret = kstrtoul(arg, 0, &offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /* setup a probe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) trace_probe_log_set_index(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) ret = traceprobe_parse_event_name(&event, &group, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) event - argv[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) char *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) char *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) tail = kstrdup(kbasename(filename), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (!tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) ptr = strpbrk(tail, ".-_");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) *ptr = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) event = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) kfree(tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) argc -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) argv += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) tu = alloc_trace_uprobe(group, event, argc, is_return);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (IS_ERR(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ret = PTR_ERR(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /* This must return -ENOMEM otherwise there is a bug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) WARN_ON_ONCE(ret != -ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) goto fail_address_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) tu->offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) tu->ref_ctr_offset = ref_ctr_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) tu->path = path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) tu->filename = filename;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /* parse arguments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) tmp = kstrdup(argv[i], GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if (!tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) trace_probe_log_set_index(i + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) ret = traceprobe_parse_probe_arg(&tu->tp, i, tmp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) is_return ? TPARG_FL_RETURN : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) kfree(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) ret = traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) ret = register_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) free_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) trace_probe_log_clear();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) fail_address_parse:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) trace_probe_log_clear();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) path_put(&path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) kfree(filename);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) static int create_or_delete_trace_uprobe(int argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (argv[0][0] == '-')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return dyn_event_release(argc, argv, &trace_uprobe_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) ret = trace_uprobe_create(argc, (const char **)argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return ret == -ECANCELED ? -EINVAL : ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) static int trace_uprobe_release(struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) struct trace_uprobe *tu = to_trace_uprobe(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return unregister_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /* Probes listing interfaces */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) struct trace_uprobe *tu = to_trace_uprobe(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) char c = is_ret_probe(tu) ? 'r' : 'p';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) trace_probe_name(&tu->tp), tu->filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) (int)(sizeof(void *) * 2), tu->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (tu->ref_ctr_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) for (i = 0; i < tu->tp.nr_args; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static int probes_seq_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) struct dyn_event *ev = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) if (!is_trace_uprobe(ev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) return trace_uprobe_show(m, ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) static const struct seq_operations probes_seq_op = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) .start = dyn_event_seq_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) .next = dyn_event_seq_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) .stop = dyn_event_seq_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) .show = probes_seq_show
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) static int probes_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) ret = security_locked_down(LOCKDOWN_TRACEFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) ret = dyn_events_release_all(&trace_uprobe_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) return seq_open(file, &probes_seq_op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) static ssize_t probes_write(struct file *file, const char __user *buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) size_t count, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return trace_parse_run_command(file, buffer, count, ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) create_or_delete_trace_uprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) static const struct file_operations uprobe_events_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) .open = probes_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) .read = seq_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) .llseek = seq_lseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) .release = seq_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) .write = probes_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /* Probes profiling interfaces */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) static int probes_profile_seq_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct dyn_event *ev = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (!is_trace_uprobe(ev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) tu = to_trace_uprobe(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) seq_printf(m, " %s %-44s %15lu\n", tu->filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) trace_probe_name(&tu->tp), tu->nhit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) static const struct seq_operations profile_seq_op = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) .start = dyn_event_seq_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) .next = dyn_event_seq_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) .stop = dyn_event_seq_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) .show = probes_profile_seq_show
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static int profile_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) ret = security_locked_down(LOCKDOWN_TRACEFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) return seq_open(file, &profile_seq_op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) static const struct file_operations uprobe_profile_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) .open = profile_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) .read = seq_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) .llseek = seq_lseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) .release = seq_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct uprobe_cpu_buffer {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) struct mutex mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) void *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) static int uprobe_buffer_refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) static int uprobe_buffer_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) int cpu, err_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) if (uprobe_cpu_buffer == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) struct page *p = alloc_pages_node(cpu_to_node(cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) GFP_KERNEL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (p == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) err_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (cpu == err_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) free_percpu(uprobe_cpu_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) static int uprobe_buffer_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) BUG_ON(!mutex_is_locked(&event_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (uprobe_buffer_refcnt++ == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) ret = uprobe_buffer_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) uprobe_buffer_refcnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) static void uprobe_buffer_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) BUG_ON(!mutex_is_locked(&event_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (--uprobe_buffer_refcnt == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) cpu)->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) free_percpu(uprobe_cpu_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) uprobe_cpu_buffer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) struct uprobe_cpu_buffer *ucb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * Use per-cpu buffers for fastest access, but we might migrate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * so the mutex makes sure we have sole access to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) mutex_lock(&ucb->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return ucb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) mutex_unlock(&ucb->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) static void __uprobe_trace_func(struct trace_uprobe *tu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) unsigned long func, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) struct uprobe_cpu_buffer *ucb, int dsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) struct trace_event_file *trace_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) struct uprobe_trace_entry_head *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) struct trace_buffer *buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) struct ring_buffer_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) int size, esize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) struct trace_event_call *call = trace_probe_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) WARN_ON(call != trace_file->event_call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) if (trace_trigger_soft_disabled(trace_file))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) size = esize + tu->tp.size + dsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) event = trace_event_buffer_lock_reserve(&buffer, trace_file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) call->event.type, size, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) entry = ring_buffer_event_data(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) if (is_ret_probe(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) entry->vaddr[0] = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) entry->vaddr[1] = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) data = DATAOF_TRACE_ENTRY(entry, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) entry->vaddr[0] = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) data = DATAOF_TRACE_ENTRY(entry, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) memcpy(data, ucb->buf, tu->tp.size + dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) event_trigger_unlock_commit(trace_file, buffer, event, entry, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /* uprobe handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) struct uprobe_cpu_buffer *ucb, int dsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) struct event_file_link *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (is_ret_probe(tu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) trace_probe_for_each_link_rcu(link, &tu->tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct uprobe_cpu_buffer *ucb, int dsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) struct event_file_link *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) trace_probe_for_each_link_rcu(link, &tu->tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) /* Event entry printers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static enum print_line_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) struct uprobe_trace_entry_head *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct trace_seq *s = &iter->seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) u8 *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) entry = (struct uprobe_trace_entry_head *)iter->ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) tu = trace_uprobe_primary_from_call(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) container_of(event, struct trace_event_call, event));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) if (unlikely(!tu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (is_ret_probe(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) trace_probe_name(&tu->tp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) entry->vaddr[1], entry->vaddr[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) data = DATAOF_TRACE_ENTRY(entry, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) trace_seq_printf(s, "%s: (0x%lx)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) trace_probe_name(&tu->tp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) entry->vaddr[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) data = DATAOF_TRACE_ENTRY(entry, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) trace_seq_putc(s, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return trace_handle_return(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) typedef bool (*filter_func_t)(struct uprobe_consumer *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) enum uprobe_filter_ctx ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) struct mm_struct *mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) tu->consumer.filter = filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) tu->inode = d_real_inode(tu->path.dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) if (tu->ref_ctr_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) ret = uprobe_register_refctr(tu->inode, tu->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) tu->ref_ctr_offset, &tu->consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) tu->inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) static void __probe_event_disable(struct trace_probe *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) struct trace_probe *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) tu = container_of(tp, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) tu = container_of(pos, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) if (!tu->inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) tu->inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) static int probe_event_enable(struct trace_event_call *call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) struct trace_event_file *file, filter_func_t filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) struct trace_probe *pos, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) bool enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) tp = trace_probe_primary_from_call(call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (WARN_ON_ONCE(!tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) enabled = trace_probe_is_enabled(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) /* This may also change "enabled" state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) if (file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) ret = trace_probe_add_file(tp, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) trace_probe_set_flag(tp, TP_FLAG_PROFILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) tu = container_of(tp, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) if (enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ret = uprobe_buffer_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) goto err_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) tu = container_of(pos, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) ret = trace_uprobe_enable(tu, filter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) __probe_event_disable(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) goto err_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) err_buffer:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) uprobe_buffer_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) err_flags:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) if (file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) trace_probe_remove_file(tp, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) static void probe_event_disable(struct trace_event_call *call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct trace_event_file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) struct trace_probe *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) tp = trace_probe_primary_from_call(call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (WARN_ON_ONCE(!tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (!trace_probe_is_enabled(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) if (file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (trace_probe_remove_file(tp, file) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (trace_probe_is_enabled(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) __probe_event_disable(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) uprobe_buffer_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) static int uprobe_event_define_fields(struct trace_event_call *event_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) int ret, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) struct uprobe_trace_entry_head field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) tu = trace_uprobe_primary_from_call(event_call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) if (unlikely(!tu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (is_ret_probe(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) size = SIZEOF_TRACE_ENTRY(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) size = SIZEOF_TRACE_ENTRY(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return traceprobe_define_arg_fields(event_call, size, &tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) #ifdef CONFIG_PERF_EVENTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) struct perf_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (filter->nr_systemwide)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if (event->hw.target->mm == mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) static inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return __uprobe_perf_filter(filter, event->hw.target->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) bool done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) write_lock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (event->hw.target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) list_del(&event->hw.tp_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) done = filter->nr_systemwide ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) (event->hw.target->flags & PF_EXITING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) trace_uprobe_filter_event(filter, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) filter->nr_systemwide--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) done = filter->nr_systemwide;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) write_unlock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) return done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) /* This returns true if the filter always covers target mm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) bool done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) write_lock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (event->hw.target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) * event->parent != NULL means copy_process(), we can avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) * uprobe_apply(). current->mm must be probed and we can rely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * on dup_mmap() which preserves the already installed bp's.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * attr.enable_on_exec means that exec/mmap will install the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) * breakpoints we need.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) done = filter->nr_systemwide ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) event->parent || event->attr.enable_on_exec ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) trace_uprobe_filter_event(filter, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) list_add(&event->hw.tp_list, &filter->perf_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) done = filter->nr_systemwide;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) filter->nr_systemwide++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) write_unlock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) return done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) static int uprobe_perf_close(struct trace_event_call *call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) struct trace_probe *pos, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) tp = trace_probe_primary_from_call(call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) if (WARN_ON_ONCE(!tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) tu = container_of(tp, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) tu = container_of(pos, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) ret = uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) static int uprobe_perf_open(struct trace_event_call *call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) struct trace_probe *pos, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) tp = trace_probe_primary_from_call(call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) if (WARN_ON_ONCE(!tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) tu = container_of(tp, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) if (trace_uprobe_filter_add(tu->tp.event->filter, event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) tu = container_of(pos, struct trace_uprobe, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) uprobe_perf_close(call, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) static bool uprobe_perf_filter(struct uprobe_consumer *uc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) enum uprobe_filter_ctx ctx, struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) struct trace_uprobe_filter *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) tu = container_of(uc, struct trace_uprobe, consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) filter = tu->tp.event->filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) read_lock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) ret = __uprobe_perf_filter(filter, mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) read_unlock(&filter->rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) static void __uprobe_perf_func(struct trace_uprobe *tu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) unsigned long func, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) struct uprobe_cpu_buffer *ucb, int dsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) struct trace_event_call *call = trace_probe_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) struct uprobe_trace_entry_head *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) int size, esize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) int rctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) if (bpf_prog_array_valid(call)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) u32 ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) ret = trace_call_bpf(call, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) size = esize + tu->tp.size + dsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) head = this_cpu_ptr(call->perf_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) if (hlist_empty(head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) entry = perf_trace_buf_alloc(size, NULL, &rctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (is_ret_probe(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) entry->vaddr[0] = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) entry->vaddr[1] = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) data = DATAOF_TRACE_ENTRY(entry, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) entry->vaddr[0] = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) data = DATAOF_TRACE_ENTRY(entry, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) memcpy(data, ucb->buf, tu->tp.size + dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) if (size - esize > tu->tp.size + dsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) int len = tu->tp.size + dsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) memset(data + len, 0, size - esize - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) head, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) /* uprobe profile handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) struct uprobe_cpu_buffer *ucb, int dsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) return UPROBE_HANDLER_REMOVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (!is_ret_probe(tu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) __uprobe_perf_func(tu, 0, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) struct uprobe_cpu_buffer *ucb, int dsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) __uprobe_perf_func(tu, func, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) const char **filename, u64 *probe_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) bool perf_type_tracepoint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) const char *pevent = trace_event_name(event->tp_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) const char *group = event->tp_event->class->system;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) if (perf_type_tracepoint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) tu = find_probe_event(pevent, group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) tu = trace_uprobe_primary_from_call(event->tp_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) if (!tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) : BPF_FD_TYPE_UPROBE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) *filename = tu->filename;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) *probe_offset = tu->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) #endif /* CONFIG_PERF_EVENTS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) struct trace_event_file *file = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) case TRACE_REG_REGISTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return probe_event_enable(event, file, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) case TRACE_REG_UNREGISTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) probe_event_disable(event, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) #ifdef CONFIG_PERF_EVENTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) case TRACE_REG_PERF_REGISTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) return probe_event_enable(event, NULL, uprobe_perf_filter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) case TRACE_REG_PERF_UNREGISTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) probe_event_disable(event, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) case TRACE_REG_PERF_OPEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) return uprobe_perf_open(event, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) case TRACE_REG_PERF_CLOSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) return uprobe_perf_close(event, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) struct uprobe_dispatch_data udd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) struct uprobe_cpu_buffer *ucb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) int dsize, esize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) tu = container_of(con, struct trace_uprobe, consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) tu->nhit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) udd.tu = tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) udd.bp_addr = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) current->utask->vaddr = (unsigned long) &udd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) if (WARN_ON_ONCE(!uprobe_cpu_buffer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) dsize = __get_data_size(&tu->tp, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) ucb = uprobe_buffer_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) ret |= uprobe_trace_func(tu, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) #ifdef CONFIG_PERF_EVENTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) ret |= uprobe_perf_func(tu, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) uprobe_buffer_put(ucb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) static int uretprobe_dispatcher(struct uprobe_consumer *con,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) unsigned long func, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) struct uprobe_dispatch_data udd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) struct uprobe_cpu_buffer *ucb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) int dsize, esize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) tu = container_of(con, struct trace_uprobe, consumer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) udd.tu = tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) udd.bp_addr = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) current->utask->vaddr = (unsigned long) &udd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (WARN_ON_ONCE(!uprobe_cpu_buffer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) dsize = __get_data_size(&tu->tp, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) ucb = uprobe_buffer_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) uretprobe_trace_func(tu, func, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) #ifdef CONFIG_PERF_EVENTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) uretprobe_perf_func(tu, func, regs, ucb, dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) uprobe_buffer_put(ucb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) static struct trace_event_functions uprobe_funcs = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) .trace = print_uprobe_event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) static struct trace_event_fields uprobe_fields_array[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) { .type = TRACE_FUNCTION_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) .define_fields = uprobe_event_define_fields },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) static inline void init_trace_event_call(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) struct trace_event_call *call = trace_probe_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) call->event.funcs = &uprobe_funcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) call->class->fields_array = uprobe_fields_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) call->class->reg = trace_uprobe_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) static int register_uprobe_event(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) init_trace_event_call(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) return trace_probe_register_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) static int unregister_uprobe_event(struct trace_uprobe *tu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) return trace_probe_unregister_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) #ifdef CONFIG_PERF_EVENTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) struct trace_event_call *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) create_local_trace_uprobe(char *name, unsigned long offs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) unsigned long ref_ctr_offset, bool is_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) struct path path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) ret = kern_path(name, LOOKUP_FOLLOW, &path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) if (!d_is_reg(path.dentry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) path_put(&path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * local trace_kprobes are not added to dyn_event, so they are never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) * searched in find_trace_kprobe(). Therefore, there is no concern of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * duplicated name "DUMMY_EVENT" here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) is_return);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) if (IS_ERR(tu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) pr_info("Failed to allocate trace_uprobe.(%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) (int)PTR_ERR(tu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) path_put(&path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) return ERR_CAST(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) tu->offset = offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) tu->path = path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) tu->ref_ctr_offset = ref_ctr_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) tu->filename = kstrdup(name, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) init_trace_event_call(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) return trace_probe_event_call(&tu->tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) free_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) void destroy_local_trace_uprobe(struct trace_event_call *event_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) struct trace_uprobe *tu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) tu = trace_uprobe_primary_from_call(event_call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) free_trace_uprobe(tu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) #endif /* CONFIG_PERF_EVENTS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) /* Make a trace interface for controling probe points */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) static __init int init_uprobe_trace(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) ret = dyn_event_register(&trace_uprobe_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) ret = tracing_init_dentry();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) trace_create_file("uprobe_events", 0644, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) NULL, &uprobe_events_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) /* Profile interface */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) trace_create_file("uprobe_profile", 0444, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) NULL, &uprobe_profile_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) fs_initcall(init_uprobe_trace);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags