^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #define pr_fmt(fmt) "kcov: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #define DISABLE_BRANCH_PROFILING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/hashtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/printk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/kcov.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/setup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /* Number of 64-bit words written per one comparison: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define KCOV_WORDS_PER_CMP 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * kcov descriptor (one per opened debugfs file).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * State transitions of the descriptor:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * - initial state after open()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * - then there must be a single ioctl(KCOV_INIT_TRACE) call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * - then, mmap() call (several calls are allowed but not useful)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * - then, ioctl(KCOV_ENABLE, arg), where arg is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * KCOV_TRACE_PC - to trace only the PCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * KCOV_TRACE_CMP - to trace only the comparison operands
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * - then, ioctl(KCOV_DISABLE) to disable the task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * Enabling/disabling ioctls can be repeated (only one task a time allowed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct kcov {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * Reference counter. We keep one for:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * - opened file descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * - task with enabled coverage (we can't unwire it from another task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * - each code section for remote coverage collection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) refcount_t refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* The lock protects mode, size, area and t. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) enum kcov_mode mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /* Size of arena (in long's). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /* Coverage buffer shared with user space. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) void *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /* Task for which we collect coverage, or NULL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* Collecting coverage from remote (background) threads. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) bool remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* Size of remote area (in long's). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned int remote_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Sequence is incremented each time kcov is reenabled, used by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * kcov_remote_stop(), see the comment there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) int sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct kcov_remote_area {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct kcov_remote {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u64 handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct hlist_node hnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static DEFINE_SPINLOCK(kcov_remote_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static DEFINE_HASHTABLE(kcov_remote_map, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct kcov_percpu_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) void *irq_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) unsigned int saved_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned int saved_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) void *saved_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct kcov *saved_kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) int saved_sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /* Must be called with kcov_remote_lock locked. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static struct kcov_remote *kcov_remote_find(u64 handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct kcov_remote *remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) hash_for_each_possible(kcov_remote_map, remote, hnode, handle) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (remote->handle == handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /* Must be called with kcov_remote_lock locked. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct kcov_remote *remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (kcov_remote_find(handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return ERR_PTR(-EEXIST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) remote = kmalloc(sizeof(*remote), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (!remote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) remote->handle = handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) remote->kcov = kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) hash_add(kcov_remote_map, &remote->hnode, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* Must be called with kcov_remote_lock locked. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct kcov_remote_area *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct list_head *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) list_for_each(pos, &kcov_remote_areas) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) area = list_entry(pos, struct kcov_remote_area, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (area->size == size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) list_del(&area->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /* Must be called with kcov_remote_lock locked. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void kcov_remote_area_put(struct kcov_remote_area *area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) INIT_LIST_HEAD(&area->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) area->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) list_add(&area->list, &kcov_remote_areas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) unsigned int mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * We are interested in code coverage as a function of a syscall inputs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * so we ignore code executed in interrupts, unless we are in a remote
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * coverage collection section in a softirq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (!in_task() && !(in_serving_softirq() && t->kcov_softirq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) mode = READ_ONCE(t->kcov_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * There is some code that runs in interrupts but for which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * in_interrupt() returns false (e.g. preempt_schedule_irq()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * READ_ONCE()/barrier() effectively provides load-acquire wrt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * interrupts, there are paired barrier()/WRITE_ONCE() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * kcov_start().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return mode == needed_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static notrace unsigned long canonicalize_ip(unsigned long ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #ifdef CONFIG_RANDOMIZE_BASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) ip -= kaslr_offset();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * Entry point from instrumented code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * This is called once per basic-block/edge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) void notrace __sanitizer_cov_trace_pc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) unsigned long *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned long ip = canonicalize_ip(_RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) unsigned long pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) area = t->kcov_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* The first 64-bit word is the number of subsequent PCs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pos = READ_ONCE(area[0]) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (likely(pos < t->kcov_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) area[pos] = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) WRITE_ONCE(area[0], pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) u64 *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) u64 count, start_index, end_pos, max_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) ip = canonicalize_ip(ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * We write all comparison arguments and types as u64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * The buffer was allocated for t->kcov_size unsigned longs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) area = (u64 *)t->kcov_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) max_pos = t->kcov_size * sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) count = READ_ONCE(area[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) start_index = 1 + count * KCOV_WORDS_PER_CMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (likely(end_pos <= max_pos)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) area[start_index] = type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) area[start_index + 1] = arg1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) area[start_index + 2] = arg2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) area[start_index + 3] = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) WRITE_ONCE(area[0], count + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) u64 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) u64 count = cases[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) u64 size = cases[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) u64 type = KCOV_CMP_CONST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) switch (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) type |= KCOV_CMP_SIZE(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) case 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) type |= KCOV_CMP_SIZE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) case 32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) type |= KCOV_CMP_SIZE(2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) case 64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) type |= KCOV_CMP_SIZE(3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) for (i = 0; i < count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) write_comp_data(type, cases[i + 2], val, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static void kcov_start(struct task_struct *t, struct kcov *kcov,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) unsigned int size, void *area, enum kcov_mode mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) int sequence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) kcov_debug("t = %px, size = %u, area = %px\n", t, size, area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) t->kcov = kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /* Cache in task struct for performance. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) t->kcov_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) t->kcov_area = area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) t->kcov_sequence = sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /* See comment in check_kcov_mode(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) WRITE_ONCE(t->kcov_mode, mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static void kcov_stop(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) t->kcov = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) t->kcov_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) t->kcov_area = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static void kcov_task_reset(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) kcov_stop(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) t->kcov_sequence = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) t->kcov_handle = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) void kcov_task_init(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) kcov_task_reset(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) t->kcov_handle = current->kcov_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static void kcov_reset(struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) kcov->t = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) kcov->mode = KCOV_MODE_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) kcov->remote = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) kcov->remote_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) kcov->sequence++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static void kcov_remote_reset(struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int bkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct kcov_remote *remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) spin_lock_irqsave(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (remote->kcov != kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) hash_del(&remote->hnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) kfree(remote);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* Do reset before unlock to prevent races with kcov_remote_start(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) kcov_reset(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) spin_unlock_irqrestore(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static void kcov_disable(struct task_struct *t, struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) kcov_task_reset(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (kcov->remote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) kcov_remote_reset(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) kcov_reset(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void kcov_get(struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) refcount_inc(&kcov->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) static void kcov_put(struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (refcount_dec_and_test(&kcov->refcount)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) kcov_remote_reset(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) vfree(kcov->area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) kfree(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) void kcov_task_exit(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) kcov = t->kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (kcov == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) spin_lock_irqsave(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * which comes down to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * WARN_ON(!kcov->remote && kcov->t != t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * For KCOV_REMOTE_ENABLE devices, the exiting task is either:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * 1. A remote task between kcov_remote_start() and kcov_remote_stop().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * In this case we should print a warning right away, since a task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * shouldn't be exiting when it's in a kcov coverage collection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * section. Here t points to the task that is collecting remote
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * coverage, and t->kcov->t points to the thread that created the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * kcov device. Which means that to detect this case we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * check that t != t->kcov->t, and this gives us the following:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * WARN_ON(kcov->remote && kcov->t != t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * 2. The task that created kcov exiting without calling KCOV_DISABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * and then again we make sure that t->kcov->t == t:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * WARN_ON(kcov->remote && kcov->t != t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * By combining all three checks into one we get:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (WARN_ON(kcov->t != t)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) spin_unlock_irqrestore(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Just to not leave dangling references behind. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) spin_unlock_irqrestore(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) kcov_put(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) int res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) void *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct kcov *kcov = vma->vm_file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) unsigned long size, off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) area = vmalloc_user(vma->vm_end - vma->vm_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (!area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) spin_lock_irqsave(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) size = kcov->size * sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) vma->vm_end - vma->vm_start != size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) res = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (!kcov->area) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) kcov->area = area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) vma->vm_flags |= VM_DONTEXPAND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) spin_unlock_irqrestore(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) for (off = 0; off < size; off += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) page = vmalloc_to_page(kcov->area + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (vm_insert_page(vma, vma->vm_start + off, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) WARN_ONCE(1, "vm_insert_page() failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) spin_unlock_irqrestore(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) vfree(area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static int kcov_open(struct inode *inode, struct file *filep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if (!kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) kcov->mode = KCOV_MODE_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) kcov->sequence = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) refcount_set(&kcov->refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) spin_lock_init(&kcov->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) filep->private_data = kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return nonseekable_open(inode, filep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static int kcov_close(struct inode *inode, struct file *filep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) kcov_put(filep->private_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static int kcov_get_mode(unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (arg == KCOV_TRACE_PC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return KCOV_MODE_TRACE_PC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) else if (arg == KCOV_TRACE_CMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) return KCOV_MODE_TRACE_CMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * Fault in a lazily-faulted vmalloc area before it can be used by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * vmalloc fault handling path is instrumented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static void kcov_fault_in_area(struct kcov *kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) unsigned long *area = kcov->area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) unsigned long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) for (offset = 0; offset < kcov->size; offset += stride)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) READ_ONCE(area[offset]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static inline bool kcov_check_handle(u64 handle, bool common_valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) bool uncommon_valid, bool zero_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) switch (handle & KCOV_SUBSYSTEM_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) case KCOV_SUBSYSTEM_COMMON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return (handle & KCOV_INSTANCE_MASK) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) common_valid : zero_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) case KCOV_SUBSYSTEM_USB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return uncommon_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) unsigned long size, unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) int mode, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct kcov_remote_arg *remote_arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct kcov_remote *remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) case KCOV_INIT_TRACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * Enable kcov in trace mode and setup buffer size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * Must happen before anything else.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (kcov->mode != KCOV_MODE_DISABLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * Size must be at least 2 to hold current position and one PC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) * Later we allocate size * sizeof(unsigned long) memory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * that must not overflow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) size = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (size < 2 || size > INT_MAX / sizeof(unsigned long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) kcov->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) kcov->mode = KCOV_MODE_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) case KCOV_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Enable coverage for the current task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * At this point user must have been enabled trace mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * and mmapped the file. Coverage collection is disabled only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * at task exit or voluntary by KCOV_DISABLE. After that it can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * be enabled for another task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (kcov->t != NULL || t->kcov != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) mode = kcov_get_mode(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (mode < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) kcov_fault_in_area(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) kcov->mode = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) kcov->sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) kcov->t = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /* Put either in kcov_task_exit() or in KCOV_DISABLE. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) kcov_get(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) case KCOV_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* Disable coverage for the current task. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) unused = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (unused != 0 || current->kcov != kcov)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (WARN_ON(kcov->t != t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) kcov_put(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) case KCOV_REMOTE_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (kcov->t != NULL || t->kcov != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) remote_arg = (struct kcov_remote_arg *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) mode = kcov_get_mode(remote_arg->trace_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (mode < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) kcov->mode = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) t->kcov = kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) kcov->t = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) kcov->remote = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) kcov->remote_size = remote_arg->area_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) spin_lock_irqsave(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) for (i = 0; i < remote_arg->num_handles; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (!kcov_check_handle(remote_arg->handles[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) false, true, false)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) spin_unlock_irqrestore(&kcov_remote_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) remote = kcov_remote_add(kcov, remote_arg->handles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (IS_ERR(remote)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) spin_unlock_irqrestore(&kcov_remote_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return PTR_ERR(remote);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (remote_arg->common_handle) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (!kcov_check_handle(remote_arg->common_handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) true, false, false)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) spin_unlock_irqrestore(&kcov_remote_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) remote = kcov_remote_add(kcov,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) remote_arg->common_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (IS_ERR(remote)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) spin_unlock_irqrestore(&kcov_remote_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) kcov_disable(t, kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return PTR_ERR(remote);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) t->kcov_handle = remote_arg->common_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) spin_unlock_irqrestore(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* Put either in kcov_task_exit() or in KCOV_DISABLE. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) kcov_get(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return -ENOTTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) int res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct kcov_remote_arg *remote_arg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) unsigned int remote_num_handles;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) unsigned long remote_arg_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (cmd == KCOV_REMOTE_ENABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (get_user(remote_num_handles, (unsigned __user *)(arg +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) offsetof(struct kcov_remote_arg, num_handles))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) remote_arg_size = struct_size(remote_arg, handles,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) remote_num_handles);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) remote_arg = memdup_user((void __user *)arg, remote_arg_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) if (IS_ERR(remote_arg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return PTR_ERR(remote_arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) if (remote_arg->num_handles != remote_num_handles) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) kfree(remote_arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) arg = (unsigned long)remote_arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) kcov = filep->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) spin_lock_irqsave(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) res = kcov_ioctl_locked(kcov, cmd, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) spin_unlock_irqrestore(&kcov->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) kfree(remote_arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) static const struct file_operations kcov_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) .open = kcov_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) .unlocked_ioctl = kcov_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) .compat_ioctl = kcov_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) .mmap = kcov_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) .release = kcov_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * of code in a kernel background thread or in a softirq to allow kcov to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * used to collect coverage from that part of code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * The handle argument of kcov_remote_start() identifies a code section that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * used for coverage collection. A userspace process passes this handle to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * coverage for the code section identified by this handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * The usage of these annotations in the kernel code is different depending on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * the type of the kernel thread whose code is being annotated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * For global kernel threads that are spawned in a limited number of instances
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * softirqs, each instance must be assigned a unique 4-byte instance id. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * instance id is then combined with a 1-byte subsystem id to get a handle via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * kcov_remote_handle(subsystem_id, instance_id).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * For local kernel threads that are spawned from system calls handler when a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * user interacts with some kernel interface (e.g. vhost workers), a handle is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * passed from a userspace process as the common_handle field of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * kcov_remote_arg struct (note, that the user must generate a handle by using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * arbitrary 4-byte non-zero number as the instance id). This common handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * then gets saved into the task_struct of the process that issued the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * KCOV_REMOTE_ENABLE ioctl. When this process issues system calls that spawn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * kernel threads, the common handle must be retrieved via kcov_common_handle()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * and passed to the spawned threads via custom annotations. Those kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * threads must in turn be annotated with kcov_remote_start(common_handle) and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * kcov_remote_stop(). All of the threads that are spawned by the same process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * obtain the same handle, hence the name "common".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * See Documentation/dev-tools/kcov.rst for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * Internally, kcov_remote_start() looks up the kcov device associated with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * provided handle, allocates an area for coverage collection, and saves the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * pointers to kcov and area into the current task_struct to allow coverage to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * be collected via __sanitizer_cov_trace_pc().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * In turns kcov_remote_stop() clears those pointers from task_struct to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * collecting coverage and copies all collected coverage into the kcov area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) static inline bool kcov_mode_enabled(unsigned int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) static void kcov_remote_softirq_start(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) unsigned int mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) mode = READ_ONCE(t->kcov_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (kcov_mode_enabled(mode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) data->saved_mode = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) data->saved_size = t->kcov_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) data->saved_area = t->kcov_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) data->saved_sequence = t->kcov_sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) data->saved_kcov = t->kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) kcov_stop(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static void kcov_remote_softirq_stop(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (data->saved_kcov) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) kcov_start(t, data->saved_kcov, data->saved_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) data->saved_area, data->saved_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) data->saved_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) data->saved_mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) data->saved_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) data->saved_area = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) data->saved_sequence = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) data->saved_kcov = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) void kcov_remote_start(u64 handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct task_struct *t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct kcov_remote *remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) unsigned int mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) void *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) int sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (WARN_ON(!kcov_check_handle(handle, true, true, true)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (!in_task() && !in_serving_softirq())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * Check that kcov_remote_start() is not called twice in background
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * threads nor called by user tasks (with enabled kcov).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) mode = READ_ONCE(t->kcov_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (WARN_ON(in_task() && kcov_mode_enabled(mode))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * Check that kcov_remote_start() is not called twice in softirqs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * Note, that kcov_remote_start() can be called from a softirq that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * happened while collecting coverage from a background thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) spin_lock(&kcov_remote_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) remote = kcov_remote_find(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (!remote) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) spin_unlock_irqrestore(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) kcov_debug("handle = %llx, context: %s\n", handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) in_task() ? "task" : "softirq");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) kcov = remote->kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) /* Put in kcov_remote_stop(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) kcov_get(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * Read kcov fields before unlock to prevent races with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * KCOV_DISABLE / kcov_remote_reset().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mode = kcov->mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) sequence = kcov->sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) if (in_task()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) size = kcov->remote_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) area = kcov_remote_area_get(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) size = CONFIG_KCOV_IRQ_AREA_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) area = this_cpu_ptr(&kcov_percpu_data)->irq_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) spin_unlock_irqrestore(&kcov_remote_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /* Can only happen when in_task(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (!area) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) area = vmalloc(size * sizeof(unsigned long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (!area) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) kcov_put(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /* Reset coverage size. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) *(u64 *)area = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if (in_serving_softirq()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) kcov_remote_softirq_start(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) t->kcov_softirq = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) kcov_start(t, kcov, size, area, mode, sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) EXPORT_SYMBOL(kcov_remote_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) static void kcov_move_area(enum kcov_mode mode, void *dst_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) unsigned int dst_area_size, void *src_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) u64 word_size = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) u64 count_size, entry_size_log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) u64 dst_len, src_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) void *dst_entries, *src_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) u64 dst_occupied, dst_free, bytes_to_move, entries_moved;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) kcov_debug("%px %u <= %px %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) dst_area, dst_area_size, src_area, *(unsigned long *)src_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) case KCOV_MODE_TRACE_PC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) dst_len = READ_ONCE(*(unsigned long *)dst_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) src_len = *(unsigned long *)src_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) count_size = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) entry_size_log = __ilog2_u64(sizeof(unsigned long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) case KCOV_MODE_TRACE_CMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) dst_len = READ_ONCE(*(u64 *)dst_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) src_len = *(u64 *)src_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) count_size = sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) /* As arm can't divide u64 integers use log of entry size. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (dst_len > ((dst_area_size * word_size - count_size) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) entry_size_log))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) dst_occupied = count_size + (dst_len << entry_size_log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) dst_free = dst_area_size * word_size - dst_occupied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) bytes_to_move = min(dst_free, src_len << entry_size_log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) dst_entries = dst_area + dst_occupied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) src_entries = src_area + count_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) memcpy(dst_entries, src_entries, bytes_to_move);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) entries_moved = bytes_to_move >> entry_size_log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) case KCOV_MODE_TRACE_PC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) case KCOV_MODE_TRACE_CMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) /* See the comment before kcov_remote_start() for usage details. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) void kcov_remote_stop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) struct task_struct *t = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) struct kcov *kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) unsigned int mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) void *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) int sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (!in_task() && !in_serving_softirq())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) mode = READ_ONCE(t->kcov_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (!kcov_mode_enabled(mode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * When in softirq, check if the corresponding kcov_remote_start()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * actually found the remote handle and started collecting coverage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (in_serving_softirq() && !t->kcov_softirq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) /* Make sure that kcov_softirq is only set when in softirq. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) kcov = t->kcov;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) area = t->kcov_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) size = t->kcov_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) sequence = t->kcov_sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) kcov_stop(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) if (in_serving_softirq()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) t->kcov_softirq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) kcov_remote_softirq_stop(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) spin_lock(&kcov->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * KCOV_DISABLE could have been called between kcov_remote_start()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * and kcov_remote_stop(), hence the sequence check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (sequence == kcov->sequence && kcov->remote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) kcov_move_area(kcov->mode, kcov->area, kcov->size, area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) spin_unlock(&kcov->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) if (in_task()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) spin_lock(&kcov_remote_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) kcov_remote_area_put(area, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) spin_unlock(&kcov_remote_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) /* Get in kcov_remote_start(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) kcov_put(kcov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) EXPORT_SYMBOL(kcov_remote_stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) /* See the comment before kcov_remote_start() for usage details. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) u64 kcov_common_handle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return current->kcov_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) EXPORT_SYMBOL(kcov_common_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) static int __init kcov_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) void *area = vmalloc(CONFIG_KCOV_IRQ_AREA_SIZE *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) sizeof(unsigned long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (!area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * The kcov debugfs file won't ever get removed and thus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * there is no need to protect it against removal races. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * use of debugfs_create_file_unsafe() is actually safe here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) device_initcall(kcov_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags